Create an Image Labeling Application using Artificial Intelligence

I have a PowerPoint party to go to soon. Yes you read that right. At this party everyone is required to present a short presentation about any topic they want. Last year I made a really cute presentation about a day in the life of my dog.

This year I have decided that I want to bore everyone to death and talk about technology, Python, Terraform and Artificial Intelligence. Specifically, I built an application that allows a user to upload an image and have it return to them a renamed file that is labeled based on the object or scene in the image.

The architecture is fairly simple. We have a user connecting to a load balancer which routes traffic to our containers. The containers connect Bedrock and S3 for image.

If you want to try it out the site is hosted at https://image-labeler.vansledright.com It will be up for some time, I haven’t decided how long I will host it for but at least through this weekend!

Here is the code that interacts with Bedrock and S3 to process the image:

def process_image():
    if not request.is_json:
        return jsonify({'error': 'Content-Type must be application/json'}), 400

    data = request.json
    file_key = data.get('fileKey')

    if not file_key:
        return jsonify({'error': 'fileKey is required'}), 400

    try:
        # Get the image from S3
        response = s3.get_object(Bucket=app.config['S3_BUCKET_NAME'], Key=file_key)
        image_data = response['Body'].read()

        # Check if image is larger than 5MB
        if len(image_data) > 5 * 1024 * 1024:
            logger.info("File size to large. Compressing image")
            image_data = compress_image(image_data)

        # Convert image to base64
        base64_image = base64.b64encode(image_data).decode('utf-8')

        
        
        # Prepare prompt for Claude
        prompt = """Please analyze the image and identify the main object or subject. 
        Respond with just the object name in lowercase, hyphenated format. For example: 'coca-cola-can' or 'golden-retriever'."""
        
        # Call Bedrock with Claude
        response = bedrock.invoke_model(
            modelId='anthropic.claude-3-sonnet-20240229-v1:0',
            body=json.dumps({
                "anthropic_version": "bedrock-2023-05-31",
                "max_tokens": 100,
                "messages": [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "text",
                                "text": prompt
                            },
                            {
                                "type": "image",
                                "source": {
                                    "type": "base64",
                                    "media_type": response['ContentType'],
                                    "data": base64_image
                                }
                            }
                        ]
                    }
                ]
            })
        )
        
        response_body = json.loads(response['body'].read())
        object_name = response_body['content'][0]['text'].strip()
        logging.info(f"Object found is: {object_name}")
        
        if not object_name:
            return jsonify({'error': 'Could not identify object in image'}), 422

        # Get file extension and create new filename
        _, ext = os.path.splitext(unquote(file_key))
        new_file_name = f"{object_name}{ext}"
        new_file_key = f'processed/{new_file_name}'
        
        # Copy object to new location
        s3.copy_object(
            Bucket=app.config['S3_BUCKET_NAME'],
            CopySource={'Bucket': app.config['S3_BUCKET_NAME'], 'Key': file_key},
            Key=new_file_key
        )
        
        # Generate download URL
        download_url = s3.generate_presigned_url(
            'get_object',
            Params={
                'Bucket': app.config['S3_BUCKET_NAME'],
                'Key': new_file_key
            },
            ExpiresIn=3600
        )
        
        return jsonify({
            'downloadUrl': download_url,
            'newFileName': new_file_name
        })
        
    except json.JSONDecodeError as e:
        logger.error(f"Error decoding Bedrock response: {str(e)}")
        return jsonify({'error': 'Invalid response from AI service'}), 500
    except Exception as e:
        logger.error(f"Error processing image: {str(e)}")
        return jsonify({'error': 'Error processing image'}), 500

If you think this project is interesting, feel free to share it with your friends or message me if you want all of the code!