How to Work with Images using ChatGPT Python API

A practical example of how to use the vision capabilities of ChatGPT

The GPT-4o model has vision capabilities that enable us to answer questions about the images. Using the ChatGPT interface, you can upload the image and ask a question:

Python SKD

That’s cool, but it would be great if we could do the same task programmatically using the Python API. For this example, we assume that the user gets the images locally.

from openai import OpenAI

import base64
import requests
import os
# OpenAI API Key
api_key = os.environ['OPENAI_API_KEY']

# Function to encode the image
def encode_image(image_path):
  with open(image_path, "rb") as image_file:
    return base64.b64encode(image_file.read()).decode('utf-8')

# Path to your image
image_path = "images/dogs/image_1.jpg"

# Getting the base64 string
base64_image = encode_image(image_path)

headers = {
  "Content-Type": "application/json",
  "Authorization": f"Bearer {api_key}"
}

payload = {
  "model": "gpt-4o",
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "can you please describe the image?"
        },
        {
          "type": "image_url",
          "image_url": {
            "url": f"data:image/jpeg;base64,{base64_image}"
          }
        }
      ]
    }
  ],
  "max_tokens": 3000
}

response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)

print(response.json()['choices'][0]['message']['content'])

The image features two dogs of starkly contrasting sizes standing side by side against a neutral, dark background. The smaller dog, a Chihuahua, is light brown and white and is looking up at the larger dog. The larger dog, a Great Dane, is dark-colored and significantly taller, looking down at the Chihuahua. The size difference between the two dogs is quite pronounced, highlighting their distinct breeds and sizes.

As you can see, we managed to upload an image that was stored locally and call the ChatGPT Python API successfully. You can also upload images by passing a URL. For more information, please take a look at the documentation.

Streamlit Application

As a bonus part, we will build a Streamlit App that enables the end user to use the file browser to upload an image and get a description of it.

import streamlit as st
import os
from PIL import Image
import base64
import requests


# Function to list files in all directories under the current working directory
def list_files_in_all_directories(root_directory, extensions):
    files = []
    for dirpath, _, filenames in os.walk(root_directory):
        for file in filenames:
            if file.endswith(extensions):
                files.append(os.path.join(dirpath, file))
    return files

# Get the current working directory
current_directory = 'images/'
# List JPG and PNG files in all directories under the current working directory
files = list_files_in_all_directories(current_directory, (".jpg", ".jpeg", ".png"))

# Streamlit app
st.title("File Browser")

# Dropdown to select a file
selected_file = st.selectbox("Select a file", files)

# Display the selected image
if selected_file:
    image = Image.open(selected_file)
    st.image(image, caption=os.path.basename(selected_file))




# OpenAI API Key
api_key = os.environ['OPENAI_API_KEY']

# Function to encode the image
def encode_image(image_path):
  with open(image_path, "rb") as image_file:
    return base64.b64encode(image_file.read()).decode('utf-8')

# Path to your image
image_path = selected_file

# Getting the base64 string
base64_image = encode_image(image_path)

headers = {
  "Content-Type": "application/json",
  "Authorization": f"Bearer {api_key}"
}

payload = {
  "model": "gpt-4o",
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "Describe what you see in the image"
        },
        {
          "type": "image_url",
          "image_url": {
            "url": f"data:image/jpeg;base64,{base64_image}"
          }
        }
      ]
    }
  ],
  "max_tokens": 3000
}

response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
response = response.json()['choices'][0]['message']['content']

st.markdown(response)