Multimodal AI

# vision.py - fetch an image, encode it, ask about it

import base64

import anthropic
import httpx

client = anthropic.Anthropic()

# Any reachable image works. Fetch it, then base64-encode the bytes.
url = "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"
image_data = base64.standard_b64encode(httpx.get(url).content).decode("utf-8")

message = client.messages.create(
    model="claude-sonnet-5",
    max_tokens=1024,
    messages=[{
        "role": "user",
        "content": [
            {
                "type": "image",
                "source": {
                    "type": "base64",
                    "media_type": "image/png",
                    "data": image_data,
                },
            },
            {"type": "text", "text": "Describe what this image shows in one sentence."},
        ],
    }],
)
print(next(b.text for b in message.content if b.type == "text"))
print(f"input tokens: {message.usage.input_tokens}")

# You can also hand the model a URL and skip the download:
#   {"type": "image", "source": {"type": "url", "url": url}}
# It is fetched server-side, so it only works if the host serves that
# request; some CDNs refuse it and the call fails with a 400.

# extract_invoice.py - PDF in, validated Invoice object out

import base64
import json

import anthropic
from pydantic import BaseModel
from reportlab.lib.pagesizes import LETTER
from reportlab.pdfgen import canvas

client = anthropic.Anthropic()

# --- Build an invoice PDF so the example runs with no external file ---
ROWS = [("Espresso beans 1kg", 2, 18.50), ("Paper filters", 3, 4.25),
        ("Ceramic mug", 1, 14.00)]

c = canvas.Canvas("invoice.pdf", pagesize=LETTER)
c.setFont("Helvetica-Bold", 16)
c.drawString(72, 720, "Blue Bottle Coffee")
c.setFont("Helvetica", 11)
c.drawString(72, 690, "Invoice INV-2044        Date 2026-08-11")
y = 650
for description, qty, unit in ROWS:
    c.drawString(72, y, f"{description:<24} {qty:>3} x {unit:>6.2f} = {qty * unit:>7.2f}")
    y -= 20
c.drawString(72, y - 20, f"TOTAL {sum(q * u for _, q, u in ROWS):.2f}")
c.save()


class LineItem(BaseModel):
    description: str
    qty: int
    unit_price: float


class Invoice(BaseModel):
    invoice_number: str
    date: str
    vendor: str
    total: float
    line_items: list[LineItem]


with open("invoice.pdf", "rb") as f:
    pdf_data = base64.standard_b64encode(f.read()).decode("utf-8")

# messages.parse applies the schema to the model's output, so what comes
# back is a validated Invoice, not JSON you still have to check.
response = client.messages.parse(
    model="claude-sonnet-5",
    max_tokens=1024,
    messages=[{
        "role": "user",
        "content": [
            {
                "type": "document",
                "source": {
                    "type": "base64",
                    "media_type": "application/pdf",
                    "data": pdf_data,
                },
            },
            {"type": "text", "text": "Extract this invoice."},
        ],
    }],
    output_format=Invoice,
)

invoice = response.parsed_output
print(json.dumps(invoice.model_dump(), indent=2))

# Claude reads the PDF natively, both its text layer AND its rendered
# pages, so tables, columns, and image-only scans work without a
# separate OCR step.

# call_summary.py - synthesize a call, transcribe it, then reason over the text

# Claude has no audio input, so speech is transcribed first by a dedicated
# service and Claude reasons over the text.

import anthropic
from openai import OpenAI

audio_api = OpenAI()          # STT and TTS live here
claude = anthropic.Anthropic()

SCRIPT = ("Hi, this is Dana from Northwind. Our checkout page has been returning "
          "500 errors since this morning and we are losing sales. I will escalate "
          "this to engineering now and we will have a fix deployed by Friday.")

# 0. Manufacture a recording, so the example needs no audio file of your own.
#    This is also the text-to-speech direction: Claude's output can be spoken
#    the same way.
audio_api.audio.speech.create(
    model="gpt-4o-mini-tts", voice="alloy", input=SCRIPT,
).write_to_file("support_call.mp3")

# 1. Speech -> text
with open("support_call.mp3", "rb") as audio:
    transcript = audio_api.audio.transcriptions.create(
        model="gpt-4o-transcribe",   # whisper-1 still works; this is the current model
        file=audio,
    ).text
print(f"transcript: {transcript[:70]}...")

# 2. Text -> analysis (this is where Claude excels)
reply = claude.messages.create(
    model="claude-sonnet-5",
    max_tokens=512,
    messages=[{"role": "user", "content": (
        "Summarize this support call in one line, then list any commitments "
        f"made as short bullets. No headings.\n\n{transcript}"
    )}],
)
print(next(b.text for b in reply.content if b.type == "text"))

# icon_pipeline.py - Claude writes the prompt, the image model renders, Claude reviews

# Claude cannot generate images. When a workflow needs one, route that step
# to an image model, exactly the hybrid routing pattern from the
# "Building AI Applications" lesson.

import base64

import anthropic
from openai import OpenAI

claude = anthropic.Anthropic()
image_client = OpenAI()

# 1. Claude turns a vague request into a detailed image prompt.
brief = "an icon for the shipping section of our app"
reply = claude.messages.create(
    model="claude-sonnet-5",
    max_tokens=200,
    messages=[{"role": "user", "content": (
        f"Write a single-sentence image generation prompt for: {brief}. "
        "Flat vector style, green on white. Return only the prompt."
    )}],
)
prompt = next(b.text for b in reply.content if b.type == "text").strip()
print(f"prompt: {prompt}")

# 2. The image model renders it.
result = image_client.images.generate(
    model="gpt-image-1.5",         # or Google Imagen, Stability, etc.
    prompt=prompt,
    size="1024x1024",
    quality="low",
)

# gpt-image models return base64 only; there is no response_format="url".
image_bytes = base64.b64decode(result.data[0].b64_json)
with open("box.png", "wb") as f:
    f.write(image_bytes)
print(f"saved box.png ({len(image_bytes) // 1024} KB)")

# 3. Claude's vision reviews the result and can request a revision.
review = claude.messages.create(
    model="claude-sonnet-5",
    max_tokens=200,
    messages=[{"role": "user", "content": [
        {"type": "image", "source": {
            "type": "base64", "media_type": "image/png",
            "data": base64.b64encode(image_bytes).decode("utf-8")}},
        {"type": "text", "text": (
            f"Does this image satisfy the brief '{brief}'? "
            "Answer in one sentence.")},
    ]}],
)
print(f"review: {next(b.text for b in review.content if b.type == 'text')}")