Vision Capabilities

from abstractcore import create_llm

# Works with any vision-capable provider
llm = create_llm("openai", model="gpt-4o")

# Single image analysis
response = llm.generate(
    "What objects do you see in this image?",
    media=["photo.jpg"]
)

# Multiple images comparison
response = llm.generate(
    "Compare these architectural styles and identify differences",
    media=["building1.jpg", "building2.jpg", "building3.jpg"]
)

# Same code works across all providers
image_files = ["chart.png", "document.pdf"]
prompt = "Analyze the data in these files"

# All work identically
openai_response = create_llm("openai", model="gpt-4o").generate(prompt, media=image_files)
anthropic_response = create_llm("anthropic", model="claude-3.5-sonnet").generate(prompt, media=image_files)
ollama_response = create_llm("ollama", model="qwen2.5vl:7b").generate(prompt, media=image_files)

# Option 1: Download local vision model (recommended)
abstractcore --download-vision-model

# Option 2: Use existing Ollama model
abstractcore --set-vision-caption qwen2.5vl:7b

# Option 3: Use cloud API
abstractcore --set-vision-provider openai --model gpt-4o

# Disable vision fallback
abstractcore --disable-vision

# After configuration, text-only models can process images seamlessly
text_llm = create_llm("lmstudio", model="qwen/qwen3-next-80b")  # No native vision

response = text_llm.generate(
    "What's happening in this image?",
    media=["complex_scene.jpg"]
)
# Works transparently: vision model analyzes image → text model processes description

# Images automatically optimized per model
llm = create_llm("openai", model="gpt-4o")
response = llm.generate("Analyze this", media=["photo.jpg"])  # Auto-resized to 4096×4096

llm = create_llm("ollama", model="qwen2.5vl:7b")
response = llm.generate("Analyze this", media=["photo.jpg"])  # Auto-resized to 3584×3584

# Get structured responses with specific requirements
llm = create_llm("openai", model="gpt-4o")

response = llm.generate("""
Analyze this image and provide:
- objects: list of objects detected
- colors: dominant colors
- setting: location/environment
- activities: what's happening

Format as JSON.
""", media=["scene.jpg"])

import json
analysis = json.loads(response.content)

# Compare multiple images
llm = create_llm("anthropic", model="claude-3.5-sonnet")

response = llm.generate(
    "Compare these three architectural designs and identify common elements",
    media=["design_a.jpg", "design_b.jpg", "design_c.jpg"]
)

# Analyze sequence of images
response = llm.generate(
    "Describe the progression shown in these time-lapse images",
    media=["hour1.jpg", "hour2.jpg", "hour3.jpg", "hour4.jpg"]
)

# Extract text from images
response = llm.generate(
    "Extract all text from this image and organize it",
    media=["receipt.jpg"]
)

# Handwriting recognition
response = llm.generate(
    "Transcribe the handwritten notes in this image",
    media=["notes.jpg"]
)

# Analyze visualizations
response = llm.generate(
    "What trends do you see in this chart? Provide the data points.",
    media=["sales_chart.png"]
)

# Compare visualizations
response = llm.generate(
    "Compare the trends shown in these two charts",
    media=["chart1.png", "chart2.png"]
)

# Defect detection
response = llm.generate(
    "Identify any defects or anomalies in this product image",
    media=["product_photo.jpg"]
)

# Comparison with standard
response = llm.generate(
    "Compare this sample with the reference and note any differences",
    media=["sample.jpg", "reference.jpg"]
)

# Detailed scene analysis
response = llm.generate(
    """Analyze this scene and provide:
    - Objects present
    - Activities occurring
    - Environmental conditions
    - Any safety concerns""",
    media=["scene.jpg"]
)

# Basic image analysis
python -m abstractcore.utils.cli --prompt "Describe this image @photo.jpg"

# Multiple images
python -m abstractcore.utils.cli --prompt "Compare @before.jpg and @after.jpg"

# Mixed media
python -m abstractcore.utils.cli --prompt "Verify the chart @chart.png matches @data.csv"