Building a Multimodal AI App with React 19 and GPT-4V: Developer Guide 2026

// Terminal — initialize the project
npm create vite@latest visual-qa-assistant -- --template react-ts
cd visual-qa-assistant
npm install openai@latest react-dropzone@14 zod
npm install -D @types/react-dropzone

// Verify React 19 is in your package.json
// If using the RC channel:
npm install react@rc react-dom@rc

// Create the directory structure
mkdir -p src/lib src/components src/hooks src/types

// src/lib/openai-client.ts
import OpenAI from "openai";
import { z } from "zod";

// Validate environment at import time — fail fast
const apiKey = process.env.OPENAI_API_KEY;
if (!apiKey) {
  throw new Error(
    "OPENAI_API_KEY is not set. Create a .env.local file with OPENAI_API_KEY=sk-..."
  );
}

// Cost ceiling per request in USD. GPT-4 Vision input is ~$2.50/M tokens,
// output ~$10/M tokens. We cap at $0.05 per call for a safety net.
const MAX_COST_PER_CALL = 0.05;
const INPUT_COST_PER_TOKEN = 2.5e-6;
const OUTPUT_COST_PER_TOKEN = 1e-5;

// Estimate tokens from image. Each image costs ~85 tokens (low-res) to
// ~1700 tokens (high-res detail:c1). We use detail: auto by default.
function estimateImageTokens(width: number, height: number): number {
  const longSide = Math.max(width, height);
  if (longSide > 2048) return 1700; // detail: low
  if (longSide > 768) return 850;
  return 85;
}

export const openai = new OpenAI({
  apiKey,
  // Default timeout — cloud proxies sometimes stall
  timeout: 30_000,
});

// Schema for the structured output we expect from GPT-4V
export const analysisSchema = z.object({
  summary: z.string().describe("Brief summary of the image content"),
  details: z.array(z.string()).describe("List of specific observations"),
  confidence: z.enum(["high", "medium", "low"]),
  suggestedQuestions: z.array(z.string()).optional(),
});

export type AnalysisResult = z.infer;

export interface MultimodalMessage {
  role: "user" | "assistant";
  content: string;
  images?: string[]; // base64 data URIs or URLs
  timestamp: number;
}

// Centralized error handling for OpenAI API failures
export class OpenAIError extends Error {
  constructor(
    message: string,
    public readonly statusCode?: number,
    public readonly type?: string // "rate_limit" | "content_filter" | "server_error"
  ) {
    super(message);
    this.name = "OpenAIError";
  }
}

export async function analyzeImage({
  imageUrl,
  prompt,
  signal,
}: {
  imageUrl: string;
  prompt: string;
  signal?: AbortSignal;
}): Promise {
  try {
    const response = await openai.chat.completions.create(
      {
        model: "gpt-4o",
        messages: [
          {
            role: "system",
            content:
              "You are a precise visual analyst. Answer the user's question about the image. Be specific and cite visual evidence.",
          },
          {
            role: "user",
            content: [
              { type: "text", text: prompt },
              {
                type: "image_url",
                image_url: { url: imageUrl, detail: "auto" },
              },
            ],
          },
        ],
        max_tokens: 1024,
        temperature: 0.3, // Lower temp for factual image analysis
      },
      { signal }
    );

    const content = response.choices[0]?.message?.content;
    if (!content) {
      throw new OpenAIError("Empty response from GPT-4V", 0, "server_error");
    }

    // Parse the JSON block from the response. GPT-4o returns plain text,
    // so we extract JSON if present, otherwise wrap in a default structure.
    let parsed: AnalysisResult;
    const jsonMatch = content.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      parsed = JSON.parse(jsonMatch[0]);
    } else {
      parsed = {
        summary: content.slice(0, 200),
        details: [content],
        confidence: "medium",
        suggestedQuestions: [],
      };
    }

    // Cost estimation log — useful for production monitoring
    const usage = response.usage;
    if (usage) {
      const estimatedCost =
        (usage.prompt_tokens || 0) * INPUT_COST_PER_TOKEN +
        (usage.completion_tokens || 0) * OUTPUT_COST_PER_TOKEN;
      console.info(`[OpenAI] tokens: ${usage.total_tokens}, cost: $${estimatedCost.toFixed(4)}`);
      if (estimatedCost > MAX_COST_PER_CALL) {
        console.warn(`[OpenAI] Cost exceeded threshold: $${estimatedCost.toFixed(4)}`);
      }
    }

    return parsed;
  } catch (err: any) {
    // Map OpenAI error codes to our custom error types
    if (err.status === 429) {
      throw new OpenAIError("Rate limit exceeded. Retry after a brief pause.", 429, "rate_limit");
    }
    if (err.status === 400 && err.type?.includes("content")) {
      throw new OpenAIError("Content policy violation in image or prompt.", 400, "content_filter");
    }
    if (err.status && err.status >= 500) {
      throw new OpenAIError("OpenAI server error. Retry in a few seconds.", err.status, "server_error");
    }
    if (err.name === "AbortError") {
      throw new OpenAIError("Request aborted.", 0, "server_error");
    }
    throw new OpenAIError(err.message || "Unknown OpenAI error.", err.status);
  }
}

// src/actions/analyze.ts — Server Action (Next.js App Router)
"use server";

import { revalidatePath } from "next/cache";
import { z } from "zod";
import { openai, analyzeImage, OpenAIError } from "@/lib/openai-client";
import { createClient } from "@/lib/supabase-server"; // your DB client

// Validation schema — runs before any API call
const AnalyzeSchema = z.object({
  prompt: z
    .string()
    .min(5, "Prompt must be at least 5 characters")
    .max(1000, "Prompt must be under 1,000 characters"),
  imageData: z
    .string()
    .refine((val) => val.startsWith("data:image/"), {
      message: "Invalid image format. Must be a data URI.",
    })
    .refine((val) => {
      // Extract base64 and check size (limit to 10MB decoded)
      const base64 = val.split(",")[1] || "";
      const byteLength = Buffer.from(base64, "base64").length;
      return byteLength <= 10 * 1024 * 1024;
    }, {
      message: "Image exceeds 10MB limit.",
    }),
});

export async function analyzeImageAction(
  prevState: { message?: string; result?: any } | null,
  formData: FormData
): Promise<{ message?: string; result?: any }> {
  // Extract and validate
  const prompt = formData.get("prompt") as string;
  const imageData = formData.get("image") as string;

  const parsed = AnalyzeSchema.safeParse({ prompt, imageData });
  if (!parsed.success) {
    return { message: parsed.error.issues[0].message };
  }

  // Rate-limit check: read from KV or Redis (pseudo-code)
  // const count = await kv.incr("analyze:" + userId);
  // if (count > 30) return { message: "Daily limit reached." };

  try {
    // Call GPT-4V
    const result = await analyzeImage({
      imageUrl: imageData, // data URIs are accepted by the SDK
      prompt: parsed.data.prompt,
    });

    // Persist to database for history
    const supabase = createClient();
    await supabase.from("analyses").insert({
      prompt: parsed.data.prompt,
      result,
      created_at: new Date().toISOString(),
    });

    // Revalidate the analyses page to show the new entry
    revalidatePath("/history");

    return { result };
  } catch (err) {
    if (err instanceof OpenAIError) {
      // Return user-friendly messages for each error category
      return { message: err.message };
    }
    console.error("[analyzeImageAction] Unexpected error:", err);
    return { message: "Something went wrong. Please try again." };
  }
}

// src/components/VisualChat.tsx
"use client";

import React, { useRef, useState, useOptimistic, useTransition } from "react";
import { useDropzone } from "react-dropzone";
import { analyzeImageAction } from "@/actions/analyze";
import type { MultimodalMessage } from "@/lib/openai-client";

interface Props {
  initialMessages: MultimodalMessage[];
}

// Helper: convert a File to a data URI with size constraints
function fileToDataURI(file: File): Promise {
  return new Promise((resolve, reject) => {
    const reader = new FileReader();
    reader.onload = () => resolve(reader.result as string);
    reader.onerror = () => reject(new Error("Failed to read file"));
    // Resize large images client-side to reduce upload size
    const img = new Image();
    img.onload = () => {
      const canvas = document.createElement("canvas");
      const MAX = 1024;
      if (img.width > MAX || img.height > MAX) {
        const scale = MAX / Math.max(img.width, img.height);
        canvas.width = img.width * scale;
        canvas.height = img.height * scale;
      } else {
        canvas.width = img.width;
        canvas.height = img.height;
      }
      const ctx = canvas.getContext("2d");
      ctx?.drawImage(img, 0, 0, canvas.width, canvas.height);
      resolve(canvas.toDataURL("image/jpeg", 0.85));
    };
    img.onerror = () => reject(new Error("Failed to load image for resize"));
    img.src = reader.result as string;
  });
}

export default function VisualChat({ initialMessages }: Props) {
  const [messages, setMessages] = useState(initialMessages);
  const [imagePreview, setImagePreview] = useState(null);
  const [pendingImage, setPendingImage] = useState(null);
  const [isPending, startTransition] = useTransition();
  const formRef = useRef(null);
  const endRef = useRef(null);

  // useOptimistic: render the "thinking" message immediately
  const [optimisticMessages, addOptimisticMessage] = useOptimistic(
    messages,
    (current, newMsg: MultimodalMessage) => [...current, newMsg]
  );

  const { getRootProps, getInputProps, isDragActive } = useDropzone({
    accept: { "image/*": [] },
    maxSize: 10 * 1024 * 1024, // 10MB
    maxFiles: 1,
    onDrop: async (acceptedFiles) => {
      const uri = await fileToDataURI(acceptedFiles[0]);
      setImagePreview(uri);
    },
    onDropRejected: (fileRejections) => {
      alert(fileRejections[0].errors[0].message);
    },
  });

  // Server Action handler — wrapped in startTransition for pending state
  async function handleSubmit(formData: FormData) {
    const prompt = formData.get("prompt") as string;
    if (!imagePreview) {
      alert("Please upload an image first.");
      return;
    }

    // Optimistically add a "user" message
    const userMsg: MultimodalMessage = {
      role: "user",
      content: prompt,
      images: [imagePreview],
      timestamp: Date.now(),
    };
    addOptimisticMessage(userMsg);

    // Optimistically add a "typing" placeholder
    const placeholderMsg: MultimodalMessage = {
      role: "assistant",
      content: "",
      timestamp: Date.now(),
    };
    addOptimisticMessage(placeholderMsg);

    try {
      // Call the Server Action
      const result = await analyzeImageAction(null, formData);
      if (result.message) {
        // Error from server — show it inline
        const errorMsg: MultimodalMessage = {
          role: "assistant",
          content: `⚠️ ${result.message}`,
          timestamp: Date.now(),
        };
        setMessages((prev) => [...prev, userMsg, errorMsg]);
      } else {
        // Success — format the structured result
        const analysis = result.result;
        const formatted = `📝 **${analysis.summary}**\n\n${analysis.details.map((d: string) => `• ${d}`).join("\n")}\n\nConfidence: ${analysis.confidence}`;
        const successMsg: MultimodalMessage = {
          role: "assistant",
          content: formatted,
          timestamp: Date.now(),
        };
        setMessages((prev) => [...prev, userMsg, successMsg]);
      }
    } catch (err: any) {
      const errorMsg: MultimodalMessage = {
        role: "assistant",
        content: `❌ Error: ${err.message}`,
        timestamp: Date.now(),
      };
      setMessages((prev) => [...prev, userMsg, errorMsg]);
    }

    // Reset the form and image preview
    setImagePreview(null);
    if (formRef.current) formRef.current.reset();
    endRef.current?.scrollIntoView({ behavior: "smooth" });
  }

  return (

// src/app/api/analyze/route.ts — Next.js 15 streaming endpoint
import { NextRequest, NextResponse } from "next/server";
import OpenAI from "openai";

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

export const runtime = "nodejs"; // Required for streaming
export const dynamic = "force-dynamic";

export async function POST(req: NextRequest) {
  try {
    const { imageUrl, prompt } = await req.json();

    if (!imageUrl || !prompt) {
      return NextResponse.json(
        { error: "imageUrl and prompt are required" },
        { status: 400 }
      );
    }

    const stream = await openai.chat.completions.create({
      model: "gpt-4o",
      messages: [
        {
          role: "system",
          content: "You are a helpful visual analyst. Respond in markdown.",
        },
        {
          role: "user",
          content: [
            { type: "text", text: prompt },
            { type: "image_url", image_url: { url: imageUrl, detail: "auto" } },
          ],
        },
      ],
      stream: true,
      max_tokens: 1024,
      temperature: 0.3,
    });

    // Convert OpenAI's async iterable into a ReadableStream
    const encoder = new TextEncoder();
    const readable = new ReadableStream({
      async start(controller) {
        try {
          for await (const chunk of stream) {
            const text = chunk.choices[0]?.delta?.content || "";
            if (text) {
              controller.enqueue(encoder.encode(`data: ${JSON.stringify({ text })}\n\n`));
            }
          }
          controller.enqueue(encoder.encode("data: {\"done\": true}\n\n"));
          controller.close();
        } catch (err: any) {
          controller.error(err);
        }
      },
    });

    return new NextResponse(readable, {
      headers: {
        "Content-Type": "text/event-stream",
        "Cache-Control": "no-cache",
        Connection: "keep-alive",
      },
    });
  } catch (err: any) {
    console.error("[stream-analyze]", err);
    return NextResponse.json(
      { error: err.message || "Internal server error" },
      { status: 500 }
    );
  }
}

// src/hooks/useStreamAnalysis.ts — Client-side streaming hook
import { useState, useRef, useCallback } from "react";

export function useStreamAnalysis() {
  const [content, setContent] = useState("");
  const [isStreaming, setIsStreaming] = useState(false);
  const [error, setError] = useState(null);
  const abortRef = useRef(null);

  const startAnalysis = useCallback(async (imageUrl: string, prompt: string) => {
    // Cancel any in-flight request
    if (abortRef.current) abortRef.current.abort();
    const controller = new AbortController();
    abortRef.current = controller;

    setContent("");
    setError(null);
    setIsStreaming(true);

    try {
      const response = await fetch("/api/analyze", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ imageUrl, prompt }),
        signal: controller.signal,
      });

      if (!response.ok || !response.body) {
        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
      }

      const reader = response.body.getReader();
      const decoder = new TextDecoder();

      while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        const chunk = decoder.decode(value, { stream: true });
        // Parse Server-Sent Events format
        const lines = chunk.split("\n\n");
        for (const line of lines) {
          if (line.startsWith("data: ")) {
            const data = JSON.parse(line.slice(6));
            if (data.done) {
              setIsStreaming(false);
              return;
            }
            setContent((prev) => prev + data.text);
          }
        }
      }
    } catch (err: any) {
      if (err.name !== "AbortError") {
        setError(err.message || "Stream failed.");
        setIsStreaming(false);
      }
    }
  }, []);

  const cancel = useCallback(() => {
    abortRef.current?.abort();
    setIsStreaming(false);
  }, []);

  return { content, isStreaming, error, startAnalysis, cancel };
}

// Example: @ai-sdk/react useChat for streaming
import { useChat } from "@ai-sdk/react";

export function useVisualQA() {
  const { messages, input, handleSubmit, isLoading, stop } = useChat({
    api: "/api/analyze",
    initialMessages: [],
  });
  return { messages, input, handleSubmit, isLoading, stop };
}

// Server-side image optimization in a Server Action
import { resize, type OutputType } from "sharp";

export async function optimizeImage(buffer: Buffer): Promise<string> {
  const resized = await resize(buffer, { width: 1024, withoutEnlargement: true });
  return resized.jpeg({ quality: 85 }).toString("base64");
}

// Robust Zod parsing with multiple extraction strategies
import { z } from "zod";

function extractJSON(text: string): unknown {
  // Strategy 1: Direct parse
  try { return JSON.parse(text); } catch {}
  // Strategy 2: Extract JSON block between  markers
  const fenced = text.match(/(?:json)?\n([\s\S]*?)\n/);
  if (fenced) {
    try { return JSON.parse(fenced[1]); } catch {}
  }
  // Strategy 3: Find first { and last } and extract between
  const start = text.indexOf("{");
  const end = text.lastIndexOf("}");
  if (start !== -1 && end > start) {
    try { return JSON.parse(text.slice(start, end + 1)); } catch {}
  }
  return null;
}