[Feature] Add VLM-based diagram validation

Add automatic VLM (Vision Language Model) validation after display_diagram
tool execution. The system captures a screenshot of the rendered diagram,
sends it to a VLM for visual analysis, and uses feedback to improve
diagram quality through the existing retry mechanism.

Changes:
- Add /api/validate-diagram endpoint for VLM validation
- Add diagram-validator.ts for client-side validation orchestration
- Add validation-prompts.ts for VLM system prompts
- Add ValidationCard component to display validation status in chat
- Add PNG capture functionality to diagram context
- Integrate validation into tool handlers with retry support (max 3)
- Add "Improve with Suggestions" button for manual regeneration
- Add settings toggle to enable/disable VLM validation
- Add getValidationModel() helper in ai-providers.ts

lib/ai-providers.ts

@@ -1177,3 +1177,27 @@ export function supportsImageInput(modelId: string): boolean {
     // Default: assume model supports images
     return true
 }
+
+/**
+ * Get the AI model for diagram validation.
+ * Uses VALIDATION_MODEL env var if set, otherwise falls back to AI_MODEL.
+ * Throws if the model doesn't support image input.
+ */
+export function getValidationModel(): ReturnType<typeof getAIModel>["model"] {
+    const modelId = process.env.VALIDATION_MODEL || process.env.AI_MODEL
+
+    if (!modelId) {
+        throw new Error(
+            "No validation model configured. Set VALIDATION_MODEL or AI_MODEL.",
+        )
+    }
+
+    if (!supportsImageInput(modelId)) {
+        throw new Error(
+            `Validation requires a vision-capable model. Model "${modelId}" does not support image input.`,
+        )
+    }
+
+    const { model } = getAIModel({ modelId })
+    return model
+}

lib/diagram-validator.ts

@@ -0,0 +1,121 @@
+/**
+ * Client-side validation orchestration for VLM-based diagram validation.
+ */
+
+import { getApiEndpoint } from "./base-path"
+
+export interface ValidationIssue {
+    type: "overlap" | "edge_routing" | "text" | "layout" | "rendering"
+    severity: "critical" | "warning"
+    description: string
+}
+
+export interface ValidationResult {
+    valid: boolean
+    issues: ValidationIssue[]
+    suggestions: string[]
+}
+
+/**
+ * Validate a rendered diagram by sending its PNG to the VLM validation API.
+ *
+ * @param pngDataUrl - Base64 PNG data URL from draw.io export
+ * @param xml - The diagram XML (for context in error messages)
+ * @param sessionId - Optional session ID for logging
+ * @returns ValidationResult with issues and suggestions
+ */
+export async function validateRenderedDiagram(
+    pngDataUrl: string,
+    xml: string,
+    sessionId?: string,
+): Promise<ValidationResult> {
+    try {
+        const response = await fetch(getApiEndpoint("/api/validate-diagram"), {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                imageData: pngDataUrl,
+                xml,
+                sessionId,
+            }),
+        })
+
+        if (!response.ok) {
+            const errorData = await response.json().catch(() => ({}))
+            console.error(
+                "[validateRenderedDiagram] API error:",
+                response.status,
+                errorData,
+            )
+            // Return valid on API error to not block the user
+            return {
+                valid: true,
+                issues: [],
+                suggestions: [],
+            }
+        }
+
+        const result: ValidationResult = await response.json()
+        return result
+    } catch (error) {
+        console.error("[validateRenderedDiagram] Failed:", error)
+        // Return valid on network error to not block the user
+        return {
+            valid: true,
+            issues: [],
+            suggestions: [],
+        }
+    }
+}
+
+/**
+ * Format validation feedback for display to the AI model.
+ * This creates a human-readable error message that guides the AI to fix issues.
+ *
+ * @param result - The validation result from VLM
+ * @returns Formatted string for tool error output
+ */
+export function formatValidationFeedback(result: ValidationResult): string {
+    const lines: string[] = []
+
+    lines.push("DIAGRAM VISUAL VALIDATION FAILED")
+    lines.push("")
+
+    // Group issues by severity
+    const criticalIssues = result.issues.filter(
+        (i) => i.severity === "critical",
+    )
+    const warnings = result.issues.filter((i) => i.severity === "warning")
+
+    if (criticalIssues.length > 0) {
+        lines.push("Critical Issues (must fix):")
+        for (const issue of criticalIssues) {
+            lines.push(`  - [${issue.type}] ${issue.description}`)
+        }
+        lines.push("")
+    }
+
+    if (warnings.length > 0) {
+        lines.push("Warnings:")
+        for (const issue of warnings) {
+            lines.push(`  - [${issue.type}] ${issue.description}`)
+        }
+        lines.push("")
+    }
+
+    if (result.suggestions.length > 0) {
+        lines.push("Suggestions to fix:")
+        for (const suggestion of result.suggestions) {
+            lines.push(`  - ${suggestion}`)
+        }
+        lines.push("")
+    }
+
+    lines.push(
+        "Please regenerate the diagram with corrected layout to fix these visual issues.",
+    )
+
+    return lines.join("\n")
+}

lib/storage.ts

@@ -24,4 +24,7 @@ export const STORAGE_KEYS = {
 
     // Chat input preferences
     sendShortcut: "next-ai-draw-io-send-shortcut",
+
+    // Diagram validation
+    vlmValidationEnabled: "next-ai-draw-io-vlm-validation-enabled",
 } as const

lib/validation-prompts.ts

@@ -0,0 +1,122 @@
+/**
+ * VLM system prompt and response parsing for diagram validation.
+ */
+
+import type { ValidationResult } from "./diagram-validator"
+
+export const VALIDATION_SYSTEM_PROMPT = `You are a diagram quality validator. Analyze the rendered diagram image for visual issues.
+
+Evaluate the diagram for the following issues:
+
+1. **Overlapping elements** (critical): Shapes covering each other inappropriately, making content unreadable
+2. **Edge routing issues** (critical): Lines/arrows crossing through shapes that are not their source or target
+3. **Text readability** (warning): Labels cut off, overlapping, or too small to read
+4. **Layout quality** (warning): Poor spacing, misalignment, or cramped elements
+5. **Rendering errors** (critical): Incomplete, corrupted, or missing visual elements
+
+Return your analysis as a JSON object with this structure:
+{
+  "valid": boolean,
+  "issues": [
+    {
+      "type": "overlap" | "edge_routing" | "text" | "layout" | "rendering",
+      "severity": "critical" | "warning",
+      "description": "Clear description of the issue and where it occurs"
+    }
+  ],
+  "suggestions": ["Specific actionable suggestions to fix the issues"]
+}
+
+Rules:
+- Set "valid" to true ONLY if there are no critical issues
+- Be specific about which elements have problems (e.g., "The 'Login' box overlaps with 'Register' box")
+- Provide actionable suggestions (e.g., "Move the Login box 50 pixels to the left")
+- Minor cosmetic issues (slight misalignment, non-uniform spacing) should be warnings, not critical
+- Empty diagrams or diagrams with only 1-2 elements should pass unless they have obvious errors
+- If the diagram looks generally acceptable, set valid to true even with minor warnings
+
+Return ONLY the JSON object, no additional text.`
+
+/**
+ * Parse the VLM response text into a ValidationResult.
+ * Handles various response formats and edge cases.
+ */
+export function parseValidationResponse(text: string): ValidationResult {
+    try {
+        // Try to extract JSON from the response
+        // The VLM might wrap the JSON in markdown code blocks
+        let jsonStr = text.trim()
+
+        // Remove markdown code block if present
+        const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/)
+        if (jsonMatch) {
+            jsonStr = jsonMatch[1].trim()
+        }
+
+        // Try to find JSON object if there's extra text
+        const objectMatch = jsonStr.match(/\{[\s\S]*\}/)
+        if (objectMatch) {
+            jsonStr = objectMatch[0]
+        }
+
+        const parsed = JSON.parse(jsonStr)
+
+        // Validate the structure
+        if (typeof parsed.valid !== "boolean") {
+            console.warn(
+                "[parseValidationResponse] Missing or invalid 'valid' field",
+            )
+            return {
+                valid: true, // Default to valid if parsing fails
+                issues: [],
+                suggestions: [],
+            }
+        }
+
+        // Normalize issues array
+        const issues = Array.isArray(parsed.issues)
+            ? parsed.issues
+                  .filter(
+                      (issue: any) =>
+                          issue &&
+                          typeof issue.type === "string" &&
+                          typeof issue.severity === "string" &&
+                          typeof issue.description === "string",
+                  )
+                  .map((issue: any) => ({
+                      type: issue.type as
+                          | "overlap"
+                          | "edge_routing"
+                          | "text"
+                          | "layout"
+                          | "rendering",
+                      severity: issue.severity as "critical" | "warning",
+                      description: issue.description,
+                  }))
+            : []
+
+        // Normalize suggestions array
+        const suggestions = Array.isArray(parsed.suggestions)
+            ? parsed.suggestions.filter((s: any) => typeof s === "string")
+            : []
+
+        return {
+            valid: parsed.valid,
+            issues,
+            suggestions,
+        }
+    } catch (error) {
+        console.error(
+            "[parseValidationResponse] Failed to parse VLM response:",
+            error,
+        )
+        console.error("[parseValidationResponse] Raw response:", text)
+
+        // If parsing fails, default to valid to avoid blocking the user
+        return {
+            valid: true,
+            issues: [],
+            suggestions: [],
+        }
+    }
+}