mirror of
https://github.com/DayuanJiang/next-ai-draw-io.git
synced 2026-09-01 17:10:24 +08:00
Add VLM-based diagram validation (#602)
* [Feature] Add VLM-based diagram validation Add automatic VLM (Vision Language Model) validation after display_diagram tool execution. The system captures a screenshot of the rendered diagram, sends it to a VLM for visual analysis, and uses feedback to improve diagram quality through the existing retry mechanism. Changes: - Add /api/validate-diagram endpoint for VLM validation - Add diagram-validator.ts for client-side validation orchestration - Add validation-prompts.ts for VLM system prompts - Add ValidationCard component to display validation status in chat - Add PNG capture functionality to diagram context - Integrate validation into tool handlers with retry support (max 3) - Add "Improve with Suggestions" button for manual regeneration - Add settings toggle to enable/disable VLM validation - Add getValidationModel() helper in ai-providers.ts * refactor(validation): use AI SDK structured outputs and address review feedback - Replace generateText + manual JSON parsing with generateObject and Zod schema for type-safe structured validation output - Use AbortSignal.timeout() instead of Promise.race for cleaner timeout handling - Add timeout validation with minimum 1000ms to handle malformed env values - Remove unused xml parameter from validateRenderedDiagram API - Remove parseValidationResponse function (now handled by schema) - Clear validationStates on session switch and new chat to prevent memory leak - Update 100ms render delay comment to clarify best-effort heuristic - Remove unused useEffect import from ValidationCard - Fix optional chaining lint warning in ValidationCard - Add unit tests for formatValidationFeedback function * refactor(validation): use AI SDK experimental_useObject hook instead of raw fetch - Change API endpoint from generateObject to streamObject for useObject compatibility - Create useValidateDiagram hook using AI SDK's experimental_useObject for reactive validation - Update useDiagramToolHandlers to accept validation function as parameter - Update chat-panel to use new useValidateDiagram hook - Remove validateRenderedDiagram function from lib/diagram-validator.ts (now in hook) - Export ValidationResultSchema from API route for client-side use * fix(validation): extract schema to shared file for client/server compatibility Move ValidationResultSchema to lib/validation-schema.ts to avoid importing server-side modules (ai-providers) into client-side code. This fixes the Turbopack build error caused by the hook importing from the API route. * fix(validation): use 'Valid' instead of 'Complete' for validation success Change ValidationCard success label from 'Complete' to 'Valid' to avoid conflicting with ToolCallCard's 'Complete' badge in E2E tests. This fixes the diagram-generation E2E test that expects a specific count of 'Complete' badges. * fix(validation): add aria-hidden to icons to prevent duplicate ID warning * fix: improve VLM validation with bug fixes and i18n - Fix race condition in pendingValidationRef (reject previous pending validation) - Fix response format consistency (use streaming for all responses) - Remove dead code (unused lastRequestRef and ValidationRequest interface) - Consolidate duplicate types (re-export from validation-schema.ts) - Add 'success_with_warnings' status for valid diagrams with warnings - Fix tool card auto-collapse (only collapse once, respect user toggle) - Set VLM validation default to disabled - Add i18n support for diagram validation settings (en/zh/ja) - Mark feature as experimental in settings UI * fix: resolve TypeScript errors in electron-standalone - Add forwardRef support to ChatInput component with ChatInputRef type - Copy electron.d.ts to electron-standalone/electron folder - Exclude electron-standalone from root tsconfig type checking * fix: return empty string for valid result with no issues in formatValidationFeedback * feat(i18n): add validation strings for ValidationCard component - Add validation section to en.json, zh.json, ja.json dictionaries - Update ValidationCard to use useDictionary hook - Replace all hardcoded English strings with i18n keys --------- Co-authored-by: dayuan.jiang <jdy.toh@gmail.com>
This commit is contained in:
yujinze
328
components/chat/ValidationCard.tsx
Normal file
328
components/chat/ValidationCard.tsx
Normal file
@@ -0,0 +1,328 @@
|
||||
"use client"
|
||||
|
||||
import {
|
||||
AlertTriangle,
|
||||
Check,
|
||||
ChevronDown,
|
||||
ChevronUp,
|
||||
Eye,
|
||||
ImageIcon,
|
||||
RefreshCw,
|
||||
X,
|
||||
} from "lucide-react"
|
||||
import Image from "next/image"
|
||||
import { useState } from "react"
|
||||
import { useDictionary } from "@/hooks/use-dictionary"
|
||||
import type { ValidationResult } from "@/lib/diagram-validator"
|
||||
|
||||
export type ValidationStatus =
|
||||
| "idle"
|
||||
| "capturing"
|
||||
| "validating"
|
||||
| "success"
|
||||
| "success_with_warnings"
|
||||
| "failed"
|
||||
| "error"
|
||||
| "skipped"
|
||||
|
||||
export interface ValidationState {
|
||||
status: ValidationStatus
|
||||
attempt?: number
|
||||
maxAttempts?: number
|
||||
result?: ValidationResult
|
||||
error?: string
|
||||
imageData?: string // Base64 PNG data URL
|
||||
}
|
||||
|
||||
interface ValidationCardProps {
|
||||
state: ValidationState
|
||||
onImproveWithSuggestions?: (feedback: string) => void
|
||||
}
|
||||
|
||||
export function ValidationCard({
|
||||
state,
|
||||
onImproveWithSuggestions,
|
||||
}: ValidationCardProps) {
|
||||
const dict = useDictionary()
|
||||
const [isExpanded, setIsExpanded] = useState(
|
||||
state.status === "validating" || state.status === "failed",
|
||||
)
|
||||
const [hasRequestedImprovement, setHasRequestedImprovement] =
|
||||
useState(false)
|
||||
|
||||
// Generate improvement feedback from validation result
|
||||
const generateImprovementFeedback = (): string => {
|
||||
if (!state.result) return ""
|
||||
|
||||
const lines: string[] = []
|
||||
lines.push(
|
||||
"Please improve the diagram based on the following visual analysis feedback:",
|
||||
)
|
||||
lines.push("")
|
||||
|
||||
if (state.result.issues.length > 0) {
|
||||
lines.push("Issues to address:")
|
||||
for (const issue of state.result.issues) {
|
||||
lines.push(
|
||||
` - [${issue.severity}] ${issue.type}: ${issue.description}`,
|
||||
)
|
||||
}
|
||||
lines.push("")
|
||||
}
|
||||
|
||||
if (state.result.suggestions.length > 0) {
|
||||
lines.push("Suggestions for improvement:")
|
||||
for (const suggestion of state.result.suggestions) {
|
||||
lines.push(` - ${suggestion}`)
|
||||
}
|
||||
lines.push("")
|
||||
}
|
||||
|
||||
lines.push("Regenerate the diagram with these improvements applied.")
|
||||
return lines.join("\n")
|
||||
}
|
||||
|
||||
const handleImproveClick = () => {
|
||||
if (
|
||||
!onImproveWithSuggestions ||
|
||||
!state.result ||
|
||||
hasRequestedImprovement
|
||||
)
|
||||
return
|
||||
setHasRequestedImprovement(true)
|
||||
const feedback = generateImprovementFeedback()
|
||||
onImproveWithSuggestions(feedback)
|
||||
}
|
||||
|
||||
// Check if we should show the improve button
|
||||
const showImproveButton =
|
||||
onImproveWithSuggestions &&
|
||||
state.result &&
|
||||
(state.status === "success" ||
|
||||
state.status === "success_with_warnings" ||
|
||||
state.status === "skipped") &&
|
||||
(state.result.issues.length > 0 || state.result.suggestions.length > 0)
|
||||
|
||||
const getStatusDisplay = () => {
|
||||
switch (state.status) {
|
||||
case "capturing":
|
||||
return {
|
||||
label: dict.validation.capturing,
|
||||
color: "text-blue-600 bg-blue-50",
|
||||
icon: (
|
||||
<div className="h-4 w-4 border-2 border-blue-600 border-t-transparent rounded-full animate-spin" />
|
||||
),
|
||||
}
|
||||
case "validating":
|
||||
return {
|
||||
label: state.attempt
|
||||
? dict.validation.validatingWithAttempt
|
||||
.replace("{attempt}", String(state.attempt))
|
||||
.replace("{max}", String(state.maxAttempts || 3))
|
||||
: dict.validation.validating,
|
||||
color: "text-blue-600 bg-blue-50",
|
||||
icon: (
|
||||
<div className="h-4 w-4 border-2 border-blue-600 border-t-transparent rounded-full animate-spin" />
|
||||
),
|
||||
}
|
||||
case "success":
|
||||
return {
|
||||
label: dict.validation.valid,
|
||||
color: "text-green-600 bg-green-50",
|
||||
icon: <Check className="h-4 w-4" aria-hidden="true" />,
|
||||
}
|
||||
case "success_with_warnings":
|
||||
return {
|
||||
label: dict.validation.validWithWarnings,
|
||||
color: "text-amber-600 bg-amber-50",
|
||||
icon: (
|
||||
<AlertTriangle className="h-4 w-4" aria-hidden="true" />
|
||||
),
|
||||
}
|
||||
case "failed":
|
||||
return {
|
||||
label: dict.validation.issuesFound,
|
||||
color: "text-yellow-600 bg-yellow-50",
|
||||
icon: (
|
||||
<AlertTriangle className="h-4 w-4" aria-hidden="true" />
|
||||
),
|
||||
}
|
||||
case "error":
|
||||
return {
|
||||
label: dict.validation.error,
|
||||
color: "text-red-600 bg-red-50",
|
||||
icon: <X className="h-4 w-4" aria-hidden="true" />,
|
||||
}
|
||||
case "skipped":
|
||||
return {
|
||||
label: dict.validation.skipped,
|
||||
color: "text-gray-600 bg-gray-50",
|
||||
icon: <Check className="h-4 w-4" aria-hidden="true" />,
|
||||
}
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const statusDisplay = getStatusDisplay()
|
||||
if (!statusDisplay || state.status === "idle") return null
|
||||
|
||||
return (
|
||||
<div className="my-3 rounded-xl border border-border/60 bg-muted/30 overflow-hidden">
|
||||
<div className="flex items-center justify-between px-4 py-3 bg-muted/50">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-6 h-6 rounded-md bg-primary/10 flex items-center justify-center">
|
||||
<Eye
|
||||
className="w-3.5 h-3.5 text-primary"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
</div>
|
||||
<span className="text-sm font-medium text-foreground/80">
|
||||
{dict.validation.title}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className={`text-xs font-medium px-2 py-0.5 rounded-full flex items-center gap-1 ${statusDisplay.color}`}
|
||||
>
|
||||
{statusDisplay.icon}
|
||||
<span className="ml-1">{statusDisplay.label}</span>
|
||||
</span>
|
||||
{(state.result || state.error) && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setIsExpanded(!isExpanded)}
|
||||
className="p-1 rounded hover:bg-muted transition-colors"
|
||||
>
|
||||
{isExpanded ? (
|
||||
<ChevronUp
|
||||
className="w-4 h-4 text-muted-foreground"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
) : (
|
||||
<ChevronDown
|
||||
className="w-4 h-4 text-muted-foreground"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Validation details when expanded */}
|
||||
{isExpanded && (state.result || state.imageData) && (
|
||||
<div className="px-4 py-3 border-t border-border/40 bg-muted/20 space-y-3">
|
||||
{/* Captured image */}
|
||||
{state.imageData && (
|
||||
<div>
|
||||
<div className="text-xs font-medium text-foreground/70 mb-2 flex items-center gap-1">
|
||||
<ImageIcon
|
||||
className="h-3 w-3"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
{dict.validation.capturedScreenshot}
|
||||
</div>
|
||||
<div className="rounded-lg border border-border/50 overflow-hidden bg-white">
|
||||
<Image
|
||||
src={state.imageData}
|
||||
alt="Captured diagram for validation"
|
||||
width={400}
|
||||
height={300}
|
||||
className="w-full h-auto max-h-48 object-contain"
|
||||
unoptimized
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Issues */}
|
||||
{state.result && state.result.issues.length > 0 && (
|
||||
<div>
|
||||
<div className="text-xs font-medium text-foreground/70 mb-2">
|
||||
{dict.validation.issuesFoundLabel}
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
{state.result.issues.map((issue, index) => (
|
||||
<div
|
||||
key={index}
|
||||
className={`text-xs px-3 py-2 rounded-lg border ${
|
||||
issue.severity === "critical"
|
||||
? "bg-red-50 border-red-200 text-red-700 dark:bg-red-950 dark:border-red-800 dark:text-red-300"
|
||||
: "bg-yellow-50 border-yellow-200 text-yellow-700 dark:bg-yellow-950 dark:border-yellow-800 dark:text-yellow-300"
|
||||
}`}
|
||||
>
|
||||
<span className="font-medium uppercase text-[10px] mr-2">
|
||||
[{issue.type}]
|
||||
</span>
|
||||
{issue.description}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Suggestions */}
|
||||
{state.result && state.result.suggestions.length > 0 && (
|
||||
<div>
|
||||
<div className="text-xs font-medium text-foreground/70 mb-2">
|
||||
{dict.validation.suggestions}
|
||||
</div>
|
||||
<ul className="text-xs text-foreground/60 space-y-1 list-disc list-inside">
|
||||
{state.result.suggestions.map(
|
||||
(suggestion, index) => (
|
||||
<li key={index}>{suggestion}</li>
|
||||
),
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Valid result message */}
|
||||
{state.result?.valid &&
|
||||
state.result.issues.length === 0 && (
|
||||
<div className="text-xs text-green-600 dark:text-green-400">
|
||||
{dict.validation.passedValidation}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Improve with Suggestions button - shown when validation passed but has suggestions */}
|
||||
{showImproveButton && (
|
||||
<div className="px-4 py-3 border-t border-border/40 bg-muted/10">
|
||||
{hasRequestedImprovement ? (
|
||||
<div className="flex items-center justify-center gap-2 px-4 py-2 text-sm font-medium text-green-600 dark:text-green-400">
|
||||
<Check className="h-4 w-4" aria-hidden="true" />
|
||||
{dict.validation.improvementRequested}
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleImproveClick}
|
||||
className="w-full flex items-center justify-center gap-2 px-4 py-2 text-sm font-medium text-primary bg-primary/10 hover:bg-primary/20 rounded-lg transition-colors"
|
||||
>
|
||||
<RefreshCw
|
||||
className="h-4 w-4"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
{dict.validation.improveWithSuggestions}
|
||||
</button>
|
||||
<p className="text-xs text-muted-foreground mt-2 text-center">
|
||||
{dict.validation.regenerateWithFeedback}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Error details when expanded */}
|
||||
{isExpanded && state.error && (
|
||||
<div className="px-4 py-3 border-t border-border/40 bg-red-50/50">
|
||||
<div className="text-xs text-red-600">{state.error}</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user