mirror of
https://github.com/DayuanJiang/next-ai-draw-io.git
synced 2026-09-03 10:00:22 +08:00
* [Feature] Add VLM-based diagram validation Add automatic VLM (Vision Language Model) validation after display_diagram tool execution. The system captures a screenshot of the rendered diagram, sends it to a VLM for visual analysis, and uses feedback to improve diagram quality through the existing retry mechanism. Changes: - Add /api/validate-diagram endpoint for VLM validation - Add diagram-validator.ts for client-side validation orchestration - Add validation-prompts.ts for VLM system prompts - Add ValidationCard component to display validation status in chat - Add PNG capture functionality to diagram context - Integrate validation into tool handlers with retry support (max 3) - Add "Improve with Suggestions" button for manual regeneration - Add settings toggle to enable/disable VLM validation - Add getValidationModel() helper in ai-providers.ts * refactor(validation): use AI SDK structured outputs and address review feedback - Replace generateText + manual JSON parsing with generateObject and Zod schema for type-safe structured validation output - Use AbortSignal.timeout() instead of Promise.race for cleaner timeout handling - Add timeout validation with minimum 1000ms to handle malformed env values - Remove unused xml parameter from validateRenderedDiagram API - Remove parseValidationResponse function (now handled by schema) - Clear validationStates on session switch and new chat to prevent memory leak - Update 100ms render delay comment to clarify best-effort heuristic - Remove unused useEffect import from ValidationCard - Fix optional chaining lint warning in ValidationCard - Add unit tests for formatValidationFeedback function * refactor(validation): use AI SDK experimental_useObject hook instead of raw fetch - Change API endpoint from generateObject to streamObject for useObject compatibility - Create useValidateDiagram hook using AI SDK's experimental_useObject for reactive validation - Update useDiagramToolHandlers to accept validation function as parameter - Update chat-panel to use new useValidateDiagram hook - Remove validateRenderedDiagram function from lib/diagram-validator.ts (now in hook) - Export ValidationResultSchema from API route for client-side use * fix(validation): extract schema to shared file for client/server compatibility Move ValidationResultSchema to lib/validation-schema.ts to avoid importing server-side modules (ai-providers) into client-side code. This fixes the Turbopack build error caused by the hook importing from the API route. * fix(validation): use 'Valid' instead of 'Complete' for validation success Change ValidationCard success label from 'Complete' to 'Valid' to avoid conflicting with ToolCallCard's 'Complete' badge in E2E tests. This fixes the diagram-generation E2E test that expects a specific count of 'Complete' badges. * fix(validation): add aria-hidden to icons to prevent duplicate ID warning * fix: improve VLM validation with bug fixes and i18n - Fix race condition in pendingValidationRef (reject previous pending validation) - Fix response format consistency (use streaming for all responses) - Remove dead code (unused lastRequestRef and ValidationRequest interface) - Consolidate duplicate types (re-export from validation-schema.ts) - Add 'success_with_warnings' status for valid diagrams with warnings - Fix tool card auto-collapse (only collapse once, respect user toggle) - Set VLM validation default to disabled - Add i18n support for diagram validation settings (en/zh/ja) - Mark feature as experimental in settings UI * fix: resolve TypeScript errors in electron-standalone - Add forwardRef support to ChatInput component with ChatInputRef type - Copy electron.d.ts to electron-standalone/electron folder - Exclude electron-standalone from root tsconfig type checking * fix: return empty string for valid result with no issues in formatValidationFeedback * feat(i18n): add validation strings for ValidationCard component - Add validation section to en.json, zh.json, ja.json dictionaries - Update ValidationCard to use useDictionary hook - Replace all hardcoded English strings with i18n keys --------- Co-authored-by: dayuan.jiang <jdy.toh@gmail.com>
329 lines
13 KiB
TypeScript
329 lines
13 KiB
TypeScript
"use client"
|
|
|
|
import {
|
|
AlertTriangle,
|
|
Check,
|
|
ChevronDown,
|
|
ChevronUp,
|
|
Eye,
|
|
ImageIcon,
|
|
RefreshCw,
|
|
X,
|
|
} from "lucide-react"
|
|
import Image from "next/image"
|
|
import { useState } from "react"
|
|
import { useDictionary } from "@/hooks/use-dictionary"
|
|
import type { ValidationResult } from "@/lib/diagram-validator"
|
|
|
|
export type ValidationStatus =
|
|
| "idle"
|
|
| "capturing"
|
|
| "validating"
|
|
| "success"
|
|
| "success_with_warnings"
|
|
| "failed"
|
|
| "error"
|
|
| "skipped"
|
|
|
|
export interface ValidationState {
|
|
status: ValidationStatus
|
|
attempt?: number
|
|
maxAttempts?: number
|
|
result?: ValidationResult
|
|
error?: string
|
|
imageData?: string // Base64 PNG data URL
|
|
}
|
|
|
|
interface ValidationCardProps {
|
|
state: ValidationState
|
|
onImproveWithSuggestions?: (feedback: string) => void
|
|
}
|
|
|
|
export function ValidationCard({
|
|
state,
|
|
onImproveWithSuggestions,
|
|
}: ValidationCardProps) {
|
|
const dict = useDictionary()
|
|
const [isExpanded, setIsExpanded] = useState(
|
|
state.status === "validating" || state.status === "failed",
|
|
)
|
|
const [hasRequestedImprovement, setHasRequestedImprovement] =
|
|
useState(false)
|
|
|
|
// Generate improvement feedback from validation result
|
|
const generateImprovementFeedback = (): string => {
|
|
if (!state.result) return ""
|
|
|
|
const lines: string[] = []
|
|
lines.push(
|
|
"Please improve the diagram based on the following visual analysis feedback:",
|
|
)
|
|
lines.push("")
|
|
|
|
if (state.result.issues.length > 0) {
|
|
lines.push("Issues to address:")
|
|
for (const issue of state.result.issues) {
|
|
lines.push(
|
|
` - [${issue.severity}] ${issue.type}: ${issue.description}`,
|
|
)
|
|
}
|
|
lines.push("")
|
|
}
|
|
|
|
if (state.result.suggestions.length > 0) {
|
|
lines.push("Suggestions for improvement:")
|
|
for (const suggestion of state.result.suggestions) {
|
|
lines.push(` - ${suggestion}`)
|
|
}
|
|
lines.push("")
|
|
}
|
|
|
|
lines.push("Regenerate the diagram with these improvements applied.")
|
|
return lines.join("\n")
|
|
}
|
|
|
|
const handleImproveClick = () => {
|
|
if (
|
|
!onImproveWithSuggestions ||
|
|
!state.result ||
|
|
hasRequestedImprovement
|
|
)
|
|
return
|
|
setHasRequestedImprovement(true)
|
|
const feedback = generateImprovementFeedback()
|
|
onImproveWithSuggestions(feedback)
|
|
}
|
|
|
|
// Check if we should show the improve button
|
|
const showImproveButton =
|
|
onImproveWithSuggestions &&
|
|
state.result &&
|
|
(state.status === "success" ||
|
|
state.status === "success_with_warnings" ||
|
|
state.status === "skipped") &&
|
|
(state.result.issues.length > 0 || state.result.suggestions.length > 0)
|
|
|
|
const getStatusDisplay = () => {
|
|
switch (state.status) {
|
|
case "capturing":
|
|
return {
|
|
label: dict.validation.capturing,
|
|
color: "text-blue-600 bg-blue-50",
|
|
icon: (
|
|
<div className="h-4 w-4 border-2 border-blue-600 border-t-transparent rounded-full animate-spin" />
|
|
),
|
|
}
|
|
case "validating":
|
|
return {
|
|
label: state.attempt
|
|
? dict.validation.validatingWithAttempt
|
|
.replace("{attempt}", String(state.attempt))
|
|
.replace("{max}", String(state.maxAttempts || 3))
|
|
: dict.validation.validating,
|
|
color: "text-blue-600 bg-blue-50",
|
|
icon: (
|
|
<div className="h-4 w-4 border-2 border-blue-600 border-t-transparent rounded-full animate-spin" />
|
|
),
|
|
}
|
|
case "success":
|
|
return {
|
|
label: dict.validation.valid,
|
|
color: "text-green-600 bg-green-50",
|
|
icon: <Check className="h-4 w-4" aria-hidden="true" />,
|
|
}
|
|
case "success_with_warnings":
|
|
return {
|
|
label: dict.validation.validWithWarnings,
|
|
color: "text-amber-600 bg-amber-50",
|
|
icon: (
|
|
<AlertTriangle className="h-4 w-4" aria-hidden="true" />
|
|
),
|
|
}
|
|
case "failed":
|
|
return {
|
|
label: dict.validation.issuesFound,
|
|
color: "text-yellow-600 bg-yellow-50",
|
|
icon: (
|
|
<AlertTriangle className="h-4 w-4" aria-hidden="true" />
|
|
),
|
|
}
|
|
case "error":
|
|
return {
|
|
label: dict.validation.error,
|
|
color: "text-red-600 bg-red-50",
|
|
icon: <X className="h-4 w-4" aria-hidden="true" />,
|
|
}
|
|
case "skipped":
|
|
return {
|
|
label: dict.validation.skipped,
|
|
color: "text-gray-600 bg-gray-50",
|
|
icon: <Check className="h-4 w-4" aria-hidden="true" />,
|
|
}
|
|
default:
|
|
return null
|
|
}
|
|
}
|
|
|
|
const statusDisplay = getStatusDisplay()
|
|
if (!statusDisplay || state.status === "idle") return null
|
|
|
|
return (
|
|
<div className="my-3 rounded-xl border border-border/60 bg-muted/30 overflow-hidden">
|
|
<div className="flex items-center justify-between px-4 py-3 bg-muted/50">
|
|
<div className="flex items-center gap-2">
|
|
<div className="w-6 h-6 rounded-md bg-primary/10 flex items-center justify-center">
|
|
<Eye
|
|
className="w-3.5 h-3.5 text-primary"
|
|
aria-hidden="true"
|
|
/>
|
|
</div>
|
|
<span className="text-sm font-medium text-foreground/80">
|
|
{dict.validation.title}
|
|
</span>
|
|
</div>
|
|
<div className="flex items-center gap-2">
|
|
<span
|
|
className={`text-xs font-medium px-2 py-0.5 rounded-full flex items-center gap-1 ${statusDisplay.color}`}
|
|
>
|
|
{statusDisplay.icon}
|
|
<span className="ml-1">{statusDisplay.label}</span>
|
|
</span>
|
|
{(state.result || state.error) && (
|
|
<button
|
|
type="button"
|
|
onClick={() => setIsExpanded(!isExpanded)}
|
|
className="p-1 rounded hover:bg-muted transition-colors"
|
|
>
|
|
{isExpanded ? (
|
|
<ChevronUp
|
|
className="w-4 h-4 text-muted-foreground"
|
|
aria-hidden="true"
|
|
/>
|
|
) : (
|
|
<ChevronDown
|
|
className="w-4 h-4 text-muted-foreground"
|
|
aria-hidden="true"
|
|
/>
|
|
)}
|
|
</button>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{/* Validation details when expanded */}
|
|
{isExpanded && (state.result || state.imageData) && (
|
|
<div className="px-4 py-3 border-t border-border/40 bg-muted/20 space-y-3">
|
|
{/* Captured image */}
|
|
{state.imageData && (
|
|
<div>
|
|
<div className="text-xs font-medium text-foreground/70 mb-2 flex items-center gap-1">
|
|
<ImageIcon
|
|
className="h-3 w-3"
|
|
aria-hidden="true"
|
|
/>
|
|
{dict.validation.capturedScreenshot}
|
|
</div>
|
|
<div className="rounded-lg border border-border/50 overflow-hidden bg-white">
|
|
<Image
|
|
src={state.imageData}
|
|
alt="Captured diagram for validation"
|
|
width={400}
|
|
height={300}
|
|
className="w-full h-auto max-h-48 object-contain"
|
|
unoptimized
|
|
/>
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* Issues */}
|
|
{state.result && state.result.issues.length > 0 && (
|
|
<div>
|
|
<div className="text-xs font-medium text-foreground/70 mb-2">
|
|
{dict.validation.issuesFoundLabel}
|
|
</div>
|
|
<div className="space-y-2">
|
|
{state.result.issues.map((issue, index) => (
|
|
<div
|
|
key={index}
|
|
className={`text-xs px-3 py-2 rounded-lg border ${
|
|
issue.severity === "critical"
|
|
? "bg-red-50 border-red-200 text-red-700 dark:bg-red-950 dark:border-red-800 dark:text-red-300"
|
|
: "bg-yellow-50 border-yellow-200 text-yellow-700 dark:bg-yellow-950 dark:border-yellow-800 dark:text-yellow-300"
|
|
}`}
|
|
>
|
|
<span className="font-medium uppercase text-[10px] mr-2">
|
|
[{issue.type}]
|
|
</span>
|
|
{issue.description}
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* Suggestions */}
|
|
{state.result && state.result.suggestions.length > 0 && (
|
|
<div>
|
|
<div className="text-xs font-medium text-foreground/70 mb-2">
|
|
{dict.validation.suggestions}
|
|
</div>
|
|
<ul className="text-xs text-foreground/60 space-y-1 list-disc list-inside">
|
|
{state.result.suggestions.map(
|
|
(suggestion, index) => (
|
|
<li key={index}>{suggestion}</li>
|
|
),
|
|
)}
|
|
</ul>
|
|
</div>
|
|
)}
|
|
|
|
{/* Valid result message */}
|
|
{state.result?.valid &&
|
|
state.result.issues.length === 0 && (
|
|
<div className="text-xs text-green-600 dark:text-green-400">
|
|
{dict.validation.passedValidation}
|
|
</div>
|
|
)}
|
|
</div>
|
|
)}
|
|
|
|
{/* Improve with Suggestions button - shown when validation passed but has suggestions */}
|
|
{showImproveButton && (
|
|
<div className="px-4 py-3 border-t border-border/40 bg-muted/10">
|
|
{hasRequestedImprovement ? (
|
|
<div className="flex items-center justify-center gap-2 px-4 py-2 text-sm font-medium text-green-600 dark:text-green-400">
|
|
<Check className="h-4 w-4" aria-hidden="true" />
|
|
{dict.validation.improvementRequested}
|
|
</div>
|
|
) : (
|
|
<>
|
|
<button
|
|
type="button"
|
|
onClick={handleImproveClick}
|
|
className="w-full flex items-center justify-center gap-2 px-4 py-2 text-sm font-medium text-primary bg-primary/10 hover:bg-primary/20 rounded-lg transition-colors"
|
|
>
|
|
<RefreshCw
|
|
className="h-4 w-4"
|
|
aria-hidden="true"
|
|
/>
|
|
{dict.validation.improveWithSuggestions}
|
|
</button>
|
|
<p className="text-xs text-muted-foreground mt-2 text-center">
|
|
{dict.validation.regenerateWithFeedback}
|
|
</p>
|
|
</>
|
|
)}
|
|
</div>
|
|
)}
|
|
|
|
{/* Error details when expanded */}
|
|
{isExpanded && state.error && (
|
|
<div className="px-4 py-3 border-t border-border/40 bg-red-50/50">
|
|
<div className="text-xs text-red-600">{state.error}</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
)
|
|
}
|