Compare commits

...

12 Commits

Author SHA1 Message Date
dayuan.jiang
011b883cdb feat(i18n): add validation strings for ValidationCard component
- Add validation section to en.json, zh.json, ja.json dictionaries
- Update ValidationCard to use useDictionary hook
- Replace all hardcoded English strings with i18n keys
2026-01-20 20:43:35 +09:00
dayuan.jiang
fd9302e736 fix: return empty string for valid result with no issues in formatValidationFeedback 2026-01-20 20:04:29 +09:00
dayuan.jiang
fa06c61538 fix: resolve TypeScript errors in electron-standalone
- Add forwardRef support to ChatInput component with ChatInputRef type
- Copy electron.d.ts to electron-standalone/electron folder
- Exclude electron-standalone from root tsconfig type checking
2026-01-20 19:49:12 +09:00
dayuan.jiang
60994d281e fix: improve VLM validation with bug fixes and i18n
- Fix race condition in pendingValidationRef (reject previous pending validation)
- Fix response format consistency (use streaming for all responses)
- Remove dead code (unused lastRequestRef and ValidationRequest interface)
- Consolidate duplicate types (re-export from validation-schema.ts)
- Add 'success_with_warnings' status for valid diagrams with warnings
- Fix tool card auto-collapse (only collapse once, respect user toggle)
- Set VLM validation default to disabled
- Add i18n support for diagram validation settings (en/zh/ja)
- Mark feature as experimental in settings UI
2026-01-20 19:45:26 +09:00
Jinze Yu
6640272f90 fix(validation): add aria-hidden to icons to prevent duplicate ID warning 2026-01-19 18:21:56 +09:00
Jinze Yu
86560e0fc0 fix(validation): use 'Valid' instead of 'Complete' for validation success
Change ValidationCard success label from 'Complete' to 'Valid' to avoid
conflicting with ToolCallCard's 'Complete' badge in E2E tests. This fixes
the diagram-generation E2E test that expects a specific count of 'Complete'
badges.
2026-01-19 10:41:58 +09:00
Jinze Yu
bb6c7c11ce Merge origin/main into features/validate-diagram-with-vlm 2026-01-19 01:35:34 +09:00
Jinze Yu
6b0296a97f fix(validation): extract schema to shared file for client/server compatibility
Move ValidationResultSchema to lib/validation-schema.ts to avoid importing
server-side modules (ai-providers) into client-side code. This fixes the
Turbopack build error caused by the hook importing from the API route.
2026-01-19 01:30:51 +09:00
Jinze Yu
85513b8c00 refactor(validation): use AI SDK experimental_useObject hook instead of raw fetch
- Change API endpoint from generateObject to streamObject for useObject compatibility
- Create useValidateDiagram hook using AI SDK's experimental_useObject for reactive validation
- Update useDiagramToolHandlers to accept validation function as parameter
- Update chat-panel to use new useValidateDiagram hook
- Remove validateRenderedDiagram function from lib/diagram-validator.ts (now in hook)
- Export ValidationResultSchema from API route for client-side use
2026-01-19 01:10:31 +09:00
Jinze Yu
24270e2622 refactor(validation): use AI SDK structured outputs and address review feedback
- Replace generateText + manual JSON parsing with generateObject and Zod schema
  for type-safe structured validation output
- Use AbortSignal.timeout() instead of Promise.race for cleaner timeout handling
- Add timeout validation with minimum 1000ms to handle malformed env values
- Remove unused xml parameter from validateRenderedDiagram API
- Remove parseValidationResponse function (now handled by schema)
- Clear validationStates on session switch and new chat to prevent memory leak
- Update 100ms render delay comment to clarify best-effort heuristic
- Remove unused useEffect import from ValidationCard
- Fix optional chaining lint warning in ValidationCard
- Add unit tests for formatValidationFeedback function
2026-01-19 00:47:49 +09:00
Jinze Yu
07c9e7d758 Merge remote-tracking branch 'origin/main' into features/validate-diagram-with-vlm 2026-01-17 20:59:51 +09:00
Jinze Yu
ee4c0149f1 [Feature] Add VLM-based diagram validation
Add automatic VLM (Vision Language Model) validation after display_diagram
tool execution. The system captures a screenshot of the rendered diagram,
sends it to a VLM for visual analysis, and uses feedback to improve
diagram quality through the existing retry mechanism.

Changes:
- Add /api/validate-diagram endpoint for VLM validation
- Add diagram-validator.ts for client-side validation orchestration
- Add validation-prompts.ts for VLM system prompts
- Add ValidationCard component to display validation status in chat
- Add PNG capture functionality to diagram context
- Integrate validation into tool handlers with retry support (max 3)
- Add "Improve with Suggestions" button for manual regeneration
- Add settings toggle to enable/disable VLM validation
- Add getValidationModel() helper in ai-providers.ts
2026-01-17 17:16:12 +09:00
20 changed files with 1741 additions and 399 deletions

View File

@@ -0,0 +1,136 @@
/**
* API endpoint for VLM-based diagram validation.
* Accepts a PNG image and streams validation results using useObject-compatible format.
*/
import { streamObject } from "ai"
import { getValidationModel } from "@/lib/ai-providers"
import { VALIDATION_SYSTEM_PROMPT } from "@/lib/validation-prompts"
import {
type ValidationResult,
ValidationResultSchema,
} from "@/lib/validation-schema"
export const maxDuration = 30
interface ValidateDiagramRequest {
imageData: string // Base64 PNG data URL
sessionId?: string
}
// Default valid result for disabled/error cases
const DEFAULT_VALID_RESULT: ValidationResult = {
valid: true,
issues: [],
suggestions: [],
}
/**
* Create a streaming response for useObject compatibility.
* useObject expects text stream format, not plain JSON.
*/
function createStreamingResponse(result: ValidationResult): Response {
const encoder = new TextEncoder()
const stream = new ReadableStream({
start(controller) {
// Stream the JSON as text (useObject parses this)
controller.enqueue(encoder.encode(JSON.stringify(result)))
controller.close()
},
})
return new Response(stream, {
headers: { "Content-Type": "text/plain; charset=utf-8" },
})
}
export async function POST(req: Request): Promise<Response> {
try {
// Check if VLM validation is enabled (default: true)
const enableValidation = process.env.ENABLE_VLM_VALIDATION !== "false"
if (!enableValidation) {
return createStreamingResponse(DEFAULT_VALID_RESULT)
}
const body: ValidateDiagramRequest = await req.json()
const { imageData, sessionId } = body
if (!imageData) {
return Response.json(
{ error: "Missing imageData" },
{ status: 400 },
)
}
// Validate image data format
if (
!imageData.startsWith("data:image/png;base64,") &&
!imageData.startsWith("data:image/")
) {
return Response.json(
{ error: "Invalid image data format" },
{ status: 400 },
)
}
// Get the validation model
let model
try {
model = getValidationModel()
} catch (error) {
console.warn(
"[validate-diagram] Validation model not available:",
error,
)
// Return valid if no vision model is configured
return createStreamingResponse(DEFAULT_VALID_RESULT)
}
// Parse timeout with validation (minimum 1000ms, default 10000ms)
const timeout =
Math.max(
1000,
parseInt(process.env.VALIDATION_TIMEOUT || "10000", 10),
) || 10000
// Stream the VLM response for useObject consumption
const result = streamObject({
model,
schema: ValidationResultSchema,
system: VALIDATION_SYSTEM_PROMPT,
messages: [
{
role: "user",
content: [
{
type: "image",
image: imageData,
},
{
type: "text",
text: "Please analyze this diagram for visual quality issues.",
},
],
},
],
maxOutputTokens: 1024,
abortSignal: AbortSignal.timeout(timeout),
onFinish: ({ object }) => {
if (sessionId && object) {
console.log(
`[validate-diagram] Session ${sessionId}: valid=${object.valid}, issues=${object.issues?.length ?? 0}`,
)
}
},
})
return result.toTextStreamResponse()
} catch (error) {
// Log with session context if available
const errorMessage =
error instanceof Error ? error.message : String(error)
console.error("[validate-diagram] Error:", errorMessage)
// On error, return valid to not block the user
return createStreamingResponse(DEFAULT_VALID_RESULT)
}
}

View File

@@ -9,7 +9,14 @@ import {
Send,
} from "lucide-react"
import type React from "react"
import { useCallback, useEffect, useRef, useState } from "react"
import {
forwardRef,
useCallback,
useEffect,
useImperativeHandle,
useRef,
useState,
} from "react"
import { toast } from "sonner"
import { ButtonWithTooltip } from "@/components/button-with-tooltip"
import { ErrorToast } from "@/components/error-toast"
@@ -138,6 +145,10 @@ function showValidationErrors(errors: string[], dict: any) {
}
}
export interface ChatInputRef {
focus: () => void
}
interface ChatInputProps {
input: string
status: "submitted" | "streaming" | "ready" | "error"
@@ -165,134 +176,212 @@ interface ChatInputProps {
onFocused?: () => void
}
export function ChatInput({
input,
status,
onSubmit,
onChange,
files = [],
onFileChange = () => {},
pdfData = new Map(),
urlData,
onUrlChange,
sessionId,
error = null,
models = [],
selectedModelId,
onModelSelect = () => {},
showUnvalidatedModels = false,
onConfigureModels = () => {},
shouldFocus = false,
onFocused,
}: ChatInputProps) {
const dict = useDictionary()
const {
chartXML,
diagramHistory,
saveDiagramToFile,
showSaveDialog,
setShowSaveDialog,
} = useDiagram()
export const ChatInput = forwardRef<ChatInputRef, ChatInputProps>(
function ChatInput(
{
input,
status,
onSubmit,
onChange,
files = [],
onFileChange = () => {},
pdfData = new Map(),
urlData,
onUrlChange,
sessionId,
error = null,
models = [],
selectedModelId,
onModelSelect = () => {},
showUnvalidatedModels = false,
onConfigureModels = () => {},
shouldFocus = false,
onFocused,
},
ref,
) {
const dict = useDictionary()
const {
chartXML,
diagramHistory,
saveDiagramToFile,
showSaveDialog,
setShowSaveDialog,
} = useDiagram()
const textareaRef = useRef<HTMLTextAreaElement>(null)
const fileInputRef = useRef<HTMLInputElement>(null)
const [isDragging, setIsDragging] = useState(false)
const textareaRef = useRef<HTMLTextAreaElement>(null)
const fileInputRef = useRef<HTMLInputElement>(null)
const [isDragging, setIsDragging] = useState(false)
// Focus the textarea when shouldFocus becomes true
// Use setTimeout to ensure focus happens after drawio iframe settles
useEffect(() => {
if (shouldFocus) {
const timer = setTimeout(() => {
// Expose focus method via ref
useImperativeHandle(ref, () => ({
focus: () => {
textareaRef.current?.focus()
onFocused?.()
}, 150)
return () => clearTimeout(timer)
}
}, [shouldFocus, onFocused])
},
}))
const [showHistory, setShowHistory] = useState(false)
const [showUrlDialog, setShowUrlDialog] = useState(false)
const [isExtractingUrl, setIsExtractingUrl] = useState(false)
const [sendShortcut, setSendShortcut] = useState("ctrl-enter")
// Allow retry when there's an error (even if status is still "streaming" or "submitted")
const isDisabled =
(status === "streaming" || status === "submitted") && !error
// Focus the textarea when shouldFocus becomes true
// Use setTimeout to ensure focus happens after drawio iframe settles
useEffect(() => {
if (shouldFocus) {
const timer = setTimeout(() => {
textareaRef.current?.focus()
onFocused?.()
}, 150)
return () => clearTimeout(timer)
}
}, [shouldFocus, onFocused])
const adjustTextareaHeight = useCallback(() => {
const textarea = textareaRef.current
if (textarea) {
textarea.style.height = "auto"
textarea.style.height = `${Math.min(textarea.scrollHeight, 200)}px`
}
}, [])
// Handle programmatic input changes (e.g., setInput("") after form submission)
useEffect(() => {
adjustTextareaHeight()
}, [input, adjustTextareaHeight])
const [showHistory, setShowHistory] = useState(false)
const [showUrlDialog, setShowUrlDialog] = useState(false)
const [isExtractingUrl, setIsExtractingUrl] = useState(false)
const [sendShortcut, setSendShortcut] = useState("ctrl-enter")
// Allow retry when there's an error (even if status is still "streaming" or "submitted")
const isDisabled =
(status === "streaming" || status === "submitted") && !error
// Load send shortcut preference from localStorage and listen for changes
useEffect(() => {
const stored = localStorage.getItem(STORAGE_KEYS.sendShortcut)
if (stored) setSendShortcut(stored)
const adjustTextareaHeight = useCallback(() => {
const textarea = textareaRef.current
if (textarea) {
textarea.style.height = "auto"
textarea.style.height = `${Math.min(textarea.scrollHeight, 200)}px`
}
}, [])
// Handle programmatic input changes (e.g., setInput("") after form submission)
useEffect(() => {
adjustTextareaHeight()
}, [input, adjustTextareaHeight])
const handleChange = (e: CustomEvent<string>) =>
setSendShortcut(e.detail)
window.addEventListener(
"sendShortcutChange",
handleChange as EventListener,
)
return () =>
window.removeEventListener(
// Load send shortcut preference from localStorage and listen for changes
useEffect(() => {
const stored = localStorage.getItem(STORAGE_KEYS.sendShortcut)
if (stored) setSendShortcut(stored)
const handleChange = (e: CustomEvent<string>) =>
setSendShortcut(e.detail)
window.addEventListener(
"sendShortcutChange",
handleChange as EventListener,
)
}, [])
return () =>
window.removeEventListener(
"sendShortcutChange",
handleChange as EventListener,
)
}, [])
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
onChange(e)
adjustTextareaHeight()
}
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
onChange(e)
adjustTextareaHeight()
}
const handleKeyDown = (e: React.KeyboardEvent) => {
const shouldSend =
sendShortcut === "enter"
? e.key === "Enter" && !e.shiftKey && !e.ctrlKey && !e.metaKey
: (e.metaKey || e.ctrlKey) && e.key === "Enter"
const handleKeyDown = (e: React.KeyboardEvent) => {
const shouldSend =
sendShortcut === "enter"
? e.key === "Enter" &&
!e.shiftKey &&
!e.ctrlKey &&
!e.metaKey
: (e.metaKey || e.ctrlKey) && e.key === "Enter"
if (shouldSend) {
e.preventDefault()
const form = e.currentTarget.closest("form")
if (form && input.trim() && !isDisabled) {
form.requestSubmit()
if (shouldSend) {
e.preventDefault()
const form = e.currentTarget.closest("form")
if (form && input.trim() && !isDisabled) {
form.requestSubmit()
}
}
}
}
const handlePaste = async (e: React.ClipboardEvent) => {
if (isDisabled) return
const handlePaste = async (e: React.ClipboardEvent) => {
if (isDisabled) return
const items = e.clipboardData.items
const imageItems = Array.from(items).filter((item) =>
item.type.startsWith("image/"),
)
const items = e.clipboardData.items
const imageItems = Array.from(items).filter((item) =>
item.type.startsWith("image/"),
)
if (imageItems.length > 0) {
const imageFiles = (
await Promise.all(
imageItems.map(async (item, index) => {
const file = item.getAsFile()
if (!file) return null
return new File(
[file],
`pasted-image-${Date.now()}-${index}.${file.type.split("/")[1]}`,
{ type: file.type },
)
}),
if (imageItems.length > 0) {
const imageFiles = (
await Promise.all(
imageItems.map(async (item, index) => {
const file = item.getAsFile()
if (!file) return null
return new File(
[file],
`pasted-image-${Date.now()}-${index}.${file.type.split("/")[1]}`,
{ type: file.type },
)
}),
)
).filter((f): f is File => f !== null)
const { validFiles, errors } = validateFiles(
imageFiles,
files.length,
dict,
)
).filter((f): f is File => f !== null)
showValidationErrors(errors, dict)
if (validFiles.length > 0) {
onFileChange([...files, ...validFiles])
}
}
}
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newFiles = Array.from(e.target.files || [])
const { validFiles, errors } = validateFiles(
newFiles,
files.length,
dict,
)
showValidationErrors(errors, dict)
if (validFiles.length > 0) {
onFileChange([...files, ...validFiles])
}
if (fileInputRef.current) {
fileInputRef.current.value = ""
}
}
const handleRemoveFile = (fileToRemove: File) => {
onFileChange(files.filter((file) => file !== fileToRemove))
if (fileInputRef.current) {
fileInputRef.current.value = ""
}
}
const triggerFileInput = () => {
fileInputRef.current?.click()
}
const handleDragOver = (e: React.DragEvent<HTMLFormElement>) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(true)
}
const handleDragLeave = (e: React.DragEvent<HTMLFormElement>) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(false)
}
const handleDrop = (e: React.DragEvent<HTMLFormElement>) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(false)
if (isDisabled) return
const droppedFiles = e.dataTransfer.files
const supportedFiles = Array.from(droppedFiles).filter((file) =>
isValidFileType(file),
)
const { validFiles, errors } = validateFiles(
imageFiles,
supportedFiles,
files.length,
dict,
)
@@ -301,278 +390,219 @@ export function ChatInput({
onFileChange([...files, ...validFiles])
}
}
}
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newFiles = Array.from(e.target.files || [])
const { validFiles, errors } = validateFiles(
newFiles,
files.length,
dict,
)
showValidationErrors(errors, dict)
if (validFiles.length > 0) {
onFileChange([...files, ...validFiles])
const handleUrlExtract = async (url: string) => {
if (!onUrlChange) return
setIsExtractingUrl(true)
try {
const existing = urlData
? new Map(urlData)
: new Map<string, UrlData>()
existing.set(url, {
url,
title: url,
content: "",
charCount: 0,
isExtracting: true,
})
onUrlChange(existing)
const data = await extractUrlContent(url)
const newUrlData = new Map(existing)
newUrlData.set(url, data)
onUrlChange(newUrlData)
setShowUrlDialog(false)
} catch (error) {
// Remove the URL from the data map on error
const newUrlData = urlData
? new Map(urlData)
: new Map<string, UrlData>()
newUrlData.delete(url)
onUrlChange(newUrlData)
showErrorToast(
<span className="text-muted-foreground">
{error instanceof Error
? error.message
: "Failed to extract URL content"}
</span>,
)
} finally {
setIsExtractingUrl(false)
}
}
if (fileInputRef.current) {
fileInputRef.current.value = ""
}
}
const handleRemoveFile = (fileToRemove: File) => {
onFileChange(files.filter((file) => file !== fileToRemove))
if (fileInputRef.current) {
fileInputRef.current.value = ""
}
}
const triggerFileInput = () => {
fileInputRef.current?.click()
}
const handleDragOver = (e: React.DragEvent<HTMLFormElement>) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(true)
}
const handleDragLeave = (e: React.DragEvent<HTMLFormElement>) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(false)
}
const handleDrop = (e: React.DragEvent<HTMLFormElement>) => {
e.preventDefault()
e.stopPropagation()
setIsDragging(false)
if (isDisabled) return
const droppedFiles = e.dataTransfer.files
const supportedFiles = Array.from(droppedFiles).filter((file) =>
isValidFileType(file),
)
const { validFiles, errors } = validateFiles(
supportedFiles,
files.length,
dict,
)
showValidationErrors(errors, dict)
if (validFiles.length > 0) {
onFileChange([...files, ...validFiles])
}
}
const handleUrlExtract = async (url: string) => {
if (!onUrlChange) return
setIsExtractingUrl(true)
try {
const existing = urlData
? new Map(urlData)
: new Map<string, UrlData>()
existing.set(url, {
url,
title: url,
content: "",
charCount: 0,
isExtracting: true,
})
onUrlChange(existing)
const data = await extractUrlContent(url)
const newUrlData = new Map(existing)
newUrlData.set(url, data)
onUrlChange(newUrlData)
setShowUrlDialog(false)
} catch (error) {
// Remove the URL from the data map on error
const newUrlData = urlData
? new Map(urlData)
: new Map<string, UrlData>()
newUrlData.delete(url)
onUrlChange(newUrlData)
showErrorToast(
<span className="text-muted-foreground">
{error instanceof Error
? error.message
: "Failed to extract URL content"}
</span>,
)
} finally {
setIsExtractingUrl(false)
}
}
return (
<form
onSubmit={onSubmit}
className={`w-full transition-all duration-200 ${
isDragging
? "ring-2 ring-primary ring-offset-2 rounded-2xl"
: ""
}`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
>
{/* File & URL previews */}
{(files.length > 0 || (urlData && urlData.size > 0)) && (
<div className="mb-3">
<FilePreviewList
files={files}
onRemoveFile={handleRemoveFile}
pdfData={pdfData}
urlData={urlData}
onRemoveUrl={
onUrlChange
? (url) => {
const next = new Map(urlData)
next.delete(url)
onUrlChange(next)
}
: undefined
}
return (
<form
onSubmit={onSubmit}
className={`w-full transition-all duration-200 ${
isDragging
? "ring-2 ring-primary ring-offset-2 rounded-2xl"
: ""
}`}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
>
{/* File & URL previews */}
{(files.length > 0 || (urlData && urlData.size > 0)) && (
<div className="mb-3">
<FilePreviewList
files={files}
onRemoveFile={handleRemoveFile}
pdfData={pdfData}
urlData={urlData}
onRemoveUrl={
onUrlChange
? (url) => {
const next = new Map(urlData)
next.delete(url)
onUrlChange(next)
}
: undefined
}
/>
</div>
)}
<div className="relative rounded-2xl border border-border bg-background shadow-sm focus-within:ring-2 focus-within:ring-primary/20 focus-within:border-primary/50 transition-all duration-200">
<Textarea
ref={textareaRef}
value={input}
onChange={handleChange}
onKeyDown={handleKeyDown}
onPaste={handlePaste}
placeholder={dict.chat.placeholder}
disabled={isDisabled}
aria-label="Chat input"
className="min-h-[60px] max-h-[200px] resize-none border-0 bg-transparent px-4 py-3 text-sm focus-visible:ring-0 focus-visible:ring-offset-0 placeholder:text-muted-foreground/60 scrollbar-thin"
/>
</div>
)}
<div className="relative rounded-2xl border border-border bg-background shadow-sm focus-within:ring-2 focus-within:ring-primary/20 focus-within:border-primary/50 transition-all duration-200">
<Textarea
ref={textareaRef}
value={input}
onChange={handleChange}
onKeyDown={handleKeyDown}
onPaste={handlePaste}
placeholder={dict.chat.placeholder}
disabled={isDisabled}
aria-label="Chat input"
className="min-h-[60px] max-h-[200px] resize-none border-0 bg-transparent px-4 py-3 text-sm focus-visible:ring-0 focus-visible:ring-offset-0 placeholder:text-muted-foreground/60 scrollbar-thin"
/>
<div className="flex items-center justify-end gap-1 px-3 py-2 border-t border-border/50">
<div className="flex items-center gap-1 overflow-x-hidden">
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={() => setShowHistory(true)}
disabled={isDisabled || diagramHistory.length === 0}
tooltipContent={dict.chat.diagramHistory}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<History className="h-4 w-4" />
</ButtonWithTooltip>
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={() => setShowSaveDialog(true)}
disabled={isDisabled || !isRealDiagram(chartXML)}
tooltipContent={dict.chat.saveDiagram}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<Download className="h-4 w-4" />
</ButtonWithTooltip>
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={triggerFileInput}
disabled={isDisabled}
tooltipContent={dict.chat.uploadFile}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<ImageIcon className="h-4 w-4" />
</ButtonWithTooltip>
{onUrlChange && (
<div className="flex items-center justify-end gap-1 px-3 py-2 border-t border-border/50">
<div className="flex items-center gap-1 overflow-x-hidden">
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={() => setShowUrlDialog(true)}
disabled={isDisabled}
tooltipContent={dict.chat.ExtractURL}
onClick={() => setShowHistory(true)}
disabled={
isDisabled || diagramHistory.length === 0
}
tooltipContent={dict.chat.diagramHistory}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<Link className="h-4 w-4" />
<History className="h-4 w-4" />
</ButtonWithTooltip>
)}
<input
type="file"
ref={fileInputRef}
className="hidden"
onChange={handleFileChange}
accept="image/*,.pdf,application/pdf,text/*,.md,.markdown,.json,.csv,.xml,.yaml,.yml,.toml"
multiple
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={() => setShowSaveDialog(true)}
disabled={
isDisabled || !isRealDiagram(chartXML)
}
tooltipContent={dict.chat.saveDiagram}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<Download className="h-4 w-4" />
</ButtonWithTooltip>
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={triggerFileInput}
disabled={isDisabled}
tooltipContent={dict.chat.uploadFile}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<ImageIcon className="h-4 w-4" />
</ButtonWithTooltip>
{onUrlChange && (
<ButtonWithTooltip
type="button"
variant="ghost"
size="sm"
onClick={() => setShowUrlDialog(true)}
disabled={isDisabled}
tooltipContent={dict.chat.ExtractURL}
className="h-8 w-8 p-0 text-muted-foreground hover:text-foreground"
>
<Link className="h-4 w-4" />
</ButtonWithTooltip>
)}
<input
type="file"
ref={fileInputRef}
className="hidden"
onChange={handleFileChange}
accept="image/*,.pdf,application/pdf,text/*,.md,.markdown,.json,.csv,.xml,.yaml,.yml,.toml"
multiple
disabled={isDisabled}
/>
</div>
<ModelSelector
models={models}
selectedModelId={selectedModelId}
onSelect={onModelSelect}
onConfigure={onConfigureModels}
disabled={isDisabled}
showUnvalidatedModels={showUnvalidatedModels}
/>
<div className="w-px h-5 bg-border mx-1" />
<Button
type="submit"
disabled={isDisabled || !input.trim()}
size="sm"
className="h-8 px-4 rounded-xl font-medium shadow-sm"
aria-label={
isDisabled ? dict.chat.sending : dict.chat.send
}
>
{isDisabled ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<>
<Send className="h-4 w-4 mr-1.5" />
{dict.chat.send}
</>
)}
</Button>
</div>
<ModelSelector
models={models}
selectedModelId={selectedModelId}
onSelect={onModelSelect}
onConfigure={onConfigureModels}
disabled={isDisabled}
showUnvalidatedModels={showUnvalidatedModels}
/>
<div className="w-px h-5 bg-border mx-1" />
<Button
type="submit"
disabled={isDisabled || !input.trim()}
size="sm"
className="h-8 px-4 rounded-xl font-medium shadow-sm"
aria-label={
isDisabled ? dict.chat.sending : dict.chat.send
}
>
{isDisabled ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<>
<Send className="h-4 w-4 mr-1.5" />
{dict.chat.send}
</>
)}
</Button>
</div>
</div>
<HistoryDialog
showHistory={showHistory}
onToggleHistory={setShowHistory}
/>
<SaveDialog
open={showSaveDialog}
onOpenChange={setShowSaveDialog}
onSave={(filename, format) =>
saveDiagramToFile(
filename,
format,
sessionId,
dict.save.savedSuccessfully,
)
}
defaultFilename={`diagram-${new Date()
.toISOString()
.slice(0, 10)}`}
/>
{onUrlChange && (
<UrlInputDialog
open={showUrlDialog}
onOpenChange={setShowUrlDialog}
onSubmit={handleUrlExtract}
isExtracting={isExtractingUrl}
<HistoryDialog
showHistory={showHistory}
onToggleHistory={setShowHistory}
/>
)}
</form>
)
}
<SaveDialog
open={showSaveDialog}
onOpenChange={setShowSaveDialog}
onSave={(filename, format) =>
saveDiagramToFile(
filename,
format,
sessionId,
dict.save.savedSuccessfully,
)
}
defaultFilename={`diagram-${new Date()
.toISOString()
.slice(0, 10)}`}
/>
{onUrlChange && (
<UrlInputDialog
open={showUrlDialog}
onOpenChange={setShowUrlDialog}
onSubmit={handleUrlExtract}
isExtracting={isExtractingUrl}
/>
)}
</form>
)
},
)

View File

@@ -28,6 +28,8 @@ import {
import { ChatLobby } from "@/components/chat/ChatLobby"
import { ToolCallCard } from "@/components/chat/ToolCallCard"
import type { DiagramOperation, ToolPartLike } from "@/components/chat/types"
import type { ValidationState } from "@/components/chat/ValidationCard"
import { ValidationCard } from "@/components/chat/ValidationCard"
import { ScrollArea } from "@/components/ui/scroll-area"
import { useDictionary } from "@/hooks/use-dictionary"
import { getApiEndpoint } from "@/lib/base-path"
@@ -148,6 +150,8 @@ interface ChatMessageDisplayProps {
onSelectSession?: (id: string) => void
onDeleteSession?: (id: string) => void
loadedMessageIdsRef?: MutableRefObject<Set<string>>
validationStates?: Record<string, ValidationState>
onImproveWithSuggestions?: (feedback: string) => void
}
export function ChatMessageDisplay({
@@ -165,6 +169,8 @@ export function ChatMessageDisplay({
onSelectSession,
onDeleteSession,
loadedMessageIdsRef,
validationStates = {},
onImproveWithSuggestions,
}: ChatMessageDisplayProps) {
const dict = useDictionary()
const { chartXML, loadDiagram: onDisplayChart } = useDiagram()
@@ -429,11 +435,15 @@ export function ChatMessageDisplay({
const toolPart = part as ToolPartLike
const { toolCallId, state, input } = toolPart
// Auto-collapse on completion, but only if user hasn't manually toggled
if (state === "output-available") {
setExpandedTools((prev) => ({
...prev,
[toolCallId]: false,
}))
setExpandedTools((prev) => {
// Only auto-collapse if not already set (user hasn't interacted)
if (prev[toolCallId] === undefined) {
return { ...prev, [toolCallId]: false }
}
return prev
})
}
if (
@@ -911,30 +921,56 @@ export function ChatMessageDisplay({
return groups.map(
(group, groupIndex) => {
if (group.type === "tool") {
const toolPart = group
.parts[0] as ToolPartLike
const toolCallId =
toolPart.toolCallId
const isDisplayDiagram =
toolPart.type ===
"tool-display_diagram"
const validationState =
validationStates[
toolCallId
]
return (
<ToolCallCard
<div
key={`${message.id}-tool-${group.startIndex}`}
part={
group
.parts[0] as ToolPartLike
}
expandedTools={
expandedTools
}
setExpandedTools={
setExpandedTools
}
onCopy={
copyMessageToClipboard
}
copiedToolCallId={
copiedToolCallId
}
copyFailedToolCallId={
copyFailedToolCallId
}
dict={dict}
/>
>
<ToolCallCard
part={
toolPart
}
expandedTools={
expandedTools
}
setExpandedTools={
setExpandedTools
}
onCopy={
copyMessageToClipboard
}
copiedToolCallId={
copiedToolCallId
}
copyFailedToolCallId={
copyFailedToolCallId
}
dict={dict}
/>
{/* Show validation card for display_diagram tools */}
{isDisplayDiagram &&
validationState && (
<ValidationCard
state={
validationState
}
onImproveWithSuggestions={
onImproveWithSuggestions
}
/>
)}
</div>
)
}

View File

@@ -29,15 +29,18 @@ import { useDiagramToolHandlers } from "@/hooks/use-diagram-tool-handlers"
import { useDictionary } from "@/hooks/use-dictionary"
import { getSelectedAIConfig, useModelConfig } from "@/hooks/use-model-config"
import { useSessionManager } from "@/hooks/use-session-manager"
import { useValidateDiagram } from "@/hooks/use-validate-diagram"
import { getApiEndpoint } from "@/lib/base-path"
import { findCachedResponse } from "@/lib/cached-responses"
import { formatMessage } from "@/lib/i18n/utils"
import { isPdfFile, isTextFile } from "@/lib/pdf-utils"
import { sanitizeMessages } from "@/lib/session-storage"
import { STORAGE_KEYS } from "@/lib/storage"
import type { UrlData } from "@/lib/url-utils"
import { type FileData, useFileProcessor } from "@/lib/use-file-processor"
import { useQuotaManager } from "@/lib/use-quota-manager"
import { cn, formatXML, isRealDiagram } from "@/lib/utils"
import type { ValidationState } from "./chat/ValidationCard"
import { ChatMessageDisplay } from "./chat-message-display"
import { DevXmlSimulator } from "./dev-xml-simulator"
@@ -75,7 +78,8 @@ interface ChatPanelProps {
// Constants for tool states
const TOOL_ERROR_STATE = "output-error" as const
const DEBUG = process.env.NODE_ENV === "development"
const MAX_AUTO_RETRY_COUNT = 1
// Increased to 3 to support VLM validation retries (matches MAX_VALIDATION_RETRIES)
const MAX_AUTO_RETRY_COUNT = 3
const MAX_CONTINUATION_RETRY_COUNT = 2 // Limit for truncation continuation retries
@@ -120,6 +124,7 @@ export default function ChatPanel({
latestSvg,
clearDiagram,
getThumbnailSvg,
captureValidationPng,
diagramHistory,
setDiagramHistory,
} = useDiagram()
@@ -173,6 +178,7 @@ export default function ChatPanel({
const [dailyTokenLimit, setDailyTokenLimit] = useState(0)
const [tpmLimit, setTpmLimit] = useState(0)
const [minimalStyle, setMinimalStyle] = useState(false)
const [vlmValidationEnabled, setVlmValidationEnabled] = useState(false)
const [shouldFocusInput, setShouldFocusInput] = useState(false)
// Restore input from sessionStorage on mount (when ChatPanel remounts due to key change)
@@ -183,6 +189,14 @@ export default function ChatPanel({
}
}, [])
// Load VLM validation setting from localStorage on mount
useEffect(() => {
const stored = localStorage.getItem(STORAGE_KEYS.vlmValidationEnabled)
if (stored !== null) {
setVlmValidationEnabled(stored === "true")
}
}, [])
// Check config on mount
useEffect(() => {
fetch(getApiEndpoint("/api/config"))
@@ -270,6 +284,46 @@ export default function ChatPanel({
> | null>(null)
const LOCAL_STORAGE_DEBOUNCE_MS = 1000 // Save at most once per second
// Validation state for displaying VLM validation progress
// Key: toolCallId, Value: ValidationState
const [validationStates, setValidationStates] = useState<
Record<string, ValidationState>
>({})
// Callback to update validation state from tool handler
const handleValidationStateChange = useCallback(
(toolCallId: string, state: ValidationState) => {
setValidationStates((prev) => ({
...prev,
[toolCallId]: state,
}))
},
[],
)
// Handler for VLM validation setting change
const handleVlmValidationChange = useCallback((value: boolean) => {
setVlmValidationEnabled(value)
localStorage.setItem(STORAGE_KEYS.vlmValidationEnabled, String(value))
}, [])
// Ref to store the sendMessage function for use in callbacks
const sendMessageRef = useRef<typeof sendMessage | null>(null)
// Callback to improve diagram with validation suggestions
const handleImproveWithSuggestions = useCallback((feedback: string) => {
if (sendMessageRef.current) {
// Send the feedback as a new user message to trigger regeneration
sendMessageRef.current({
role: "user",
parts: [{ type: "text", text: feedback }],
})
}
}, [])
// VLM validation hook using AI SDK's useObject
const { validateWithFallback } = useValidateDiagram()
// Diagram tool handlers (display_diagram, edit_diagram, append_diagram)
const { handleToolCall } = useDiagramToolHandlers({
partialXmlRef,
@@ -278,6 +332,11 @@ export default function ChatPanel({
onDisplayChart,
onFetchChart,
onExport,
captureValidationPng,
validateDiagram: validateWithFallback,
enableVlmValidation: vlmValidationEnabled,
sessionId,
onValidationStateChange: handleValidationStateChange,
})
const { messages, sendMessage, addToolOutput, status, error, setMessages } =
@@ -426,6 +485,11 @@ export default function ChatPanel({
},
})
// Store sendMessage in ref for use in callbacks (like handleImproveWithSuggestions)
useEffect(() => {
sendMessageRef.current = sendMessage
}, [sendMessage])
// Ref to track latest messages for unload persistence
const messagesRef = useRef(messages)
useEffect(() => {
@@ -820,6 +884,7 @@ export default function ChatPanel({
} else {
justLoadedSessionIdRef.current = null
}
setValidationStates({}) // Clear validation states when switching sessions
syncUIWithSession(sessionData)
router.replace(`?session=${sessionId}`, { scroll: false })
}
@@ -860,6 +925,7 @@ export default function ChatPanel({
setInput("")
clearDiagram()
setDiagramHistory([])
setValidationStates({}) // Clear validation states to prevent memory leak
handleFileChange([]) // Use handleFileChange to also clear pdfData
setUrlData(new Map())
const newSessionId = `session-${Date.now()}-${Math.random()
@@ -1266,6 +1332,8 @@ export default function ChatPanel({
onSelectSession={handleSelectSession}
onDeleteSession={handleDeleteSession}
loadedMessageIdsRef={loadedMessageIdsRef}
validationStates={validationStates}
onImproveWithSuggestions={handleImproveWithSuggestions}
/>
</main>
@@ -1315,6 +1383,8 @@ export default function ChatPanel({
onToggleDarkMode={onToggleDarkMode}
minimalStyle={minimalStyle}
onMinimalStyleChange={setMinimalStyle}
vlmValidationEnabled={vlmValidationEnabled}
onVlmValidationChange={handleVlmValidationChange}
/>
<ModelConfigDialog

View File

@@ -67,8 +67,8 @@ export function ToolCallCard({
}: ToolCallCardProps) {
const callId = part.toolCallId
const { state, input, output } = part
// Default to collapsed if tool is complete, expanded if still streaming
const isExpanded = expandedTools[callId] ?? state !== "output-available"
// Default to expanded for all states (user can manually collapse if needed)
const isExpanded = expandedTools[callId] ?? true
const toolName = part.type?.replace("tool-", "")
const isCopied = copiedToolCallId === callId

View File

@@ -0,0 +1,328 @@
"use client"
import {
AlertTriangle,
Check,
ChevronDown,
ChevronUp,
Eye,
ImageIcon,
RefreshCw,
X,
} from "lucide-react"
import Image from "next/image"
import { useState } from "react"
import { useDictionary } from "@/hooks/use-dictionary"
import type { ValidationResult } from "@/lib/diagram-validator"
export type ValidationStatus =
| "idle"
| "capturing"
| "validating"
| "success"
| "success_with_warnings"
| "failed"
| "error"
| "skipped"
export interface ValidationState {
status: ValidationStatus
attempt?: number
maxAttempts?: number
result?: ValidationResult
error?: string
imageData?: string // Base64 PNG data URL
}
interface ValidationCardProps {
state: ValidationState
onImproveWithSuggestions?: (feedback: string) => void
}
export function ValidationCard({
state,
onImproveWithSuggestions,
}: ValidationCardProps) {
const dict = useDictionary()
const [isExpanded, setIsExpanded] = useState(
state.status === "validating" || state.status === "failed",
)
const [hasRequestedImprovement, setHasRequestedImprovement] =
useState(false)
// Generate improvement feedback from validation result
const generateImprovementFeedback = (): string => {
if (!state.result) return ""
const lines: string[] = []
lines.push(
"Please improve the diagram based on the following visual analysis feedback:",
)
lines.push("")
if (state.result.issues.length > 0) {
lines.push("Issues to address:")
for (const issue of state.result.issues) {
lines.push(
` - [${issue.severity}] ${issue.type}: ${issue.description}`,
)
}
lines.push("")
}
if (state.result.suggestions.length > 0) {
lines.push("Suggestions for improvement:")
for (const suggestion of state.result.suggestions) {
lines.push(` - ${suggestion}`)
}
lines.push("")
}
lines.push("Regenerate the diagram with these improvements applied.")
return lines.join("\n")
}
const handleImproveClick = () => {
if (
!onImproveWithSuggestions ||
!state.result ||
hasRequestedImprovement
)
return
setHasRequestedImprovement(true)
const feedback = generateImprovementFeedback()
onImproveWithSuggestions(feedback)
}
// Check if we should show the improve button
const showImproveButton =
onImproveWithSuggestions &&
state.result &&
(state.status === "success" ||
state.status === "success_with_warnings" ||
state.status === "skipped") &&
(state.result.issues.length > 0 || state.result.suggestions.length > 0)
const getStatusDisplay = () => {
switch (state.status) {
case "capturing":
return {
label: dict.validation.capturing,
color: "text-blue-600 bg-blue-50",
icon: (
<div className="h-4 w-4 border-2 border-blue-600 border-t-transparent rounded-full animate-spin" />
),
}
case "validating":
return {
label: state.attempt
? dict.validation.validatingWithAttempt
.replace("{attempt}", String(state.attempt))
.replace("{max}", String(state.maxAttempts || 3))
: dict.validation.validating,
color: "text-blue-600 bg-blue-50",
icon: (
<div className="h-4 w-4 border-2 border-blue-600 border-t-transparent rounded-full animate-spin" />
),
}
case "success":
return {
label: dict.validation.valid,
color: "text-green-600 bg-green-50",
icon: <Check className="h-4 w-4" aria-hidden="true" />,
}
case "success_with_warnings":
return {
label: dict.validation.validWithWarnings,
color: "text-amber-600 bg-amber-50",
icon: (
<AlertTriangle className="h-4 w-4" aria-hidden="true" />
),
}
case "failed":
return {
label: dict.validation.issuesFound,
color: "text-yellow-600 bg-yellow-50",
icon: (
<AlertTriangle className="h-4 w-4" aria-hidden="true" />
),
}
case "error":
return {
label: dict.validation.error,
color: "text-red-600 bg-red-50",
icon: <X className="h-4 w-4" aria-hidden="true" />,
}
case "skipped":
return {
label: dict.validation.skipped,
color: "text-gray-600 bg-gray-50",
icon: <Check className="h-4 w-4" aria-hidden="true" />,
}
default:
return null
}
}
const statusDisplay = getStatusDisplay()
if (!statusDisplay || state.status === "idle") return null
return (
<div className="my-3 rounded-xl border border-border/60 bg-muted/30 overflow-hidden">
<div className="flex items-center justify-between px-4 py-3 bg-muted/50">
<div className="flex items-center gap-2">
<div className="w-6 h-6 rounded-md bg-primary/10 flex items-center justify-center">
<Eye
className="w-3.5 h-3.5 text-primary"
aria-hidden="true"
/>
</div>
<span className="text-sm font-medium text-foreground/80">
{dict.validation.title}
</span>
</div>
<div className="flex items-center gap-2">
<span
className={`text-xs font-medium px-2 py-0.5 rounded-full flex items-center gap-1 ${statusDisplay.color}`}
>
{statusDisplay.icon}
<span className="ml-1">{statusDisplay.label}</span>
</span>
{(state.result || state.error) && (
<button
type="button"
onClick={() => setIsExpanded(!isExpanded)}
className="p-1 rounded hover:bg-muted transition-colors"
>
{isExpanded ? (
<ChevronUp
className="w-4 h-4 text-muted-foreground"
aria-hidden="true"
/>
) : (
<ChevronDown
className="w-4 h-4 text-muted-foreground"
aria-hidden="true"
/>
)}
</button>
)}
</div>
</div>
{/* Validation details when expanded */}
{isExpanded && (state.result || state.imageData) && (
<div className="px-4 py-3 border-t border-border/40 bg-muted/20 space-y-3">
{/* Captured image */}
{state.imageData && (
<div>
<div className="text-xs font-medium text-foreground/70 mb-2 flex items-center gap-1">
<ImageIcon
className="h-3 w-3"
aria-hidden="true"
/>
{dict.validation.capturedScreenshot}
</div>
<div className="rounded-lg border border-border/50 overflow-hidden bg-white">
<Image
src={state.imageData}
alt="Captured diagram for validation"
width={400}
height={300}
className="w-full h-auto max-h-48 object-contain"
unoptimized
/>
</div>
</div>
)}
{/* Issues */}
{state.result && state.result.issues.length > 0 && (
<div>
<div className="text-xs font-medium text-foreground/70 mb-2">
{dict.validation.issuesFoundLabel}
</div>
<div className="space-y-2">
{state.result.issues.map((issue, index) => (
<div
key={index}
className={`text-xs px-3 py-2 rounded-lg border ${
issue.severity === "critical"
? "bg-red-50 border-red-200 text-red-700 dark:bg-red-950 dark:border-red-800 dark:text-red-300"
: "bg-yellow-50 border-yellow-200 text-yellow-700 dark:bg-yellow-950 dark:border-yellow-800 dark:text-yellow-300"
}`}
>
<span className="font-medium uppercase text-[10px] mr-2">
[{issue.type}]
</span>
{issue.description}
</div>
))}
</div>
</div>
)}
{/* Suggestions */}
{state.result && state.result.suggestions.length > 0 && (
<div>
<div className="text-xs font-medium text-foreground/70 mb-2">
{dict.validation.suggestions}
</div>
<ul className="text-xs text-foreground/60 space-y-1 list-disc list-inside">
{state.result.suggestions.map(
(suggestion, index) => (
<li key={index}>{suggestion}</li>
),
)}
</ul>
</div>
)}
{/* Valid result message */}
{state.result?.valid &&
state.result.issues.length === 0 && (
<div className="text-xs text-green-600 dark:text-green-400">
{dict.validation.passedValidation}
</div>
)}
</div>
)}
{/* Improve with Suggestions button - shown when validation passed but has suggestions */}
{showImproveButton && (
<div className="px-4 py-3 border-t border-border/40 bg-muted/10">
{hasRequestedImprovement ? (
<div className="flex items-center justify-center gap-2 px-4 py-2 text-sm font-medium text-green-600 dark:text-green-400">
<Check className="h-4 w-4" aria-hidden="true" />
{dict.validation.improvementRequested}
</div>
) : (
<>
<button
type="button"
onClick={handleImproveClick}
className="w-full flex items-center justify-center gap-2 px-4 py-2 text-sm font-medium text-primary bg-primary/10 hover:bg-primary/20 rounded-lg transition-colors"
>
<RefreshCw
className="h-4 w-4"
aria-hidden="true"
/>
{dict.validation.improveWithSuggestions}
</button>
<p className="text-xs text-muted-foreground mt-2 text-center">
{dict.validation.regenerateWithFeedback}
</p>
</>
)}
</div>
)}
{/* Error details when expanded */}
{isExpanded && state.error && (
<div className="px-4 py-3 border-t border-border/40 bg-red-50/50">
<div className="text-xs text-red-600">{state.error}</div>
</div>
)}
</div>
)
}

View File

@@ -67,6 +67,8 @@ interface SettingsDialogProps {
onToggleDarkMode: () => void
minimalStyle?: boolean
onMinimalStyleChange?: (value: boolean) => void
vlmValidationEnabled?: boolean
onVlmValidationChange?: (value: boolean) => void
}
export const STORAGE_ACCESS_CODE_KEY = "next-ai-draw-io-access-code"
@@ -88,6 +90,8 @@ function SettingsContent({
onToggleDarkMode,
minimalStyle = false,
onMinimalStyleChange = () => {},
vlmValidationEnabled = false,
onVlmValidationChange = () => {},
}: SettingsDialogProps) {
const dict = useDictionary()
const router = useRouter()
@@ -410,6 +414,25 @@ function SettingsContent({
</div>
</SettingItem>
{/* VLM Diagram Validation */}
<SettingItem
label={dict.settings.diagramValidation}
description={dict.settings.diagramValidationDescription}
>
<div className="flex items-center gap-2">
<Switch
id="vlm-validation"
checked={vlmValidationEnabled}
onCheckedChange={onVlmValidationChange}
/>
<span className="text-sm text-muted-foreground">
{vlmValidationEnabled
? dict.settings.enabled
: dict.settings.disabled}
</span>
</div>
</SettingItem>
{/* Send Shortcut */}
<SettingItem
label={dict.settings.sendShortcut}

View File

@@ -31,6 +31,7 @@ interface DiagramContextType {
successMessage?: string,
) => void
getThumbnailSvg: () => Promise<string | null>
captureValidationPng: () => Promise<string | null>
isDrawioReady: boolean
onDrawioLoad: () => void
resetDrawioReady: () => void
@@ -51,6 +52,8 @@ export function DiagramProvider({ children }: { children: React.ReactNode }) {
const hasCalledOnLoadRef = useRef(false)
const drawioRef = useRef<DrawIoEmbedRef | null>(null)
const resolverRef = useRef<((value: string) => void) | null>(null)
// Resolver for PNG export (used for VLM validation)
const pngResolverRef = useRef<((value: string) => void) | null>(null)
// Track if we're expecting an export for history (user-initiated)
const expectHistoryExportRef = useRef<boolean>(false)
// Track if diagram has been restored after DrawIO remount (e.g., theme change)
@@ -147,6 +150,37 @@ export function DiagramProvider({ children }: { children: React.ReactNode }) {
}
}
// Capture current diagram as PNG for VLM validation
const captureValidationPng = async (): Promise<string | null> => {
if (!drawioRef.current) return null
// Don't export if diagram is empty
if (!isRealDiagram(chartXML)) return null
try {
const pngData = await Promise.race([
new Promise<string>((resolve) => {
pngResolverRef.current = resolve
drawioRef.current?.exportDiagram({ format: "png" })
}),
new Promise<string>((_, reject) =>
setTimeout(
() => reject(new Error("PNG export timeout")),
5000,
),
),
])
// PNG data should be a base64 data URL
if (pngData?.startsWith("data:image/png")) {
return pngData
}
return null
} catch {
// Timeout is expected occasionally - don't log as error
return null
}
}
const loadDiagram = (
chart: string,
skipValidation?: boolean,
@@ -186,6 +220,13 @@ export function DiagramProvider({ children }: { children: React.ReactNode }) {
}
const handleDiagramExport = (data: any) => {
// Handle PNG export for VLM validation
if (pngResolverRef.current && data.data?.startsWith("data:image/png")) {
pngResolverRef.current(data.data)
pngResolverRef.current = null
return
}
// Handle save to file if requested (process raw data before extraction)
if (saveResolverRef.current.resolver) {
const format = saveResolverRef.current.format
@@ -353,6 +394,7 @@ export function DiagramProvider({ children }: { children: React.ReactNode }) {
clearDiagram,
saveDiagramToFile,
getThumbnailSvg,
captureValidationPng,
isDrawioReady,
onDrawioLoad,
resetDrawioReady,

View File

@@ -1,5 +1,12 @@
import type { MutableRefObject } from "react"
import { useRef } from "react"
import type { DiagramOperation } from "@/components/chat/types"
import type {
ValidationState,
ValidationStatus,
} from "@/components/chat/ValidationCard"
import type { ValidationResult } from "@/lib/diagram-validator"
import { formatValidationFeedback } from "@/lib/diagram-validator"
import { isMxCellXmlComplete, wrapWithMxFile } from "@/lib/utils"
const DEBUG = process.env.NODE_ENV === "development"
@@ -30,6 +37,14 @@ type AddToolOutputParams = AddToolOutputSuccess | AddToolOutputError
type AddToolOutputFn = (params: AddToolOutputParams) => void
const MAX_VALIDATION_RETRIES = 3
// Type for the validation function passed from useValidateDiagram hook
type ValidateDiagramFn = (
imageData: string,
sessionId?: string,
) => Promise<ValidationResult>
interface UseDiagramToolHandlersParams {
partialXmlRef: MutableRefObject<string>
editDiagramOriginalXmlRef: MutableRefObject<Map<string, string>>
@@ -37,6 +52,14 @@ interface UseDiagramToolHandlersParams {
onDisplayChart: (xml: string, skipValidation?: boolean) => string | null
onFetchChart: (saveToHistory?: boolean) => Promise<string>
onExport: () => void
captureValidationPng?: () => Promise<string | null>
validateDiagram?: ValidateDiagramFn
enableVlmValidation?: boolean
sessionId?: string
onValidationStateChange?: (
toolCallId: string,
state: ValidationState,
) => void
}
/**
@@ -53,7 +76,34 @@ export function useDiagramToolHandlers({
onDisplayChart,
onFetchChart,
onExport,
captureValidationPng,
validateDiagram,
enableVlmValidation = true,
sessionId,
onValidationStateChange,
}: UseDiagramToolHandlersParams) {
// Track validation retry count per tool call
const validationRetryCountRef = useRef<Map<string, number>>(new Map())
// Helper to update validation state
const updateValidationState = (
toolCallId: string,
status: ValidationStatus,
options?: {
attempt?: number
maxAttempts?: number
result?: ValidationResult
error?: string
imageData?: string
},
) => {
if (onValidationStateChange) {
onValidationStateChange(toolCallId, {
status,
...options,
})
}
}
const handleToolCall = async (
{ toolCall }: { toolCall: ToolCall },
addToolOutput: AddToolOutputFn,
@@ -155,7 +205,159 @@ ${finalXml}
// Success - diagram will be rendered by chat-message-display
if (DEBUG) {
console.log(
"[display_diagram] Success! Adding tool output with state: output-available",
"[display_diagram] Success! Checking if VLM validation is enabled...",
)
}
// VLM validation after successful display
if (
enableVlmValidation &&
captureValidationPng &&
validateDiagram
) {
let capturedPngData: string | null = null
try {
// Notify UI that we're starting capture
updateValidationState(toolCall.toolCallId, "capturing")
// Small delay (100ms) to allow diagram rendering to complete before capture.
// This is a best-effort heuristic and may need adjustment for complex diagrams or slower devices.
await new Promise((resolve) => setTimeout(resolve, 100))
capturedPngData = await captureValidationPng()
if (capturedPngData) {
if (DEBUG) {
console.log(
"[display_diagram] Captured PNG for validation",
)
}
const retryCount =
validationRetryCountRef.current.get(
toolCall.toolCallId,
) || 0
// Notify UI that we're validating (include the image)
updateValidationState(
toolCall.toolCallId,
"validating",
{
attempt: retryCount + 1,
maxAttempts: MAX_VALIDATION_RETRIES,
imageData: capturedPngData,
},
)
const result = await validateDiagram(
capturedPngData,
sessionId,
)
if (!result.valid) {
if (retryCount < MAX_VALIDATION_RETRIES) {
validationRetryCountRef.current.set(
toolCall.toolCallId,
retryCount + 1,
)
const feedback =
formatValidationFeedback(result)
if (DEBUG) {
console.log(
`[display_diagram] Validation failed (attempt ${retryCount + 1}/${MAX_VALIDATION_RETRIES}):`,
result.issues,
)
}
// Notify UI of validation failure (include the image)
updateValidationState(
toolCall.toolCallId,
"failed",
{
attempt: retryCount + 1,
maxAttempts: MAX_VALIDATION_RETRIES,
result,
imageData: capturedPngData,
},
)
addToolOutput({
tool: "display_diagram",
toolCallId: toolCall.toolCallId,
state: "output-error",
errorText: `[Validation attempt ${retryCount + 1}/${MAX_VALIDATION_RETRIES}]\n${feedback}`,
})
return
} else {
// Max retries reached - accept the diagram with warning
if (DEBUG) {
console.log(
"[display_diagram] Max validation retries reached, accepting diagram",
)
}
validationRetryCountRef.current.delete(
toolCall.toolCallId,
)
// Notify UI that we're accepting with issues (include the image)
updateValidationState(
toolCall.toolCallId,
"skipped",
{ result, imageData: capturedPngData },
)
addToolOutput({
tool: "display_diagram",
toolCallId: toolCall.toolCallId,
output: "Diagram displayed (validation issues noted but max retries reached).",
})
return
}
} else {
// Validation passed - clean up retry count
validationRetryCountRef.current.delete(
toolCall.toolCallId,
)
if (DEBUG) {
console.log(
"[display_diagram] Validation passed!",
)
}
// Notify UI of success (include the image)
// Use "success_with_warnings" if valid but has issues
const hasWarnings = result.issues.length > 0
updateValidationState(
toolCall.toolCallId,
hasWarnings
? "success_with_warnings"
: "success",
{ result, imageData: capturedPngData },
)
}
} else {
// PNG capture failed - skip validation
updateValidationState(toolCall.toolCallId, "skipped")
}
} catch (error) {
// VLM validation error - log but don't block the user
console.warn(
"[display_diagram] VLM validation error:",
error,
)
updateValidationState(toolCall.toolCallId, "error", {
error:
error instanceof Error
? error.message
: "Validation failed",
imageData: capturedPngData || undefined,
})
}
}
if (DEBUG) {
console.log(
"[display_diagram] Adding tool output with state: output-available",
)
}
addToolOutput({

View File

@@ -0,0 +1,136 @@
"use client"
/**
* Hook for VLM-based diagram validation using AI SDK's useObject.
*/
import { experimental_useObject as useObject } from "@ai-sdk/react"
import { useCallback, useRef } from "react"
import { getApiEndpoint } from "@/lib/base-path"
import {
type ValidationResult,
ValidationResultSchema,
} from "@/lib/validation-schema"
export type { ValidationResult }
// Default valid result for fallback cases
const DEFAULT_VALID_RESULT: ValidationResult = {
valid: true,
issues: [],
suggestions: [],
}
interface UseValidateDiagramOptions {
onSuccess?: (result: ValidationResult) => void
onError?: (error: Error) => void
}
// Track pending validation promises for imperative API
type PendingValidation = {
resolve: (result: ValidationResult) => void
reject: (error: Error) => void
}
export function useValidateDiagram(options: UseValidateDiagramOptions = {}) {
const { onSuccess, onError } = options
const pendingValidationRef = useRef<PendingValidation | null>(null)
const { object, submit, isLoading, error, stop } = useObject({
api: getApiEndpoint("/api/validate-diagram"),
schema: ValidationResultSchema,
onFinish: ({
object,
error: finishError,
}: {
object: ValidationResult | undefined
error: Error | undefined
}) => {
if (finishError) {
console.error(
"[useValidateDiagram] Validation error:",
finishError,
)
onError?.(finishError)
pendingValidationRef.current?.reject(finishError)
pendingValidationRef.current = null
return
}
if (object) {
const result = object as ValidationResult
onSuccess?.(result)
pendingValidationRef.current?.resolve(result)
pendingValidationRef.current = null
}
},
onError: (err: Error) => {
console.error("[useValidateDiagram] Stream error:", err)
onError?.(err)
pendingValidationRef.current?.reject(err)
pendingValidationRef.current = null
},
})
/**
* Validate a diagram image.
* Returns a promise that resolves with the validation result.
*/
const validate = useCallback(
async (
imageData: string,
sessionId?: string,
): Promise<ValidationResult> => {
// Reject any pending validation to prevent promise leaks
if (pendingValidationRef.current) {
pendingValidationRef.current.reject(
new Error("Validation superseded by new request"),
)
pendingValidationRef.current = null
}
return new Promise((resolve, reject) => {
// Store the promise handlers
pendingValidationRef.current = { resolve, reject }
// Submit the validation request
submit({ imageData, sessionId })
})
},
[submit],
)
/**
* Validate with fallback - returns default valid result on error.
* Use this to avoid blocking the user on validation failures.
*/
const validateWithFallback = useCallback(
async (
imageData: string,
sessionId?: string,
): Promise<ValidationResult> => {
try {
return await validate(imageData, sessionId)
} catch (error) {
console.warn(
"[useValidateDiagram] Validation failed, using fallback:",
error,
)
return DEFAULT_VALID_RESULT
}
},
[validate],
)
return {
// Validation functions
validate,
validateWithFallback,
stop,
// State
isValidating: isLoading,
partialResult: object as ValidationResult | undefined,
error,
}
}

View File

@@ -1177,3 +1177,27 @@ export function supportsImageInput(modelId: string): boolean {
// Default: assume model supports images
return true
}
/**
* Get the AI model for diagram validation.
* Uses VALIDATION_MODEL env var if set, otherwise falls back to AI_MODEL.
* Throws if the model doesn't support image input.
*/
export function getValidationModel(): ReturnType<typeof getAIModel>["model"] {
const modelId = process.env.VALIDATION_MODEL || process.env.AI_MODEL
if (!modelId) {
throw new Error(
"No validation model configured. Set VALIDATION_MODEL or AI_MODEL.",
)
}
if (!supportsImageInput(modelId)) {
throw new Error(
`Validation requires a vision-capable model. Model "${modelId}" does not support image input.`,
)
}
const { model } = getAIModel({ modelId })
return model
}

64
lib/diagram-validator.ts Normal file
View File

@@ -0,0 +1,64 @@
/**
* Types and utilities for VLM-based diagram validation.
* The actual validation is performed via useValidateDiagram hook using AI SDK's useObject.
*/
// Re-export types from the schema file (single source of truth)
export type { ValidationIssue, ValidationResult } from "./validation-schema"
import type { ValidationResult } from "./validation-schema"
/**
* Format validation feedback for display to the AI model.
* This creates a human-readable error message that guides the AI to fix issues.
*
* @param result - The validation result from VLM
* @returns Formatted string for tool error output
*/
export function formatValidationFeedback(result: ValidationResult): string {
// If validation passed with no issues, return empty string
if (result.valid && result.issues.length === 0) {
return ""
}
const lines: string[] = []
lines.push("DIAGRAM VISUAL VALIDATION FAILED")
lines.push("")
// Group issues by severity
const criticalIssues = result.issues.filter(
(i) => i.severity === "critical",
)
const warnings = result.issues.filter((i) => i.severity === "warning")
if (criticalIssues.length > 0) {
lines.push("Critical Issues (must fix):")
for (const issue of criticalIssues) {
lines.push(` - [${issue.type}] ${issue.description}`)
}
lines.push("")
}
if (warnings.length > 0) {
lines.push("Warnings:")
for (const issue of warnings) {
lines.push(` - [${issue.type}] ${issue.description}`)
}
lines.push("")
}
if (result.suggestions.length > 0) {
lines.push("Suggestions to fix:")
for (const suggestion of result.suggestions) {
lines.push(` - ${suggestion}`)
}
lines.push("")
}
lines.push(
"Please regenerate the diagram with corrected layout to fix these visual issues.",
)
return lines.join("\n")
}

View File

@@ -115,7 +115,11 @@
"httpProxy": "HTTP Proxy",
"httpsProxy": "HTTPS Proxy",
"applyProxy": "Apply",
"proxyApplied": "Proxy settings applied"
"proxyApplied": "Proxy settings applied",
"diagramValidation": "Diagram Validation (Experimental)",
"diagramValidationDescription": "Use a vision language model to validate generated diagrams. Requires a VLM like GPT-5.2 or Sonnet-4.5.",
"enabled": "Enabled",
"disabled": "Disabled"
},
"save": {
"title": "Save Diagram",
@@ -248,6 +252,24 @@
"searchPlaceholder": "Search chats...",
"noResults": "No chats found"
},
"validation": {
"title": "Validate Diagram",
"capturing": "Capturing",
"validating": "Validating",
"validatingWithAttempt": "Validating ({attempt}/{max})",
"valid": "Valid",
"validWithWarnings": "Valid with Warnings",
"issuesFound": "Issues Found",
"error": "Error",
"skipped": "Skipped",
"capturedScreenshot": "Captured Screenshot:",
"issuesFoundLabel": "Issues Found:",
"suggestions": "Suggestions:",
"passedValidation": "Diagram passed visual validation - no issues detected.",
"improvementRequested": "Improvement requested - check the new diagram below",
"improveWithSuggestions": "Improve with Suggestions",
"regenerateWithFeedback": "Regenerate the diagram using the validation feedback"
},
"modelConfig": {
"title": "AI Model Configuration",
"description": "Configure multiple AI providers and models",

View File

@@ -115,7 +115,11 @@
"httpProxy": "HTTP プロキシ",
"httpsProxy": "HTTPS プロキシ",
"applyProxy": "適用",
"proxyApplied": "プロキシ設定が適用されました"
"proxyApplied": "プロキシ設定が適用されました",
"diagramValidation": "ダイアグラム検証(実験的)",
"diagramValidationDescription": "視覚言語モデルを使用して生成されたダイアグラムを検証します。GPT-5.2 や Sonnet-4.5 などの VLM が必要です。",
"enabled": "有効",
"disabled": "無効"
},
"save": {
"title": "ダイアグラムを保存",
@@ -248,6 +252,24 @@
"searchPlaceholder": "チャットを検索...",
"noResults": "チャットが見つかりません"
},
"validation": {
"title": "ダイアグラムを検証",
"capturing": "キャプチャ中",
"validating": "検証中",
"validatingWithAttempt": "検証中 ({attempt}/{max})",
"valid": "有効",
"validWithWarnings": "有効(警告あり)",
"issuesFound": "問題が見つかりました",
"error": "エラー",
"skipped": "スキップ",
"capturedScreenshot": "キャプチャした画像:",
"issuesFoundLabel": "検出された問題:",
"suggestions": "提案:",
"passedValidation": "ダイアグラムは視覚検証に合格しました - 問題は検出されませんでした。",
"improvementRequested": "改善リクエスト済み - 下の新しいダイアグラムを確認してください",
"improveWithSuggestions": "提案で改善",
"regenerateWithFeedback": "検証フィードバックを使用してダイアグラムを再生成"
},
"modelConfig": {
"title": "AIモデル設定",
"description": "複数のAIプロバイダーとモデルを設定",

View File

@@ -115,7 +115,11 @@
"httpProxy": "HTTP 代理",
"httpsProxy": "HTTPS 代理",
"applyProxy": "应用",
"proxyApplied": "代理设置已应用"
"proxyApplied": "代理设置已应用",
"diagramValidation": "图表验证(实验性)",
"diagramValidationDescription": "使用视觉语言模型验证生成的图表。需要支持视觉的模型,如 GPT-5.2 或 Sonnet-4.5。",
"enabled": "已启用",
"disabled": "已禁用"
},
"save": {
"title": "保存图表",
@@ -248,6 +252,24 @@
"searchPlaceholder": "搜索对话...",
"noResults": "未找到对话"
},
"validation": {
"title": "验证图表",
"capturing": "截图中",
"validating": "验证中",
"validatingWithAttempt": "验证中 ({attempt}/{max})",
"valid": "通过",
"validWithWarnings": "通过(有警告)",
"issuesFound": "发现问题",
"error": "错误",
"skipped": "已跳过",
"capturedScreenshot": "截图预览:",
"issuesFoundLabel": "发现的问题:",
"suggestions": "建议:",
"passedValidation": "图表通过视觉验证 - 未发现问题。",
"improvementRequested": "改进请求已发送 - 请查看下方新图表",
"improveWithSuggestions": "根据建议改进",
"regenerateWithFeedback": "使用验证反馈重新生成图表"
},
"modelConfig": {
"title": "AI 模型配置",
"description": "配置多个 AI 提供商和模型",

View File

@@ -24,4 +24,7 @@ export const STORAGE_KEYS = {
// Chat input preferences
sendShortcut: "next-ai-draw-io-send-shortcut",
// Diagram validation
vlmValidationEnabled: "next-ai-draw-io-vlm-validation-enabled",
} as const

22
lib/validation-prompts.ts Normal file
View File

@@ -0,0 +1,22 @@
/**
* VLM system prompt for diagram validation.
* Note: Response parsing is now handled via AI SDK's structured outputs (generateObject with schema).
*/
export const VALIDATION_SYSTEM_PROMPT = `You are a diagram quality validator. Analyze the rendered diagram image for visual issues.
Evaluate the diagram for the following issues:
1. **Overlapping elements** (critical): Shapes covering each other inappropriately, making content unreadable
2. **Edge routing issues** (critical): Lines/arrows crossing through shapes that are not their source or target
3. **Text readability** (warning): Labels cut off, overlapping, or too small to read
4. **Layout quality** (warning): Poor spacing, misalignment, or cramped elements
5. **Rendering errors** (critical): Incomplete, corrupted, or missing visual elements
Rules:
- Set "valid" to true ONLY if there are no critical issues
- Be specific about which elements have problems (e.g., "The 'Login' box overlaps with 'Register' box")
- Provide actionable suggestions (e.g., "Move the Login box 50 pixels to the left")
- Minor cosmetic issues (slight misalignment, non-uniform spacing) should be warnings, not critical
- Empty diagrams or diagrams with only 1-2 elements should pass unless they have obvious errors
- If the diagram looks generally acceptable, set valid to true even with minor warnings`

38
lib/validation-schema.ts Normal file
View File

@@ -0,0 +1,38 @@
/**
* Shared validation schema for VLM-based diagram validation.
* This file can be safely imported on both client and server.
*/
import { z } from "zod"
// Schema for structured validation output
export const ValidationResultSchema = z.object({
valid: z.boolean().describe("True if there are no critical issues"),
issues: z
.array(
z.object({
type: z
.enum([
"overlap",
"edge_routing",
"text",
"layout",
"rendering",
])
.describe("Type of visual issue"),
severity: z
.enum(["critical", "warning"])
.describe("Severity level"),
description: z
.string()
.describe("Clear description of the issue"),
}),
)
.describe("List of visual issues found"),
suggestions: z
.array(z.string())
.describe("Actionable suggestions to fix issues"),
})
export type ValidationResult = z.infer<typeof ValidationResultSchema>
export type ValidationIssue = ValidationResult["issues"][number]

View File

@@ -0,0 +1,116 @@
import { describe, expect, it } from "vitest"
import {
formatValidationFeedback,
type ValidationResult,
} from "@/lib/diagram-validator"
describe("formatValidationFeedback", () => {
it("formats result with critical issues", () => {
const result: ValidationResult = {
valid: false,
issues: [
{
type: "overlap",
severity: "critical",
description: "Box A overlaps with Box B",
},
],
suggestions: ["Move Box A to the left"],
}
const feedback = formatValidationFeedback(result)
expect(feedback).toContain("DIAGRAM VISUAL VALIDATION FAILED")
expect(feedback).toContain("Critical Issues (must fix):")
expect(feedback).toContain("[overlap] Box A overlaps with Box B")
expect(feedback).toContain("Suggestions to fix:")
expect(feedback).toContain("Move Box A to the left")
expect(feedback).toContain(
"Please regenerate the diagram with corrected layout",
)
})
it("formats result with warnings only", () => {
const result: ValidationResult = {
valid: true,
issues: [
{
type: "text",
severity: "warning",
description: "Label text is small",
},
],
suggestions: [],
}
const feedback = formatValidationFeedback(result)
expect(feedback).toContain("Warnings:")
expect(feedback).toContain("[text] Label text is small")
expect(feedback).not.toContain("Critical Issues")
})
it("formats result with both critical issues and warnings", () => {
const result: ValidationResult = {
valid: false,
issues: [
{
type: "edge_routing",
severity: "critical",
description: "Edge crosses through node",
},
{
type: "layout",
severity: "warning",
description: "Uneven spacing",
},
],
suggestions: ["Reroute the edge", "Adjust spacing"],
}
const feedback = formatValidationFeedback(result)
expect(feedback).toContain("Critical Issues (must fix):")
expect(feedback).toContain("[edge_routing] Edge crosses through node")
expect(feedback).toContain("Warnings:")
expect(feedback).toContain("[layout] Uneven spacing")
expect(feedback).toContain("Reroute the edge")
expect(feedback).toContain("Adjust spacing")
})
it("returns empty string for valid result with no issues", () => {
const result: ValidationResult = {
valid: true,
issues: [],
suggestions: [],
}
const feedback = formatValidationFeedback(result)
expect(feedback).toBe("")
})
it("formats result with multiple suggestions", () => {
const result: ValidationResult = {
valid: false,
issues: [
{
type: "rendering",
severity: "critical",
description: "Missing element",
},
],
suggestions: [
"Check the XML syntax",
"Ensure all elements are defined",
"Verify parent-child relationships",
],
}
const feedback = formatValidationFeedback(result)
expect(feedback).toContain("Check the XML syntax")
expect(feedback).toContain("Ensure all elements are defined")
expect(feedback).toContain("Verify parent-child relationships")
})
})

View File

@@ -30,5 +30,11 @@
".next/types/**/*.ts",
".next/dev/types/**/*.ts"
],
"exclude": ["node_modules", "packages", "electron", "dist-electron"]
"exclude": [
"node_modules",
"packages",
"electron",
"electron-standalone",
"dist-electron"
]
}