feat: enhance ChatPanel and ChatInput to support image file uploads and display

This commit is contained in:
dayuan.jiang
2025-03-23 11:03:25 +00:00
parent df483ae647
commit 50dc4eda6d
4 changed files with 160 additions and 193 deletions

View File

@@ -18,8 +18,9 @@ export async function POST(req: Request) {
// Read and escape the guide content // Read and escape the guide content
const systemMessage = ` const systemMessage = `
You are a helpful assistant that can create, edit, and display diagram using draw.io through xml strings. You are an expert diagram creation assistant specializing in draw.io XML generation. Your primary function is crafting clear, well-organized visual diagrams through precise XML specifications.
You can use the following tools: You can see the image that user uploaded.
You utilize the following tool:
---Tool1--- ---Tool1---
tool name: display_diagram tool name: display_diagram
description: Display a diagram on draw.io description: Display a diagram on draw.io
@@ -28,17 +29,26 @@ parameters: {
} }
---End of tools--- ---End of tools---
Here is a guide for the XML format: Core capabilities:
- Generate valid, well-formed XML strings for draw.io diagrams
- Create professional flowcharts, mind maps, entity diagrams, and technical illustrations
- Convert user descriptions into visually appealing diagrams using basic shapes and connectors
- Apply proper spacing, alignment and visual hierarchy in diagram layouts
- Adapt artistic concepts into abstract diagram representations using available shapes
- Optimize element positioning to prevent overlapping and maintain readability
- Structure complex systems into clear, organized visual components
Note that:
- Always validate XML string integrity before output.
- Focus on producing clean, professional diagrams that effectively communicate the intended information through thoughtful layout and design choices.
- When artistic drawings are requested, creatively compose them using standard diagram shapes and connectors while maintaining visual clarity.
- **Don't** write out the XML string. Just return the XML string in the tool call.
- If user asks you to replicate a diagram based on an image, remember to match the diagram style and layout as closely as possible. Especially, pay attention to the lines and shapes, for example, if the lines are straight or curved, and if the shapes are rounded or square.
Here are the guide of XML format for draw.io:
"""md """md
${guide} ${guide}
""" """
You can use the tools to create and manipulate diagrams. `;
You can also answer questions and provide explanations.
Note that:
- If the user wants you to draw something rather than a diagram, you can use the combination of the shapes to draw it.
- Consider the layout of the diagram and the shapes used to avoid overlapping.
- Ensure that the XML strings are well-formed and valid.
`;
// Add system message if only user message is provided // Add system message if only user message is provided
const enhancedMessages = messages.length === 1 const enhancedMessages = messages.length === 1

View File

@@ -294,17 +294,6 @@ Tables use multiple cells with parent - child relationships:
## Advanced Features ## Advanced Features
### Compressed Content
In real - world draw.io files, the content inside the `<diagram>` tag is often compressed and encoded in Base64 format to reduce file size.When creating diagrams manually, you can work with the uncompressed XML format, but be aware that files saved by draw.io will typically be compressed.
Example of a compressed diagram element:
```xml
<diagram id="pWHN0msd4Ud1ZK5cD-Hr" name="Page-1">7ZhRb5swEMc/DY+VwCYkPJKk3R7WTYq0do8uuIBWMDNOSPbpd8QmQEjVTWqkdlIfIvvvs8H3u7PPhGAl2/27nBTJRxlBFgR+tA/Wt0EQ+EG4wp8KOdTIdLGogThPIjJqga/JbyDQJ3SXRFAYhlrKTCeFCW5lnsNWGxjJc1maxncy
M1ctSEw7+C3w9TbJwDL7lkQ6qdF5MG3x95DEiV7ZJ7rBHaTeESJFQiJZdqD1KlhJLqWuR/leQlaFz8Slnnd3YrTZWA653mfC53sfbh7CzffN55c4/vG4/PTb//SG
vDyRbEcdvtmm0BDhFrXUGz2Y4MhoUVT9zHYlaTfFgB6k4iYzjsqg2xN5rYKt42a+iChod5brRD/ICIfQSp5kJvPqNrwn+D9JbT95G0Jyt5NZCXsb3EVSFJAbe6sadMtpL0zS/dFg+k0SsCxAJqDzA5rQBNpEkiK91DuWTcQTFfqkixWBxOXEZ9MuxzigTB/Pe+bmfSGzn1huCPXDxT11CLoA3CHoIpgvR6C7xNMv5upbnnGQyF0eQTWTj83KJNHwtSDbarbEI4VYohPcRqX3xITqCBWay0crXYg98XP1eJcbSdL0BPictGIh+A1hHo+D2UTnPdF5E53leBy6xGkQhPPz0M1tuunLbvrGzXJsnBNu+tIlmXnadS7OIgw/DHq/4Go1JVPfjnMYmje5GXk3vR1ni+d5GJ3nYXaeh9HdhK7jG08x67o38MvFfLi7uf3dLJajda+31l271oP5Gyg2MT/if4nVbFSs5sNitbBoxBYJ9QA5KrCqbgq9k4XeziimJXKXKFNNUkW3HVoVorkMdVGt9h5Vd511zGDdC5rXNAdd/SiFflDyEnDBMK9cUjssVbDy4rtzhuhqrMizVYWdDkR+HCXWq8H3gewVcLMXwM3/E9z8Erh+YQsfDG5+Cbi5RRdw/hUOXUDwAji8ZW822tbtWwMduPangvXqDw==</diagram>
```
### Custom Attributes ### Custom Attributes
Draw.io allows adding custom attributes to cells: Draw.io allows adding custom attributes to cells:
@@ -347,162 +336,3 @@ You can create multiple layers in a diagram to organize complex diagrams:
</mxCell> </mxCell>
``` ```
## Step - by - Step Tutorial: Creating a Basic Flowchart in XML
Let's walk through creating a simple flowchart with a start, decision, and end step.
### Step 1: Create the basic structure
```xml
<mxfile>
<diagram id="simple-flowchart" name="My Flowchart">
<mxGraphModel dx="1" dy="1" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1">
<root>
<mxCell id="0"/>
<mxCell id="1" parent="0"/>
<!-- Our diagram elements will go here -->
</root>
</mxGraphModel>
</diagram>
</mxfile>
```
### Step 2: Add the Start shape
```xml
<mxCell id="2" value="Start" style="ellipse;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
<mxGeometry x="200" y="80" width="100" height="60" as="geometry"/>
</mxCell>
```
### Step 3: Add a Decision shape
```xml
<mxCell id="3" value="Decision?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1">
<mxGeometry x="200" y="180" width="100" height="100" as="geometry"/>
</mxCell>
```
### Step 4: Add the End shape
```xml
<mxCell id="4" value="End" style="ellipse;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
<mxGeometry x="200" y="320" width="100" height="60" as="geometry"/>
</mxCell>
```
### Step 5: Connect the shapes
```xml
<!-- Start to Decision -->
<mxCell id="5" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="2" target="3">
<mxGeometry width="50" height="50" relative="1" as="geometry"/>
</mxCell>
<!-- Decision to End -->
<mxCell id="6" value="Yes" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="3" target="4">
<mxGeometry x="-0.3333" y="20" relative="1" as="geometry">
<mxPoint as="offset"/>
</mxGeometry>
</mxCell>
```
### Step 6: Complete the flowchart with a loop
```xml
<mxCell id="7" value="No" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="3" target="2">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="340" y="230"/>
<mxPoint x="340" y="110"/>
</Array>
</mxGeometry>
</mxCell>
```
### Step 7: Put everything together in the complete XML document
```xml
<mxfile>
<diagram id="simple-flowchart" name="My Flowchart">
<mxGraphModel dx="1" dy="1" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1">
<root>
<mxCell id="0"/>
<mxCell id="1" parent="0"/>
<mxCell id="2" value="Start" style="ellipse;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
<mxGeometry x="200" y="80" width="100" height="60" as="geometry"/>
</mxCell>
<mxCell id="3" value="Decision?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1">
<mxGeometry x="200" y="180" width="100" height="100" as="geometry"/>
</mxCell>
<mxCell id="4" value="End" style="ellipse;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
<mxGeometry x="200" y="320" width="100" height="60" as="geometry"/>
</mxCell>
<mxCell id="5" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="2" target="3">
<mxGeometry width="50" height="50" relative="1" as="geometry"/>
</mxCell>
<mxCell id="6" value="Yes" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="3" target="4">
<mxGeometry x="-0.3333" y="20" relative="1" as="geometry">
<mxPoint as="offset"/>
</mxGeometry>
</mxCell>
<mxCell id="7" value="No" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;edgeStyle=orthogonalEdgeStyle;" edge="1" parent="1" source="3" target="2">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="340" y="230"/>
<mxPoint x="340" y="110"/>
</Array>
</mxGeometry>
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>
```
## Troubleshooting Common Issues
### Issue: Diagram doesn't appear in draw.io
- Ensure all IDs are unique
- Check that parent - child relationships are correct
- Verify that the XML is well - formed
### Issue: Shapes don't connect properly
- Check that connectors have correct source and target IDs
- Ensure the edge attribute is set to "1" for connectors
### Issue: Text formatting is incorrect
- Verify that `html=1` is included in the style for HTML formatting
- Check text alignment attributes in the style
### Issue: Wrong positioning
- Double - check the x, y, width, height values in the geometry
- Ensure that the coordinates are within the visible area of the diagram
## XML vs.Compressed Diagram Content
Draw.io saves files with compressed diagram content to reduce file size.When opening a draw.io XML file in a text editor, you'll typically see the diagram content as a base64-encoded compressed string rather than as plain XML.
To work with the raw XML when editing files manually:
1. Open the diagram in draw.io
2. Go to File > Export as > XML... > (uncheck "Compressed")
3. Save the file
4. The resulting file will have human - readable uncompressed XML
## Resources
- [Official diagrams.net Documentation](https://www.diagrams.net/doc/)
-[mxGraph Documentation](https://jgraph.github.io/mxgraph/docs/manual.html) (the underlying library for draw.io)
-[draw.io XML Format Reference](https://desk.draw.io/support/solutions/articles/16000067790)
## Conclusion
This guide provides a comprehensive overview of the draw.io XML schema and should help you understand and manually create draw.io diagrams.While the GUI interface is generally easier for diagram creation, understanding the XML structure is valuable for programmatic diagram generation, version control, and advanced customization.
Remember that the schema may evolve with new versions of draw.io, but the core structure and concepts described in this guide should remain relevant.

View File

@@ -3,7 +3,8 @@
import React, { useCallback, useRef, useEffect } from "react" import React, { useCallback, useRef, useEffect } from "react"
import { Button } from "@/components/ui/button" import { Button } from "@/components/ui/button"
import { Textarea } from "@/components/ui/textarea" import { Textarea } from "@/components/ui/textarea"
import { Loader2, Send, Trash } from "lucide-react" import { Loader2, Send, Trash, Image as ImageIcon, X } from "lucide-react"
import Image from "next/image"
interface ChatInputProps { interface ChatInputProps {
input: string input: string
@@ -11,10 +12,21 @@ interface ChatInputProps {
onSubmit: (e: React.FormEvent<HTMLFormElement>) => void onSubmit: (e: React.FormEvent<HTMLFormElement>) => void
onChange: (e: React.ChangeEvent<HTMLTextAreaElement>) => void onChange: (e: React.ChangeEvent<HTMLTextAreaElement>) => void
setMessages: (messages: any[]) => void setMessages: (messages: any[]) => void
files?: FileList
onFileChange?: (files: FileList | undefined) => void
} }
export function ChatInput({ input, status, onSubmit, onChange, setMessages }: ChatInputProps) { export function ChatInput({
input,
status,
onSubmit,
onChange,
setMessages,
files,
onFileChange
}: ChatInputProps) {
const textareaRef = useRef<HTMLTextAreaElement>(null) const textareaRef = useRef<HTMLTextAreaElement>(null)
const fileInputRef = useRef<HTMLInputElement>(null)
// Auto-resize textarea based on content // Auto-resize textarea based on content
const adjustTextareaHeight = useCallback(() => { const adjustTextareaHeight = useCallback(() => {
@@ -40,8 +52,63 @@ export function ChatInput({ input, status, onSubmit, onChange, setMessages }: Ch
} }
} }
// Handle file changes
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
if (onFileChange) {
onFileChange(e.target.files || undefined)
}
}
// Clear file selection
const clearFiles = () => {
if (fileInputRef.current) {
fileInputRef.current.value = ''
}
if (onFileChange) {
onFileChange(undefined)
}
}
// Trigger file input click
const triggerFileInput = () => {
fileInputRef.current?.click()
}
return ( return (
<form onSubmit={onSubmit} className="w-full space-y-2"> <form onSubmit={onSubmit} className="w-full space-y-2">
{/* File preview area */}
{files && files.length > 0 && (
<div className="flex flex-wrap gap-2 mt-2 p-2 bg-muted/50 rounded-md">
{Array.from(files).map((file, index) => (
<div key={index} className="relative group">
<div className="w-20 h-20 border rounded-md overflow-hidden bg-muted">
{file.type.startsWith('image/') ? (
<Image
src={URL.createObjectURL(file)}
alt={file.name}
width={80}
height={80}
className="object-cover w-full h-full"
/>
) : (
<div className="flex items-center justify-center h-full text-xs text-center p-1">
{file.name}
</div>
)}
</div>
<button
type="button"
onClick={clearFiles}
className="absolute -top-2 -right-2 bg-destructive text-destructive-foreground rounded-full p-1 opacity-0 group-hover:opacity-100 transition-opacity"
aria-label="Remove file"
>
<X className="h-3 w-3" />
</button>
</div>
))}
</div>
)}
<Textarea <Textarea
ref={textareaRef} ref={textareaRef}
value={input} value={input}
@@ -52,17 +119,42 @@ export function ChatInput({ input, status, onSubmit, onChange, setMessages }: Ch
aria-label="Chat input" aria-label="Chat input"
className="min-h-[80px] resize-none transition-all duration-200" className="min-h-[80px] resize-none transition-all duration-200"
/> />
<div className="flex justify-between gap-2"> <div className="flex justify-between gap-2">
<Button <div className="flex gap-2">
type="button" <Button
variant="outline" type="button"
size="default" variant="outline"
onClick={() => setMessages([])} size="default"
title="Clear messages" onClick={() => setMessages([])}
> title="Clear messages"
<Trash className="mr-2 h-4 w-4" /> >
Start a new conversation <Trash className="mr-2 h-4 w-4" />
</Button> Start a new conversation
</Button>
<Button
type="button"
variant="outline"
size="icon"
onClick={triggerFileInput}
disabled={status === "streaming"}
title="Upload image"
>
<ImageIcon className="h-4 w-4" />
</Button>
<input
type="file"
ref={fileInputRef}
className="hidden"
onChange={handleFileChange}
accept="image/*"
multiple
disabled={status === "streaming"}
/>
</div>
<Button <Button
type="submit" type="submit"
disabled={status === "streaming" || !input.trim()} disabled={status === "streaming" || !input.trim()}

View File

@@ -2,6 +2,7 @@
import type React from "react" import type React from "react"
import { useRef, useEffect, useState } from "react" import { useRef, useEffect, useState } from "react"
import Image from "next/image"
import { Card, CardContent, CardFooter, CardHeader, CardTitle } from "@/components/ui/card" import { Card, CardContent, CardFooter, CardHeader, CardTitle } from "@/components/ui/card"
import { ScrollArea } from "@/components/ui/scroll-area" import { ScrollArea } from "@/components/ui/scroll-area"
@@ -17,6 +18,9 @@ interface ChatPanelProps {
export default function ChatPanel({ onDisplayChart, onFetchChart }: ChatPanelProps) { export default function ChatPanel({ onDisplayChart, onFetchChart }: ChatPanelProps) {
// Add a step counter to track updates // Add a step counter to track updates
const stepCounterRef = useRef<number>(0); const stepCounterRef = useRef<number>(0);
// Add state for file attachments
const [files, setFiles] = useState<FileList | undefined>(undefined);
// Remove the currentXmlRef and related useEffect // Remove the currentXmlRef and related useEffect
const { messages, input, handleInputChange, handleSubmit, status, error, setInput, setMessages, data } = useChat({ const { messages, input, handleInputChange, handleSubmit, status, error, setInput, setMessages, data } = useChat({
maxSteps: 5, maxSteps: 5,
@@ -63,13 +67,23 @@ export default function ChatPanel({ onDisplayChart, onFetchChart }: ChatPanelPro
""" """
` `
) )
handleSubmit(e) handleSubmit(e, {
experimental_attachments: files,
})
// Clear files after submission
setFiles(undefined);
} catch (error) { } catch (error) {
console.error("Error fetching chart data:", error); console.error("Error fetching chart data:", error);
} }
} }
} }
// Helper function to handle file changes
const handleFileChange = (newFiles: FileList | undefined) => {
setFiles(newFiles);
}
// Helper function to render tool invocations // Helper function to render tool invocations
const renderToolInvocation = (toolInvocation: any) => { const renderToolInvocation = (toolInvocation: any) => {
const callId = toolInvocation.toolCallId; const callId = toolInvocation.toolCallId;
@@ -166,6 +180,25 @@ export default function ChatPanel({ onDisplayChart, onFetchChart }: ChatPanelPro
message.content message.content
)} )}
</div> </div>
{/* Display image attachments */}
{message?.experimental_attachments?.filter(attachment =>
attachment?.contentType?.startsWith('image/')
).map((attachment, index) => (
<div key={`${message.id}-${index}`} className={`mt-2 ${message.role === "user" ? "text-right" : "text-left"}`}>
<div className="inline-block">
<Image
src={attachment.url}
width={200}
height={200}
alt={attachment.name ?? `attachment-${index}`}
className="rounded-md border"
style={{ objectFit: 'contain' }}
/>
</div>
</div>
))}
{/* Legacy support for function_call format */} {/* Legacy support for function_call format */}
{(message as any).function_call && ( {(message as any).function_call && (
<div className="mt-2 text-left"> <div className="mt-2 text-left">
@@ -192,6 +225,8 @@ export default function ChatPanel({ onDisplayChart, onFetchChart }: ChatPanelPro
onSubmit={onFormSubmit} onSubmit={onFormSubmit}
onChange={handleInputChange} onChange={handleInputChange}
setMessages={setMessages} setMessages={setMessages}
files={files}
onFileChange={handleFileChange}
/> />
</CardFooter> </CardFooter>
</Card> </Card>