feat: add token counting utility for system prompts (#153)

Co-authored-by: dayuan.jiang <jiangdy@amazon.co.jp>

lib/system-prompts.ts

@@ -1,9 +1,12 @@
 /**
  * System prompts for different AI models
  * Extended prompt is used for models with higher cache token minimums (Opus 4.5, Haiku 4.5)
+ *
+ * Token counting utilities are in a separate file (token-counter.ts) to avoid
+ * WebAssembly issues with Next.js server-side rendering.
  */
 
-// Default system prompt (~2700 tokens) - works with all models
+// Default system prompt (~1900 tokens) - works with all models
 export const DEFAULT_SYSTEM_PROMPT = `
 You are an expert diagram creation assistant specializing in draw.io XML generation.
 Your primary function is chat with user and crafting clear, well-organized visual diagrams through precise XML specifications.
@@ -132,8 +135,8 @@ Common styles:
 
 `
 
-// Extended additions (~1800 tokens) - appended for models with 4000 token cache minimum
-// Total EXTENDED_SYSTEM_PROMPT = ~4500 tokens
+// Extended additions (~2600 tokens) - appended for models with 4000 token cache minimum
+// Total EXTENDED_SYSTEM_PROMPT = ~4400 tokens
 const EXTENDED_ADDITIONS = `
 
 ## Extended Tool Reference

lib/token-counter.ts

@@ -0,0 +1,38 @@
+/**
+ * Token counting utilities using Anthropic's tokenizer
+ *
+ * This file is separate from system-prompts.ts because the @anthropic-ai/tokenizer
+ * package uses WebAssembly which doesn't work well with Next.js server-side rendering.
+ * Import this file only in scripts or client-side code, not in API routes.
+ */
+
+import { countTokens } from "@anthropic-ai/tokenizer"
+import { DEFAULT_SYSTEM_PROMPT, EXTENDED_SYSTEM_PROMPT } from "./system-prompts"
+
+/**
+ * Count the number of tokens in a text string using Anthropic's tokenizer
+ * @param text - The text to count tokens for
+ * @returns The number of tokens
+ */
+export function countTextTokens(text: string): number {
+    return countTokens(text)
+}
+
+/**
+ * Get token counts for the system prompts
+ * Useful for debugging and optimizing prompt sizes
+ * @returns Object with token counts for default and extended prompts
+ */
+export function getSystemPromptTokenCounts(): {
+    default: number
+    extended: number
+    additions: number
+} {
+    const defaultTokens = countTokens(DEFAULT_SYSTEM_PROMPT)
+    const extendedTokens = countTokens(EXTENDED_SYSTEM_PROMPT)
+    return {
+        default: defaultTokens,
+        extended: extendedTokens,
+        additions: extendedTokens - defaultTokens,
+    }
+}