feat: add token counting utility for system prompts (#153)

Co-authored-by: dayuan.jiang <jiangdy@amazon.co.jp>

lib/system-prompts.ts

@@ -1,9 +1,12 @@
 /**
  * System prompts for different AI models
  * Extended prompt is used for models with higher cache token minimums (Opus 4.5, Haiku 4.5)
+ *
+ * Token counting utilities are in a separate file (token-counter.ts) to avoid
+ * WebAssembly issues with Next.js server-side rendering.
  */
 
-// Default system prompt (~2700 tokens) - works with all models
+// Default system prompt (~1900 tokens) - works with all models
 export const DEFAULT_SYSTEM_PROMPT = `
 You are an expert diagram creation assistant specializing in draw.io XML generation.
 Your primary function is chat with user and crafting clear, well-organized visual diagrams through precise XML specifications.
@@ -132,8 +135,8 @@ Common styles:
 
 `
 
-// Extended additions (~1800 tokens) - appended for models with 4000 token cache minimum
+// Extended additions (~2600 tokens) - appended for models with 4000 token cache minimum
-// Total EXTENDED_SYSTEM_PROMPT = ~4500 tokens
+// Total EXTENDED_SYSTEM_PROMPT = ~4400 tokens
 const EXTENDED_ADDITIONS = `
 
 ## Extended Tool Reference

lib/token-counter.ts

@@ -0,0 +1,38 @@
+/**
+ * Token counting utilities using Anthropic's tokenizer
+ *
+ * This file is separate from system-prompts.ts because the @anthropic-ai/tokenizer
+ * package uses WebAssembly which doesn't work well with Next.js server-side rendering.
+ * Import this file only in scripts or client-side code, not in API routes.
+ */
+
+import { countTokens } from "@anthropic-ai/tokenizer"
+import { DEFAULT_SYSTEM_PROMPT, EXTENDED_SYSTEM_PROMPT } from "./system-prompts"
+
+/**
+ * Count the number of tokens in a text string using Anthropic's tokenizer
+ * @param text - The text to count tokens for
+ * @returns The number of tokens
+ */
+export function countTextTokens(text: string): number {
+    return countTokens(text)
+}
+
+/**
+ * Get token counts for the system prompts
+ * Useful for debugging and optimizing prompt sizes
+ * @returns Object with token counts for default and extended prompts
+ */
+export function getSystemPromptTokenCounts(): {
+    default: number
+    extended: number
+    additions: number
+} {
+    const defaultTokens = countTokens(DEFAULT_SYSTEM_PROMPT)
+    const extendedTokens = countTokens(EXTENDED_SYSTEM_PROMPT)
+    return {
+        default: defaultTokens,
+        extended: extendedTokens,
+        additions: extendedTokens - defaultTokens,
+    }
+}

package-lock.json

@@ -22,6 +22,7 @@
                 "@langfuse/tracing": "^4.4.9",
                 "@next/third-parties": "^16.0.6",
                 "@openrouter/ai-sdk-provider": "^1.2.3",
+                "@opentelemetry/exporter-trace-otlp-http": "^0.208.0",
                 "@opentelemetry/sdk-trace-node": "^2.2.0",
                 "@radix-ui/react-dialog": "^1.1.6",
                 "@radix-ui/react-label": "^2.1.8",
@@ -55,6 +56,7 @@
                 "zod": "^4.1.12"
             },
             "devDependencies": {
+                "@anthropic-ai/tokenizer": "^0.0.4",
                 "@biomejs/biome": "2.3.8",
                 "@tailwindcss/postcss": "^4",
                 "@tailwindcss/typography": "^0.5.19",
@@ -332,6 +334,34 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/@anthropic-ai/tokenizer": {
+            "version": "0.0.4",
+            "resolved": "https://registry.npmjs.org/@anthropic-ai/tokenizer/-/tokenizer-0.0.4.tgz",
+            "integrity": "sha512-EHRKbxlxlc8W4KCBEseByJ7YwyYCmgu9OyN59H9+IYIGPoKv8tXyQXinkeGDI+cI8Tiuz9wk2jZb/kK7AyvL7g==",
+            "dev": true,
+            "license": "Apache-2.0",
+            "dependencies": {
+                "@types/node": "^18.11.18",
+                "tiktoken": "^1.0.10"
+            }
+        },
+        "node_modules/@anthropic-ai/tokenizer/node_modules/@types/node": {
+            "version": "18.19.130",
+            "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz",
+            "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "undici-types": "~5.26.4"
+            }
+        },
+        "node_modules/@anthropic-ai/tokenizer/node_modules/undici-types": {
+            "version": "5.26.5",
+            "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+            "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@asamuzakjp/css-color": {
             "version": "3.1.1",
             "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.1.1.tgz",
@@ -2713,7 +2743,6 @@
             "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz",
             "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==",
             "license": "Apache-2.0",
-            "peer": true,
             "dependencies": {
                 "@opentelemetry/api": "^1.3.0"
             },
@@ -2753,7 +2782,6 @@
             "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.208.0.tgz",
             "integrity": "sha512-jbzDw1q+BkwKFq9yxhjAJ9rjKldbt5AgIy1gmEIJjEV/WRxQ3B6HcLVkwbjJ3RcMif86BDNKR846KJ0tY0aOJA==",
             "license": "Apache-2.0",
-            "peer": true,
             "dependencies": {
                 "@opentelemetry/core": "2.2.0",
                 "@opentelemetry/otlp-exporter-base": "0.208.0",
@@ -2773,7 +2801,6 @@
             "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.208.0.tgz",
             "integrity": "sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==",
             "license": "Apache-2.0",
-            "peer": true,
             "dependencies": {
                 "@opentelemetry/core": "2.2.0",
                 "@opentelemetry/otlp-transformer": "0.208.0"
@@ -2790,7 +2817,6 @@
             "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.208.0.tgz",
             "integrity": "sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==",
             "license": "Apache-2.0",
-            "peer": true,
             "dependencies": {
                 "@opentelemetry/api-logs": "0.208.0",
                 "@opentelemetry/core": "2.2.0",
@@ -2828,7 +2854,6 @@
             "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.208.0.tgz",
             "integrity": "sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==",
             "license": "Apache-2.0",
-            "peer": true,
             "dependencies": {
                 "@opentelemetry/api-logs": "0.208.0",
                 "@opentelemetry/core": "2.2.0",
@@ -2846,7 +2871,6 @@
             "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.2.0.tgz",
             "integrity": "sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==",
             "license": "Apache-2.0",
-            "peer": true,
             "dependencies": {
                 "@opentelemetry/core": "2.2.0",
                 "@opentelemetry/resources": "2.2.0"
@@ -2905,36 +2929,31 @@
             "version": "1.1.2",
             "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
             "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/base64": {
             "version": "1.1.2",
             "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
             "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/codegen": {
             "version": "2.0.4",
             "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
             "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/eventemitter": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
             "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/fetch": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
             "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
             "license": "BSD-3-Clause",
-            "peer": true,
             "dependencies": {
                 "@protobufjs/aspromise": "^1.1.1",
                 "@protobufjs/inquire": "^1.1.0"
@@ -2944,36 +2963,31 @@
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
             "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/inquire": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
             "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/path": {
             "version": "1.1.2",
             "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
             "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/pool": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
             "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@protobufjs/utf8": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
             "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
-            "license": "BSD-3-Clause",
+            "license": "BSD-3-Clause"
-            "peer": true
         },
         "node_modules/@radix-ui/number": {
             "version": "1.1.0",
@@ -9339,8 +9353,7 @@
             "version": "5.3.2",
             "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
             "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
-            "license": "Apache-2.0",
+            "license": "Apache-2.0"
-            "peer": true
         },
         "node_modules/longest-streak": {
             "version": "3.1.0",
@@ -10917,7 +10930,6 @@
             "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==",
             "hasInstallScript": true,
             "license": "BSD-3-Clause",
-            "peer": true,
             "dependencies": {
                 "@protobufjs/aspromise": "^1.1.2",
                 "@protobufjs/base64": "^1.1.2",
@@ -12056,6 +12068,13 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/tiktoken": {
+            "version": "1.0.22",
+            "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz",
+            "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/tinyglobby": {
             "version": "0.2.15",
             "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",

package.json

@@ -66,6 +66,7 @@
         ]
     },
     "devDependencies": {
+        "@anthropic-ai/tokenizer": "^0.0.4",
         "@biomejs/biome": "2.3.8",
         "@tailwindcss/postcss": "^4",
         "@tailwindcss/typography": "^0.5.19",