diff --git a/app/api/parse-url/route.ts b/app/api/parse-url/route.ts
index 4001454..794b711 100644
--- a/app/api/parse-url/route.ts
+++ b/app/api/parse-url/route.ts
@@ -7,6 +7,31 @@ const MAX_CONTENT_LENGTH = 150000 // Match PDF limit
const EXTRACT_TIMEOUT_MS = 15000
const USER_AGENT = "Mozilla/5.0 (compatible; NextAIDrawio/1.0)"
+// Detect the page's charset so non-UTF-8 pages (Shift_JIS/GBK/EUC/Big5, common
+// on CJK sites) are decoded correctly. Response.text() always assumes UTF-8 and
+// would produce mojibake; the article-extractor library does the same detection
+// when it fetches the page itself, which we no longer rely on.
+function detectCharset(
+ contentType: string | null,
+ buffer: ArrayBuffer,
+): string {
+ // 1. HTTP Content-Type header charset (most authoritative).
+ const headerCharset = contentType?.match(/charset=([^;]+)/i)?.[1]?.trim()
+ // 2. / in the first bytes of the document.
+ const head = new TextDecoder("utf-8").decode(buffer.slice(0, 4096))
+ const metaCharset =
+ head.match(/]+charset=["']?\s*([\w-]+)/i)?.[1] ||
+ head.match(/]+content=["'][^"']*charset=([\w-]+)/i)?.[1]
+ const charset = (headerCharset || metaCharset || "utf-8").toLowerCase()
+ // TextDecoder throws on unknown encoding labels; fall back to UTF-8.
+ try {
+ new TextDecoder(charset)
+ return charset
+ } catch {
+ return "utf-8"
+ }
+}
+
export async function POST(req: Request) {
try {
const { url } = await req.json()
@@ -72,7 +97,9 @@ export async function POST(req: Request) {
)
}
- html = await response.text()
+ const buffer = await response.arrayBuffer()
+ const charset = detectCharset(contentType, buffer)
+ html = new TextDecoder(charset).decode(buffer)
} catch (err: any) {
if (err?.name === "AbortError") {
return NextResponse.json(
@@ -90,7 +117,14 @@ export async function POST(req: Request) {
clearTimeout(timeoutId)
}
- const article = await extractFromHtml(html, url)
+ // extractFromHtml throws (not returns null) on empty/non-HTML bodies,
+ // so map any parse error to the same 400 as the no-content case.
+ let article: Awaited>
+ try {
+ article = await extractFromHtml(html, url)
+ } catch {
+ article = null
+ }
if (!article || !article.content) {
return NextResponse.json(
diff --git a/lib/ssrf-protection.ts b/lib/ssrf-protection.ts
index f9fad85..6b43ddf 100644
--- a/lib/ssrf-protection.ts
+++ b/lib/ssrf-protection.ts
@@ -41,6 +41,7 @@ function isPrivateIp(ip: string): boolean {
if (a === 169 && b === 254) return true // 169.254.0.0/16 (link-local)
if (a === 127) return true // 127.0.0.0/8 (loopback)
if (a === 0) return true // 0.0.0.0/8
+ if (a === 100 && b >= 64 && b <= 127) return true // 100.64.0.0/10 (CGNAT, used by some cloud internal networks)
}
return false
diff --git a/tests/unit/ssrf-protection.test.ts b/tests/unit/ssrf-protection.test.ts
index 53b1830..a05eab3 100644
--- a/tests/unit/ssrf-protection.test.ts
+++ b/tests/unit/ssrf-protection.test.ts
@@ -31,9 +31,26 @@ describe("isPrivateUrl", () => {
expect(await isPrivateUrl("http://10.0.0.5/")).toBe(true)
expect(await isPrivateUrl("http://192.168.1.1/")).toBe(true)
expect(await isPrivateUrl("http://169.254.169.254/")).toBe(true)
+ expect(await isPrivateUrl("http://0.0.0.0/")).toBe(true)
+ // 100.64.0.0/10 CGNAT (RFC 6598), routable in some cloud internal nets
+ expect(await isPrivateUrl("http://100.64.0.1/")).toBe(true)
+ expect(await isPrivateUrl("http://100.127.255.255/")).toBe(true)
expect(lookupMock).not.toHaveBeenCalled()
})
+ it("treats CGNAT boundaries correctly", async () => {
+ // 100.63.x and 100.128.x are outside 100.64.0.0/10 → public
+ lookupMock.mockResolvedValue([{ address: "100.63.255.255", family: 4 }])
+ expect(await isPrivateUrl("http://just-below.example/")).toBe(false)
+ lookupMock.mockResolvedValue([{ address: "100.128.0.1", family: 4 }])
+ expect(await isPrivateUrl("http://just-above.example/")).toBe(false)
+ })
+
+ it("blocks a hostname that resolves to a private IPv6 address", async () => {
+ lookupMock.mockResolvedValue([{ address: "fd00::1", family: 6 }])
+ expect(await isPrivateUrl("http://v6.example.com/")).toBe(true)
+ })
+
it("allows public URLs that resolve to public IPs", async () => {
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }])
expect(await isPrivateUrl("https://example.com/article")).toBe(false)