From 7b6eb39fa5548e63740944f8545b10da5e0dc522 Mon Sep 17 00:00:00 2001
From: Dayuan Jiang <34411969+DayuanJiang@users.noreply.github.com>
Date: Thu, 21 May 2026 23:54:23 +0900
Subject: [PATCH] fix(parse-url): block SSRF via private/internal URLs (#845)

/api/parse-url accepted any URL the user submitted, fetched it via
@extractus/article-extractor, and returned the body as Markdown. With
ALLOW_PRIVATE_URLS unset (the default after #600) the SSRF guard
short-circuited entirely, so an unauthenticated POST could probe
container ports, read AWS IMDS / GCP metadata, and reach same-VPC
internal services.

- parse-url now always rejects private URLs regardless of
  ALLOW_PRIVATE_URLS. The flag's only legitimate use case is local
  LLM provider baseUrl overrides (validate-model, chat); article
  extraction has no business fetching internal hosts. Local LLM
  setups (Ollama, LM Studio, etc.) are unaffected.
- Strip a trailing dot from the hostname before equality checks so
  the FQDN form "localhost." (which still resolves to 127.0.0.1) is
  caught by the existing string match.

Known follow-ups (not addressed here):
- DNS rebinding: hostnames are matched as strings; a public domain
  resolving to 127.0.0.1 (e.g. localtest.me) is not caught.
- HTTP redirects: @extractus/article-extractor uses cross-fetch with
  default redirect: "follow" and exposes no hook, so a public URL
  302-ing to an internal host still leaks.
---
 app/api/parse-url/route.ts | 8 +++++---
 lib/ssrf-protection.ts     | 4 +++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/api/parse-url/route.ts b/app/api/parse-url/route.ts
index bf3f8f4..4dd6612 100644
--- a/app/api/parse-url/route.ts
+++ b/app/api/parse-url/route.ts
@@ -1,7 +1,7 @@
 import { extract } from "@extractus/article-extractor"
 import { NextResponse } from "next/server"
 import TurndownService from "turndown"
-import { allowPrivateUrls, isPrivateUrl } from "@/lib/ssrf-protection"
+import { isPrivateUrl } from "@/lib/ssrf-protection"
 
 const MAX_CONTENT_LENGTH = 150000 // Match PDF limit
 const EXTRACT_TIMEOUT_MS = 15000
@@ -28,8 +28,10 @@ export async function POST(req: Request) {
             )
         }
 
-        // SSRF protection
-        if (!allowPrivateUrls && isPrivateUrl(url)) {
+        // SSRF protection: parse-url has no use case for fetching internal
+        // hosts, so private URLs are always rejected. ALLOW_PRIVATE_URLS only
+        // governs LLM provider baseUrl overrides (validate-model, chat).
+        if (isPrivateUrl(url)) {
             return NextResponse.json(
                 { error: "Cannot access private/internal URLs" },
                 { status: 400 },
diff --git a/lib/ssrf-protection.ts b/lib/ssrf-protection.ts
index 3c599f2..8dbf2ca 100644
--- a/lib/ssrf-protection.ts
+++ b/lib/ssrf-protection.ts
@@ -9,7 +9,9 @@
 export function isPrivateUrl(urlString: string): boolean {
     try {
         const url = new URL(urlString)
-        const hostname = url.hostname.toLowerCase()
+        // Strip a trailing dot so FQDN forms like "localhost." (which still
+        // resolve to 127.0.0.1) cannot bypass the equality checks below.
+        const hostname = url.hostname.toLowerCase().replace(/\.$/, "")
 
         // Block localhost
         if (