mirror of
https://github.com/DayuanJiang/next-ai-draw-io.git
synced 2026-09-01 17:10:24 +08:00
fix: SSRF in /api/parse-url via DNS bypass and redirects (#878)
* fix: resolve DNS before SSRF check and block redirects in parse-url isPrivateUrl() did string-only hostname matching and never resolved DNS, so a public-looking name that maps to an internal IP (e.g. 127-0-0-1.sslip.io -> 127.0.0.1) passed the check while fetch/extract later resolved it and reached internal services (GHSA-wqcv-5qvx-vx75). - isPrivateUrl is now async: it keeps the fast string/literal-IP path, then resolves the hostname via DNS and rejects if any address is private. - parse-url now fetches the page itself with redirect: "error" and parses via extractFromHtml(), since article-extractor follows redirects internally and drops a redirect option, which allowed a public URL to 302 to an internal host. - Update validate-model call site to await; add regression tests. * fix: preserve charset detection and block CGNAT range in parse-url SSRF fix Follow-up to the multi-reviewer review of the SSRF fix: - Restore charset handling lost when switching from extract() to response.text(): non-UTF-8 pages (Shift_JIS/GBK/EUC/Big5, common on CJK sites this project targets) decoded as mojibake. Now read the body as bytes, detect charset from Content-Type / <meta charset>, and decode with TextDecoder before extractFromHtml. - Wrap extractFromHtml in try/catch: it throws (not returns null) on empty/non-HTML bodies, which previously surfaced as a 500 instead of the intended 400. - Add 100.64.0.0/10 (RFC 6598 CGNAT) to isPrivateIp; it is routable inside some cloud internal networks and was a residual SSRF target. - Add tests for CGNAT, its boundaries, 0.0.0.0, and DNS-resolved IPv6.
This commit is contained in:
Dayuan Jiang
@@ -1,4 +1,4 @@
|
||||
import { extract } from "@extractus/article-extractor"
|
||||
import { extractFromHtml } from "@extractus/article-extractor"
|
||||
import { NextResponse } from "next/server"
|
||||
import TurndownService from "turndown"
|
||||
import { isPrivateUrl } from "@/lib/ssrf-protection"
|
||||
@@ -7,6 +7,31 @@ const MAX_CONTENT_LENGTH = 150000 // Match PDF limit
|
||||
const EXTRACT_TIMEOUT_MS = 15000
|
||||
const USER_AGENT = "Mozilla/5.0 (compatible; NextAIDrawio/1.0)"
|
||||
|
||||
// Detect the page's charset so non-UTF-8 pages (Shift_JIS/GBK/EUC/Big5, common
|
||||
// on CJK sites) are decoded correctly. Response.text() always assumes UTF-8 and
|
||||
// would produce mojibake; the article-extractor library does the same detection
|
||||
// when it fetches the page itself, which we no longer rely on.
|
||||
function detectCharset(
|
||||
contentType: string | null,
|
||||
buffer: ArrayBuffer,
|
||||
): string {
|
||||
// 1. HTTP Content-Type header charset (most authoritative).
|
||||
const headerCharset = contentType?.match(/charset=([^;]+)/i)?.[1]?.trim()
|
||||
// 2. <meta charset> / <meta http-equiv> in the first bytes of the document.
|
||||
const head = new TextDecoder("utf-8").decode(buffer.slice(0, 4096))
|
||||
const metaCharset =
|
||||
head.match(/<meta[^>]+charset=["']?\s*([\w-]+)/i)?.[1] ||
|
||||
head.match(/<meta[^>]+content=["'][^"']*charset=([\w-]+)/i)?.[1]
|
||||
const charset = (headerCharset || metaCharset || "utf-8").toLowerCase()
|
||||
// TextDecoder throws on unknown encoding labels; fall back to UTF-8.
|
||||
try {
|
||||
new TextDecoder(charset)
|
||||
return charset
|
||||
} catch {
|
||||
return "utf-8"
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(req: Request) {
|
||||
try {
|
||||
const { url } = await req.json()
|
||||
@@ -31,21 +56,31 @@ export async function POST(req: Request) {
|
||||
// SSRF protection: parse-url has no use case for fetching internal
|
||||
// hosts, so private URLs are always rejected. ALLOW_PRIVATE_URLS only
|
||||
// governs LLM provider baseUrl overrides (validate-model, chat).
|
||||
if (isPrivateUrl(url)) {
|
||||
if (await isPrivateUrl(url)) {
|
||||
return NextResponse.json(
|
||||
{ error: "Cannot access private/internal URLs" },
|
||||
{ status: 400 },
|
||||
)
|
||||
}
|
||||
const headController = new AbortController()
|
||||
const headTimeout = setTimeout(() => headController.abort(), 3000)
|
||||
// Fetch the page ourselves so we control redirect handling. The
|
||||
// article-extractor library follows redirects internally and ignores a
|
||||
// `redirect` option, which would let a public URL 302 to an internal
|
||||
// host and bypass the SSRF check above. `redirect: "error"` rejects any
|
||||
// redirect outright.
|
||||
const controller = new AbortController()
|
||||
const timeoutId = setTimeout(() => {
|
||||
controller.abort()
|
||||
}, EXTRACT_TIMEOUT_MS)
|
||||
|
||||
let html: string
|
||||
try {
|
||||
const headResponse = await fetch(url, {
|
||||
method: "HEAD",
|
||||
const response = await fetch(url, {
|
||||
headers: { "User-Agent": USER_AGENT },
|
||||
signal: headController.signal,
|
||||
redirect: "error",
|
||||
signal: controller.signal,
|
||||
})
|
||||
const contentType = headResponse.headers.get("content-type")
|
||||
|
||||
const contentType = response.headers.get("content-type")
|
||||
if (contentType?.includes("application/pdf")) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
@@ -54,27 +89,17 @@ export async function POST(req: Request) {
|
||||
{ status: 422 },
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
"HEAD pre-check failed, proceeding with extraction:",
|
||||
err,
|
||||
)
|
||||
} finally {
|
||||
clearTimeout(headTimeout)
|
||||
}
|
||||
|
||||
// Extract article content with timeout to avoid tying up server resources
|
||||
const controller = new AbortController()
|
||||
const timeoutId = setTimeout(() => {
|
||||
controller.abort()
|
||||
}, EXTRACT_TIMEOUT_MS)
|
||||
if (!response.ok) {
|
||||
return NextResponse.json(
|
||||
{ error: "Could not fetch URL content" },
|
||||
{ status: 400 },
|
||||
)
|
||||
}
|
||||
|
||||
let article
|
||||
try {
|
||||
article = await extract(url, undefined, {
|
||||
headers: { "User-Agent": USER_AGENT },
|
||||
signal: controller.signal,
|
||||
})
|
||||
const buffer = await response.arrayBuffer()
|
||||
const charset = detectCharset(contentType, buffer)
|
||||
html = new TextDecoder(charset).decode(buffer)
|
||||
} catch (err: any) {
|
||||
if (err?.name === "AbortError") {
|
||||
return NextResponse.json(
|
||||
@@ -82,11 +107,25 @@ export async function POST(req: Request) {
|
||||
{ status: 504 },
|
||||
)
|
||||
}
|
||||
throw err
|
||||
// Redirects are rejected with a TypeError ("failed to fetch" /
|
||||
// "unexpected redirect") when redirect: "error" is set.
|
||||
return NextResponse.json(
|
||||
{ error: "Could not fetch URL content" },
|
||||
{ status: 400 },
|
||||
)
|
||||
} finally {
|
||||
clearTimeout(timeoutId)
|
||||
}
|
||||
|
||||
// extractFromHtml throws (not returns null) on empty/non-HTML bodies,
|
||||
// so map any parse error to the same 400 as the no-content case.
|
||||
let article: Awaited<ReturnType<typeof extractFromHtml>>
|
||||
try {
|
||||
article = await extractFromHtml(html, url)
|
||||
} catch {
|
||||
article = null
|
||||
}
|
||||
|
||||
if (!article || !article.content) {
|
||||
return NextResponse.json(
|
||||
{ error: "Could not extract content from URL" },
|
||||
|
||||
@@ -56,7 +56,7 @@ export async function POST(req: Request) {
|
||||
}
|
||||
|
||||
// SECURITY: Block SSRF attacks via custom baseUrl
|
||||
if (baseUrl && !allowPrivateUrls() && isPrivateUrl(baseUrl)) {
|
||||
if (baseUrl && !allowPrivateUrls() && (await isPrivateUrl(baseUrl))) {
|
||||
return NextResponse.json(
|
||||
{ valid: false, error: "Invalid base URL" },
|
||||
{ status: 400 },
|
||||
|
||||
Reference in New Issue
Block a user