fix(data): cleanup uses failed candidate status instead of 504

The stale-pending cleanup task previously hardcoded status_code=504 and a
generic timeout message for every usage row it finalized. When a request
had already been observed as failing — e.g. upstream Connection reset by
peer, watchdog 504, or an authenticated 4xx — the cleanup overwrote that
context with a misleading "服务器超时" outcome and 504 status, hiding the
real cause from the dashboards and customer.

Pull the most recent failed/cancelled candidate per stale request_id and,
if present, finalize the usage row with the candidate's status_code
(defaulting to 502 when none was recorded) and error_message. Requests
that have no terminal candidate (truly stuck pending/streaming) keep the
existing 504 + timeout-message behavior, since they really are timeouts
from the cleanup's perspective. Applied to all three SQL backends with
parameterized UPDATE statements.

The Postgres failed-candidate lookup orders by
COALESCE(finished_at, started_at, created_at) DESC, matching the MySQL
and SQLite ORDER BY clauses so the three backends pick the same
"most recent terminal candidate" under every NULL combination of timing
columns.

Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
stabey
2026-05-21 16:56:56 +08:00
co-authored by Claude Opus 4.7
parent 40434005c0
commit cadc45c5b8
4 changed files with 315 additions and 14 deletions
@@ -519,13 +519,20 @@ WHERE request_id = ?
continue;
}
let error_message = stale_pending_error_message(&row.status, timeout_minutes);
let candidate_info =
latest_failed_candidate_mysql(&mut tx, &row.request_id).await?;
let (status_code, error_message) = resolve_stale_pending_failure(
candidate_info.as_ref(),
&row.status,
timeout_minutes,
);
let status_code_i64 = i64::from(status_code);
if row.billing_status == "pending" {
sqlx::query(
r#"
UPDATE `usage`
SET status = 'failed',
status_code = 504,
status_code = ?,
error_message = ?,
billing_status = 'void',
finalized_at = ?,
@@ -534,6 +541,7 @@ SET status = 'failed',
WHERE request_id = ?
"#,
)
.bind(status_code_i64)
.bind(&error_message)
.bind(to_i64(now_unix_secs, "usage finalized_at")?)
.bind(&row.request_id)
@@ -551,11 +559,12 @@ WHERE request_id = ?
r#"
UPDATE `usage`
SET status = 'failed',
status_code = 504,
status_code = ?,
error_message = ?
WHERE request_id = ?
"#,
)
.bind(status_code_i64)
.bind(&error_message)
.bind(&row.request_id)
.execute(&mut *tx)
@@ -688,6 +697,67 @@ fn stale_pending_error_message(status: &str, timeout_minutes: u64) -> String {
format!("请求超时: 状态 '{status}' 超过 {timeout_minutes} 分钟未完成")
}
struct FailedCandidateCleanupInfo {
status_code: Option<u16>,
error_message: Option<String>,
}
fn resolve_stale_pending_failure(
candidate: Option<&FailedCandidateCleanupInfo>,
status: &str,
timeout_minutes: u64,
) -> (u16, String) {
match candidate {
Some(info) => (
info.status_code.unwrap_or(502),
info.error_message
.clone()
.unwrap_or_else(|| stale_pending_error_message(status, timeout_minutes)),
),
None => (504, stale_pending_error_message(status, timeout_minutes)),
}
}
async fn latest_failed_candidate_mysql(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
request_id: &str,
) -> Result<Option<FailedCandidateCleanupInfo>, DataLayerError> {
let row = sqlx::query(
r#"
SELECT status_code, error_message
FROM request_candidates
WHERE request_id = ?
AND status IN ('failed', 'cancelled')
ORDER BY
COALESCE(finished_at, started_at, created_at) DESC,
retry_index DESC,
candidate_index DESC
LIMIT 1
"#,
)
.bind(request_id)
.fetch_optional(&mut **tx)
.await
.map_sql_err()?;
let Some(row) = row else {
return Ok(None);
};
let status_code = row
.try_get::<Option<i64>, _>("status_code")
.map_sql_err()?
.and_then(|value| u16::try_from(value).ok());
let error_message = row
.try_get::<Option<String>, _>("error_message")
.map_sql_err()?
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
Ok(Some(FailedCandidateCleanupInfo {
status_code,
error_message,
}))
}
fn bind_upsert<'q>(
mut query: sqlx::query::Query<'q, sqlx::MySql, sqlx::mysql::MySqlArguments>,
usage: &'q UpsertUsageRecord,