fix: restore security hardening compatibility and validation

Restore authorized rule reveal, explicit full HTTP capture and retention, video task business fields, and valid payment URLs. Add opt-in credential preservation for trusted recovery, fix frontend type contracts and async races, and eliminate PostgreSQL test fixture resource leaks. Document audit coverage and successful fmt and CI-scoped Clippy checks.
This commit is contained in:
elky
2026-09-07 21:14:27 +08:00
parent a5c3699ae9
commit a90d564931
191 changed files with 6785 additions and 1643 deletions
+3 -3
View File
@@ -115,9 +115,9 @@ ADMIN_USERNAME=admin123456
# Fake-IP 域名及内网 DNS。仅信任管理员配置的上游;没有严格 DNS 过滤开关。 # Fake-IP 域名及内网 DNS。仅信任管理员配置的上游;没有严格 DNS 过滤开关。
# URL 协议、字面 IP、TLS 证书,以及隧道中继和登录 OAuth 的校验仍保留。 # URL 协议、字面 IP、TLS 证书,以及隧道中继和登录 OAuth 的校验仍保留。
# 可选 Provider OAuth 客户端。Gemini CLI 授权及刷新必须配置 client secret。 # 可选 Provider OAuth 客户端。Gemini CLI 和 Antigravity 默认使用内置 native-app
# Antigravity 默认使用内置 native-app 客户端凭据;自定义 client ID 时必须同时配置 # 客户端凭据;自定义 client ID 时必须同时配置对应的 client secret。
# 对应的 client secret。未配置 client ID 时使用内置的公开 native-app client ID # 显式配置的 client secret 优先于默认值
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID= # AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET= # AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID= # AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
@@ -496,6 +496,7 @@ fn access_for_route(method: &http::Method, decision: &GatewayControlDecision) ->
Some("admin:endpoints_manage"), Some("admin:endpoints_manage"),
Some( Some(
"reveal_key" "reveal_key"
| "reveal_endpoint_rules"
| "export_key" | "export_key"
| "create_provider_key" | "create_provider_key"
| "update_key" | "update_key"
@@ -1490,6 +1491,12 @@ mod tests {
fn plaintext_credential_reads_require_admin_permission() { fn plaintext_credential_reads_require_admin_permission() {
let read_only_permissions = read_only_management_token_permissions(); let read_only_permissions = read_only_management_token_permissions();
let cases = [ let cases = [
(
"admin:endpoints_manage",
"reveal_endpoint_rules",
None,
"admin:endpoints_manage:admin",
),
( (
"admin:endpoints_manage", "admin:endpoints_manage",
"reveal_key", "reveal_key",
@@ -302,6 +302,19 @@ pub(super) fn classify_admin_endpoints_family_route(
"admin:endpoints_manage", "admin:endpoints_manage",
false, false,
)) ))
} else if method == http::Method::GET
&& normalized_path
.strip_prefix("/api/admin/endpoints/")
.and_then(|path| path.strip_suffix("/rules/reveal"))
.is_some_and(|endpoint_id| !endpoint_id.is_empty() && !endpoint_id.contains('/'))
{
Some(classified(
"admin_proxy",
"endpoints_manage",
"reveal_endpoint_rules",
"admin:endpoints_manage",
false,
))
} else if method == http::Method::GET } else if method == http::Method::GET
&& normalized_path.starts_with("/api/admin/endpoints/") && normalized_path.starts_with("/api/admin/endpoints/")
&& !normalized_path.starts_with("/api/admin/endpoints/health/") && !normalized_path.starts_with("/api/admin/endpoints/health/")
@@ -381,6 +381,28 @@ fn classifies_admin_get_endpoint_as_admin_proxy_route() {
assert!(!decision.is_execution_runtime_candidate()); assert!(!decision.is_execution_runtime_candidate());
} }
#[test]
fn classifies_admin_reveal_endpoint_rules_as_admin_proxy_route() {
let headers = headers(&[]);
let uri: Uri = "/api/admin/endpoints/endpoint-1/rules/reveal"
.parse()
.expect("uri should parse");
let decision =
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
assert_eq!(decision.route_family.as_deref(), Some("endpoints_manage"));
assert_eq!(
decision.route_kind.as_deref(),
Some("reveal_endpoint_rules")
);
assert_eq!(
decision.auth_endpoint_signature.as_deref(),
Some("admin:endpoints_manage")
);
assert!(!decision.is_execution_runtime_candidate());
}
#[test] #[test]
fn classifies_admin_create_endpoint_as_admin_proxy_route() { fn classifies_admin_create_endpoint_as_admin_proxy_route() {
let headers = http::HeaderMap::new(); let headers = http::HeaderMap::new();
@@ -41,16 +41,9 @@ fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestR
return UsageRequestRecordLevel::Basic; return UsageRequestRecordLevel::Basic;
}; };
if value.eq_ignore_ascii_case("basic") if value.eq_ignore_ascii_case("full") {
|| value.eq_ignore_ascii_case("base") UsageRequestRecordLevel::Full
|| value.eq_ignore_ascii_case("headers")
|| value.eq_ignore_ascii_case("minimal")
|| value.eq_ignore_ascii_case("none")
{
UsageRequestRecordLevel::Basic
} else { } else {
// Raw HTTP payload capture is disabled at the runtime boundary. The setting remains
// accepted for compatibility, but no longer authorizes collecting request/response data.
UsageRequestRecordLevel::Basic UsageRequestRecordLevel::Basic
} }
} }
@@ -501,7 +494,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn usage_runtime_access_disables_full_http_capture() { async fn usage_runtime_access_honors_explicit_full_http_capture() {
let state = GatewayDataState::disabled().with_system_config_values_for_tests([( let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
"request_record_level".to_string(), "request_record_level".to_string(),
json!("full"), json!("full"),
@@ -511,7 +504,32 @@ mod tests {
.await .await
.expect("request record level should read"); .expect("request record level should read");
assert_eq!(level, UsageRequestRecordLevel::Basic); assert_eq!(level, UsageRequestRecordLevel::Full);
}
#[tokio::test]
async fn usage_runtime_access_honors_legacy_full_without_overriding_current_config() {
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
"request_log_level".to_string(),
json!(" FULL "),
)]);
assert_eq!(
UsageRuntimeAccess::request_record_level(&state)
.await
.unwrap(),
UsageRequestRecordLevel::Full
);
let state = state.with_system_config_values_for_tests([
("request_log_level".to_string(), json!("full")),
("request_record_level".to_string(), json!("basic")),
]);
assert_eq!(
UsageRuntimeAccess::request_record_level(&state)
.await
.unwrap(),
UsageRequestRecordLevel::Basic
);
} }
#[tokio::test] #[tokio::test]
@@ -426,11 +426,8 @@ mod tests {
assert!(candidate.finished_at_unix_ms.is_some()); assert!(candidate.finished_at_unix_ms.is_some());
} }
/// The guard holds no request body, and the persistence boundary intentionally
/// rejects request/response capture material. A dropped-attempt settlement
/// must not re-introduce an inline body or a caller-controlled body reference.
#[tokio::test] #[tokio::test]
async fn settling_a_dropped_attempt_does_not_reintroduce_request_body_capture() { async fn settling_a_dropped_attempt_respects_disabled_request_body_capture() {
let usage_repository = Arc::new(InMemoryUsageReadRepository::default()); let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default()); let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let state = test_state(&usage_repository, &request_candidate_repository); let state = test_state(&usage_repository, &request_candidate_repository);
@@ -444,8 +441,6 @@ mod tests {
candidate_started_unix_ms, candidate_started_unix_ms,
) )
.await; .await;
// This deliberately supplies capture material to prove that the usage
// persistence boundary strips it before either lifecycle write stores it.
let captured_body = json!({"stream": true, "service_tier": "priority"}); let captured_body = json!({"stream": true, "service_tier": "priority"});
let mut capture = build_pending_usage_record( let mut capture = build_pending_usage_record(
&plan, &plan,
@@ -481,7 +476,10 @@ mod tests {
.expect("cancelled usage should be recorded"); .expect("cancelled usage should be recorded");
assert_eq!(usage.provider_request_body, None); assert_eq!(usage.provider_request_body, None);
assert_eq!(usage.provider_request_body_ref, None); assert_eq!(usage.provider_request_body_ref, None);
assert_eq!(usage.provider_request_body_state, None); assert_eq!(
usage.provider_request_body_state,
Some(UsageBodyCaptureState::Disabled)
);
} }
#[tokio::test] #[tokio::test]
@@ -14661,15 +14661,19 @@ mod tests {
assert_eq!(usage.status_code, Some(302)); assert_eq!(usage.status_code, Some(302));
assert_eq!(usage.error_category.as_deref(), Some("redirect")); assert_eq!(usage.error_category.as_deref(), Some("redirect"));
assert!(usage.error_message.is_none()); assert!(usage.error_message.is_none());
// HTTP capture is intentionally disabled at the persistence boundary. Keep the assert_eq!(
// protocol facts above, but do not turn provider/client headers into an audit store. usage.client_response_headers.as_ref().unwrap()["content-type"],
assert!(usage.client_response_headers.is_none()); json!("application/json")
assert!(usage.response_headers.is_none()); );
assert_eq!(
usage.response_headers.as_ref().unwrap()["content-type"],
json!("text/html")
);
assert!( assert!(
usage.response_body.is_none(), usage.response_body.is_none(),
"upstream redirect did not include a body" "upstream redirect did not include a body"
); );
assert!(usage.client_response_body.is_none()); assert_eq!(usage.client_response_body.as_ref(), Some(&body_json));
let candidates = request_candidate_repository let candidates = request_candidate_repository
.list_by_request_id("req-remote-runtime-stream-redirect") .list_by_request_id("req-remote-runtime-stream-redirect")
.await .await
@@ -6,6 +6,7 @@ mod extractors;
mod list; mod list;
pub(crate) mod payloads; pub(crate) mod payloads;
mod reads; mod reads;
mod reveal;
mod support; mod support;
mod update; mod update;
@@ -41,6 +42,10 @@ pub(crate) async fn maybe_build_local_admin_endpoints_routes_response(
return Ok(Some(response)); return Ok(Some(response));
} }
if let Some(response) = reveal::maybe_handle(state, request_context).await? {
return Ok(Some(response));
}
if let Some(response) = defaults::maybe_handle(state, request_context, request_body).await? { if let Some(response) = defaults::maybe_handle(state, request_context, request_body).await? {
return Ok(Some(response)); return Ok(Some(response));
} }
@@ -0,0 +1,72 @@
use super::extractors::admin_endpoint_id;
use super::support::build_admin_endpoints_data_unavailable_response;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{
attach_admin_audit_response, mark_sensitive_admin_response_no_store,
};
use crate::GatewayError;
use axum::{
body::Body,
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) async fn maybe_handle(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.decision() else {
return Ok(None);
};
if decision.route_family.as_deref() != Some("endpoints_manage")
|| decision.route_kind.as_deref() != Some("reveal_endpoint_rules")
{
return Ok(None);
}
if !state.has_provider_catalog_data_reader() {
return Ok(Some(build_admin_endpoints_data_unavailable_response()));
}
let Some(endpoint_id) = request_context
.path()
.strip_suffix("/rules/reveal")
.and_then(admin_endpoint_id)
else {
return Ok(Some(
(
StatusCode::NOT_FOUND,
Json(json!({ "detail": "Endpoint 不存在" })),
)
.into_response(),
));
};
let Some(endpoint) = state
.read_provider_catalog_endpoints_by_ids(std::slice::from_ref(&endpoint_id))
.await?
.into_iter()
.next()
else {
return Ok(Some(
(
StatusCode::NOT_FOUND,
Json(json!({ "detail": "Endpoint 不存在" })),
)
.into_response(),
));
};
let payload = json!({
"header_rules": endpoint.header_rules.as_ref().and_then(|value| value.as_array()).cloned().unwrap_or_default(),
"body_rules": endpoint.body_rules.as_ref().and_then(|value| value.as_array()).cloned().unwrap_or_default(),
"response_header_rules": endpoint.config.as_ref().and_then(|config| config.get("response_header_rules")).and_then(|value| value.as_array()).cloned().unwrap_or_default(),
});
Ok(Some(mark_sensitive_admin_response_no_store(
attach_admin_audit_response(
Json(payload).into_response(),
"admin_endpoint_rules_revealed",
"reveal_endpoint_rules",
"provider_endpoint",
&endpoint_id,
),
)))
}
+64 -3
View File
@@ -117,8 +117,8 @@ where
use aether_crypto::warm_python_fernet_secret; use aether_crypto::warm_python_fernet_secret;
use aether_data::lifecycle::export::{ use aether_data::lifecycle::export::{
copy_database_records, export_database_jsonl, import_database_jsonl, DataCopyOptions, copy_database_records, export_database_jsonl, import_database_jsonl_with_options,
ExportDomain, MAX_JSONL_INPUT_BYTES, DataCopyOptions, DataImportOptions, ExportDomain, MAX_JSONL_INPUT_BYTES,
}; };
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig}; use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
use aether_gateway::{ use aether_gateway::{
@@ -1351,6 +1351,11 @@ struct DataExportArgs {
struct DataImportArgs { struct DataImportArgs {
#[arg(long)] #[arg(long)]
input: PathBuf, input: PathBuf,
#[arg(
long,
help = "Preserve passwords and API/management credentials from a trusted import; imported sessions remain revoked. Without this flag identity credentials are revoked."
)]
preserve_credentials: bool,
} }
#[derive(ClapArgs, Debug, Clone)] #[derive(ClapArgs, Debug, Clone)]
@@ -1382,6 +1387,11 @@ struct DataCopyArgs {
#[arg(long)] #[arg(long)]
omit_request_body_details: bool, omit_request_body_details: bool,
#[arg(
long,
help = "Preserve passwords and API/management credentials from the trusted source; imported sessions remain revoked. The target must use the source encryption key."
)]
preserve_credentials: bool,
} }
impl GatewayLoggingArgs { impl GatewayLoggingArgs {
@@ -2905,12 +2915,23 @@ async fn run_data_import(
let driver = database.driver; let driver = database.driver;
let input_path = args.input.clone(); let input_path = args.input.clone();
let input = tokio::task::spawn_blocking(move || read_data_import_input(&input_path)).await??; let input = tokio::task::spawn_blocking(move || read_data_import_input(&input_path)).await??;
let imported = import_database_jsonl(database, &input).await?; if !args.preserve_credentials {
warn!("identity credentials will be revoked; use --preserve-credentials only for trusted recovery or migration");
}
let imported = import_database_jsonl_with_options(
database,
&input,
DataImportOptions {
preserve_credentials: args.preserve_credentials,
},
)
.await?;
info!( info!(
driver = %driver, driver = %driver,
input = %args.input.display(), input = %args.input.display(),
imported, imported,
preserve_credentials = args.preserve_credentials,
"database import complete" "database import complete"
); );
println!( println!(
@@ -3171,6 +3192,9 @@ async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Er
let target_driver = target.driver; let target_driver = target.driver;
let domains = requested_domains(&args.domains); let domains = requested_domains(&args.domains);
let created_at_unix_secs = current_unix_secs()?; let created_at_unix_secs = current_unix_secs()?;
if !args.preserve_credentials {
warn!("identity credentials will be revoked; use --preserve-credentials only for trusted recovery or migration");
}
let imported = copy_database_records( let imported = copy_database_records(
source, source,
target, target,
@@ -3178,6 +3202,7 @@ async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Er
created_at_unix_secs, created_at_unix_secs,
DataCopyOptions { DataCopyOptions {
omit_request_body_details: args.omit_request_body_details, omit_request_body_details: args.omit_request_body_details,
preserve_credentials: args.preserve_credentials,
}, },
) )
.await?; .await?;
@@ -3186,6 +3211,7 @@ async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Er
source_driver = %source_driver, source_driver = %source_driver,
target_driver = %target_driver, target_driver = %target_driver,
imported, imported,
preserve_credentials = args.preserve_credentials,
"database copy complete" "database copy complete"
); );
println!( println!(
@@ -4357,6 +4383,41 @@ mod tests {
}; };
assert!(copy.source_allow_insecure); assert!(copy.source_allow_insecure);
assert!(!copy.target_allow_insecure); assert!(!copy.target_allow_insecure);
assert!(!copy.preserve_credentials);
}
#[test]
fn data_import_and_copy_require_explicit_credential_preservation() {
for preserve in [false, true] {
let mut import_args = vec!["aether-gateway", "import", "--input", "trusted.jsonl"];
let mut copy_args = vec![
"aether-gateway",
"copy",
"--source-driver",
"postgres",
"--source-url",
"postgres://localhost/source",
"--target-driver",
"postgres",
"--target-url",
"postgres://localhost/target",
];
if preserve {
import_args.push("--preserve-credentials");
copy_args.push("--preserve-credentials");
}
let Some(DataCommand::Import(import)) =
Args::try_parse_from(import_args).unwrap().command
else {
panic!("expected import command");
};
let Some(DataCommand::Copy(copy)) = Args::try_parse_from(copy_args).unwrap().command
else {
panic!("expected copy command");
};
assert_eq!(import.preserve_credentials, preserve);
assert_eq!(copy.preserve_credentials, preserve);
}
} }
#[cfg(unix)] #[cfg(unix)]
@@ -115,9 +115,6 @@ pub(super) fn usage_cleanup_window(
usage_cleanup_window_with_override(now_utc, settings, None) usage_cleanup_window_with_override(now_utc, settings, None)
} }
/// Clamp is non-aggressive: each tier's cutoff becomes `max(policy_cutoff, now - override)`.
/// A later cutoff = fewer records deleted, so the override can only make cleanup more
/// conservative than the configured retention, never more destructive.
pub(super) fn usage_cleanup_window_with_override( pub(super) fn usage_cleanup_window_with_override(
now_utc: DateTime<Utc>, now_utc: DateTime<Utc>,
settings: UsageCleanupSettings, settings: UsageCleanupSettings,
@@ -135,9 +132,9 @@ pub(super) fn usage_cleanup_window_with_override(
}; };
let manual_cutoff = now_utc - override_duration; let manual_cutoff = now_utc - override_duration;
UsageCleanupWindow { UsageCleanupWindow {
detail_cutoff: policy.detail_cutoff.max(manual_cutoff), detail_cutoff: policy.detail_cutoff.min(manual_cutoff),
compressed_cutoff: policy.compressed_cutoff.max(manual_cutoff), compressed_cutoff: policy.compressed_cutoff.min(manual_cutoff),
header_cutoff: policy.header_cutoff.max(manual_cutoff), header_cutoff: policy.header_cutoff.min(manual_cutoff),
log_cutoff: policy.log_cutoff.max(manual_cutoff), log_cutoff: policy.log_cutoff.min(manual_cutoff),
} }
} }
@@ -1140,19 +1140,40 @@ fn usage_cleanup_window_with_override_is_always_non_aggressive() {
let override_duration = chrono::Duration::days(180); let override_duration = chrono::Duration::days(180);
let clamped = usage_cleanup_window_with_override(now_utc, settings, Some(override_duration)); let clamped = usage_cleanup_window_with_override(now_utc, settings, Some(override_duration));
assert_eq!(clamped.detail_cutoff, policy.detail_cutoff); assert_eq!(clamped.detail_cutoff, now_utc - override_duration);
assert_eq!(clamped.compressed_cutoff, policy.compressed_cutoff); assert_eq!(clamped.compressed_cutoff, now_utc - override_duration);
assert_eq!(clamped.header_cutoff, policy.header_cutoff); assert_eq!(clamped.header_cutoff, now_utc - override_duration);
assert_eq!(clamped.log_cutoff, now_utc - override_duration); assert_eq!(clamped.log_cutoff, policy.log_cutoff);
assert!(clamped.log_cutoff > policy.log_cutoff); assert!(clamped.log_cutoff <= policy.log_cutoff);
let far_override = chrono::Duration::days(5); let far_override = chrono::Duration::days(5);
let far = usage_cleanup_window_with_override(now_utc, settings, Some(far_override)); let far = usage_cleanup_window_with_override(now_utc, settings, Some(far_override));
assert_eq!(far.detail_cutoff, now_utc - far_override); assert_eq!(far, policy);
assert_eq!(far.compressed_cutoff, now_utc - far_override);
assert_eq!(far.header_cutoff, now_utc - far_override); for days in [0, 5, 30, 180, 400] {
assert_eq!(far.log_cutoff, now_utc - far_override); let cutoff = now_utc - chrono::Duration::days(days);
assert!(far.log_cutoff > policy.log_cutoff); let window = usage_cleanup_window_with_override(
now_utc,
settings,
Some(chrono::Duration::days(days)),
);
for (actual, configured) in [
(window.detail_cutoff, policy.detail_cutoff),
(window.compressed_cutoff, policy.compressed_cutoff),
(window.header_cutoff, policy.header_cutoff),
(window.log_cutoff, policy.log_cutoff),
] {
assert!(actual <= configured);
assert!(actual <= cutoff);
for age in [1, 7, 15, 30, 90, 180, 365, 401] {
let created_at = now_utc - chrono::Duration::days(age);
if created_at < actual {
assert!(created_at < configured);
assert!(created_at < cutoff);
}
}
}
}
let passthrough = usage_cleanup_window_with_override(now_utc, settings, None); let passthrough = usage_cleanup_window_with_override(now_utc, settings, None);
assert_eq!(passthrough, policy); assert_eq!(passthrough, policy);
+17 -4
View File
@@ -326,7 +326,7 @@ async fn gateway_reads_video_task_detail_via_internal_async_task_endpoint() {
} }
#[tokio::test] #[tokio::test]
async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endpoint() { async fn gateway_redirects_persisted_openai_video_url_from_authenticated_internal_endpoint() {
let repository = Arc::new(InMemoryVideoTaskRepository::default()); let repository = Arc::new(InMemoryVideoTaskRepository::default());
let mut task = sample_video_task( let mut task = sample_video_task(
"task-redirect", "task-redirect",
@@ -341,7 +341,10 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endp
.upsert(task) .upsert(task)
.await .await
.expect("upsert should succeed"); .expect("upsert should succeed");
assert_eq!(stored.video_url, None); assert_eq!(
stored.video_url.as_deref(),
Some("https://8.8.8.8/video-task-redirect.mp4")
);
let state = AppState::new() let state = AppState::new()
.expect("gateway state should build") .expect("gateway state should build")
@@ -349,7 +352,10 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endp
let (gateway_url, gateway_handle, access_token) = let (gateway_url, gateway_handle, access_token) =
start_authenticated_operational_server(state).await; start_authenticated_operational_server(state).await;
let client = authenticated_operational_client(&access_token); let client = super::authenticated_operational_client_with_builder(
reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()),
&access_token,
);
let response = client let response = client
.get(format!( .get(format!(
"{gateway_url}/_gateway/async-tasks/video-tasks/task-redirect/video" "{gateway_url}/_gateway/async-tasks/video-tasks/task-redirect/video"
@@ -358,7 +364,14 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endp
.await .await
.expect("request should succeed"); .expect("request should succeed");
assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT);
assert_eq!(
response
.headers()
.get("location")
.and_then(|value| value.to_str().ok()),
stored.video_url.as_deref()
);
gateway_handle.abort(); gateway_handle.abort();
} }
@@ -1,3 +1,4 @@
mod keys; mod keys;
mod quota; mod quota;
mod routes; mod routes;
mod rules_reveal;
@@ -0,0 +1,151 @@
use std::sync::Arc;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use axum::body::Body;
use http::{HeaderMap, HeaderValue, Method, Request, StatusCode};
use http_body_util::BodyExt;
use serde_json::{json, Value};
use super::super::super::{build_router_with_state, sample_endpoint, sample_provider, AppState};
use crate::admin_api::{maybe_build_local_admin_response, AdminRouteRequest};
use crate::audit::AdminAuditEvent;
use crate::constants::{
GATEWAY_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER, TRUSTED_ADMIN_USER_ID_HEADER,
TRUSTED_ADMIN_USER_ROLE_HEADER,
};
use crate::control::resolve_public_request_context;
use crate::data::GatewayDataState;
use crate::tests::send_request;
fn seeded_state() -> AppState {
let mut endpoint = sample_endpoint(
"endpoint-rules",
"provider-rules",
"openai:chat",
"https://example.test",
);
endpoint.header_rules =
Some(json!([{"action": "set", "key": "x-auth", "value": "request-secret"}]));
endpoint.body_rules =
Some(json!([{"action": "set", "path": "auth.token", "value": "body-secret"}]));
endpoint.config = Some(json!({
"private_token": "unrelated-secret",
"response_header_rules": [{"action": "set", "key": "x-auth", "value": "response-secret"}]
}));
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider("provider-rules", "custom", 10)],
vec![endpoint],
vec![],
));
AppState::new().unwrap().with_data_state_for_tests(
GatewayDataState::with_provider_catalog_reader_for_tests(repository),
)
}
fn admin_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
for (name, value) in [
(GATEWAY_HEADER, "rust-phase3b"),
(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user"),
(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin"),
(TRUSTED_ADMIN_SESSION_ID_HEADER, "admin-session"),
] {
headers.insert(name, HeaderValue::from_static(value));
}
headers
}
#[tokio::test]
async fn endpoint_rules_reveal_is_scoped_audited_and_not_cached() {
let state = seeded_state();
let context = resolve_public_request_context(
&state,
&Method::GET,
&"/api/admin/endpoints/endpoint-rules/rules/reveal"
.parse()
.unwrap(),
&admin_headers(),
"reveal-test",
)
.await
.unwrap();
let response = maybe_build_local_admin_response(AdminRouteRequest::new(
&state,
&context,
&"127.0.0.1:12345".parse().unwrap(),
&admin_headers(),
None,
))
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()[http::header::CACHE_CONTROL], "no-store");
assert_eq!(response.headers()[http::header::PRAGMA], "no-cache");
let audit = response.extensions().get::<AdminAuditEvent>().unwrap();
assert_eq!(audit.event_name, "admin_endpoint_rules_revealed");
assert_eq!(audit.action, "reveal_endpoint_rules");
assert_eq!(audit.target_id, "endpoint-rules");
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["header_rules"][0]["value"], "request-secret");
assert_eq!(payload["body_rules"][0]["value"], "body-secret");
assert_eq!(
payload["response_header_rules"][0]["value"],
"response-secret"
);
assert_eq!(payload.as_object().unwrap().len(), 3);
assert!(!payload.to_string().contains("unrelated-secret"));
}
#[tokio::test]
async fn endpoint_rules_reveal_denies_anonymous_and_non_admin_requests() {
let router = build_router_with_state(seeded_state());
for role in [None, Some("user")] {
let mut request =
Request::builder().uri("/api/admin/endpoints/endpoint-rules/rules/reveal");
if let Some(role) = role {
request = request
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "normal-user")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, role)
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "user-session");
}
let response = send_request(router.clone(), request.body(Body::empty()).unwrap()).await;
assert!(matches!(
response.status(),
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
));
let body = response.into_body().collect().await.unwrap().to_bytes();
assert!(!String::from_utf8_lossy(&body).contains("request-secret"));
}
}
#[tokio::test]
async fn endpoint_rules_reveal_returns_not_found_and_data_unavailable_without_fallback() {
for (state, expected) in [
(seeded_state(), StatusCode::NOT_FOUND),
(AppState::new().unwrap(), StatusCode::SERVICE_UNAVAILABLE),
] {
let context = resolve_public_request_context(
&state,
&Method::GET,
&"/api/admin/endpoints/missing/rules/reveal".parse().unwrap(),
&admin_headers(),
"reveal-missing-test",
)
.await
.unwrap();
let response = maybe_build_local_admin_response(AdminRouteRequest::new(
&state,
&context,
&"127.0.0.1:12345".parse().unwrap(),
&admin_headers(),
None,
))
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), expected);
}
}
@@ -221,13 +221,17 @@ async fn gateway_handles_admin_video_tasks_list_locally_with_trusted_admin_princ
assert_eq!(payload["pages"], json!(1)); assert_eq!(payload["pages"], json!(1));
assert_eq!(payload["items"].as_array().map(Vec::len), Some(1)); assert_eq!(payload["items"].as_array().map(Vec::len), Some(1));
assert_eq!(payload["items"][0]["id"], "task-completed"); assert_eq!(payload["items"][0]["id"], "task-completed");
// Video-task persistence intentionally drops user-facing PII. The admin assert_eq!(payload["items"][0]["username"], "alice");
// projection must therefore use the privacy-safe fallback when no separate
// user snapshot is joined.
assert_eq!(payload["items"][0]["username"], "Unknown");
assert_eq!(payload["items"][0]["provider_name"], "OpenAI"); assert_eq!(payload["items"][0]["provider_name"], "OpenAI");
assert_eq!(payload["items"][0]["status"], "completed"); assert_eq!(payload["items"][0]["status"], "completed");
assert!(payload["items"][0]["prompt"].is_null()); assert_eq!(
payload["items"][0]["prompt"],
format!("{}...", "x".repeat(100))
);
assert_eq!(
payload["items"][0]["video_url"],
"https://8.8.8.8/task-completed.mp4"
);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort(); gateway_handle.abort();
@@ -393,7 +397,9 @@ async fn gateway_handles_admin_video_task_detail_locally_with_trusted_admin_prin
assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse"); let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["id"], "task-detail"); assert_eq!(payload["id"], "task-detail");
assert_eq!(payload["username"], "Unknown"); assert_eq!(payload["prompt"], "detail prompt");
assert_eq!(payload["video_url"], "https://8.8.8.8/task-detail.mp4");
assert_eq!(payload["username"], "charlie");
assert_eq!(payload["provider_name"], "OpenAI"); assert_eq!(payload["provider_name"], "OpenAI");
assert_eq!(payload["endpoint"]["id"], "endpoint-1"); assert_eq!(payload["endpoint"]["id"], "endpoint-1");
assert_eq!(payload["endpoint"]["api_format"], "openai:video"); assert_eq!(payload["endpoint"]["api_format"], "openai:video");
@@ -734,7 +740,7 @@ async fn local_admin_video_task_cancel_attaches_explicit_audit() {
} }
#[tokio::test] #[tokio::test]
async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstream() { async fn gateway_redirects_persisted_openai_video_url_without_forwarding_admin_request() {
let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits = Arc::new(Mutex::new(0usize));
let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream_hits_clone = Arc::clone(&upstream_hits);
let upstream = Router::new().route( let upstream = Router::new().route(
@@ -762,7 +768,10 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstrea
)) ))
.await .await
.expect("task should upsert"); .expect("task should upsert");
assert_eq!(stored.video_url, None); assert_eq!(
stored.video_url.as_deref(),
Some("https://8.8.8.8/task-redirect.mp4")
);
let (_upstream_url, upstream_handle) = start_server(upstream).await; let (_upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state( let gateway = build_router_with_state(
@@ -788,7 +797,14 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstrea
.await .await
.expect("request should succeed"); .expect("request should succeed");
assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT);
assert_eq!(
response
.headers()
.get(http::header::LOCATION)
.and_then(|value| value.to_str().ok()),
stored.video_url.as_deref()
);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort(); gateway_handle.abort();
@@ -796,22 +812,22 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstrea
} }
#[tokio::test] #[tokio::test]
async fn local_admin_video_task_video_is_unavailable_after_openai_url_sanitization() { async fn local_admin_video_task_download_preserves_signed_url_and_attaches_audit() {
let repository = Arc::new(InMemoryVideoTaskRepository::default()); let repository = Arc::new(InMemoryVideoTaskRepository::default());
let stored = repository let mut task = sample_admin_video_task(
.upsert(sample_admin_video_task( "task-video-audit",
"task-video-audit", VideoTaskStatus::Completed,
VideoTaskStatus::Completed, 1_710_000_550,
1_710_000_550, "user-5",
"user-5", "frank",
"frank", "provider-openai",
"provider-openai", "gpt-video",
"gpt-video", "video audit prompt",
"video audit prompt", );
)) task.video_url =
.await Some("https://8.8.8.8/video.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1".to_string());
.expect("task should upsert"); let stored = repository.upsert(task).await.expect("task should upsert");
assert_eq!(stored.video_url, None); assert_eq!(stored.prompt.as_deref(), Some("video audit prompt"));
let state = AppState::new() let state = AppState::new()
.expect("gateway state should build") .expect("gateway state should build")
@@ -825,8 +841,15 @@ async fn local_admin_video_task_video_is_unavailable_after_openai_url_sanitizati
) )
.await; .await;
assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT);
assert!(response.extensions().get::<AdminAuditEvent>().is_none()); assert_eq!(
response
.headers()
.get(http::header::LOCATION)
.and_then(|value| value.to_str().ok()),
stored.video_url.as_deref()
);
assert!(response.extensions().get::<AdminAuditEvent>().is_some());
} }
#[tokio::test] #[tokio::test]
+240 -42
View File
@@ -12,6 +12,7 @@ use super::{
UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER, UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
}; };
use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER; use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER;
use aether_data_contracts::repository::usage::UsageBodyCaptureState;
fn deep_nested_metadata(levels: usize) -> serde_json::Value { fn deep_nested_metadata(levels: usize) -> serde_json::Value {
let mut current = json!({"leaf": "value"}); let mut current = json!({"leaf": "value"});
@@ -84,6 +85,58 @@ where
stored.expect("usage should be present once the expected status is observed") stored.expect("usage should be present once the expected status is observed")
} }
async fn load_admin_usage_capture_detail(
state: &crate::AppState,
usage_id: &str,
include_bodies: bool,
) -> serde_json::Value {
use crate::admin_api::{maybe_build_local_admin_response, AdminRouteRequest};
use crate::constants::{
GATEWAY_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER, TRUSTED_ADMIN_USER_ID_HEADER,
TRUSTED_ADMIN_USER_ROLE_HEADER,
};
use crate::control::resolve_public_request_context;
use http_body_util::BodyExt;
let mut headers = http::HeaderMap::new();
for (name, value) in [
(GATEWAY_HEADER, "rust-phase3b"),
(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user"),
(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin"),
(TRUSTED_ADMIN_SESSION_ID_HEADER, "admin-session"),
] {
headers.insert(name, HeaderValue::from_static(value));
}
let uri = format!("/api/admin/usage/{usage_id}?include_bodies={include_bodies}")
.parse()
.unwrap();
let context = resolve_public_request_context(
state,
&http::Method::GET,
&uri,
&headers,
"usage-full-detail",
)
.await
.unwrap();
let response = maybe_build_local_admin_response(AdminRouteRequest::new(
state,
&context,
&"127.0.0.1:12345".parse().unwrap(),
&headers,
None,
))
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert!(response
.extensions()
.get::<crate::audit::AdminAuditEvent>()
.is_some());
serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap()
}
#[test] #[test]
fn gateway_handles_local_openai_chat_sync_report_with_local_reporting_when_usage_runtime_enabled() { fn gateway_handles_local_openai_chat_sync_report_with_local_reporting_when_usage_runtime_enabled() {
run_async_test_on_large_stack( run_async_test_on_large_stack(
@@ -348,7 +401,7 @@ async fn gateway_truncates_deep_request_echo_for_local_openai_chat_sync_usage_im
Arc::clone(&request_candidate_repository), Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository), Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY, DEVELOPMENT_ENCRYPTION_KEY,
), ).with_system_config_values_for_tests([("request_record_level".to_string(), json!("full"))]),
) )
.with_usage_runtime_for_tests(UsageRuntimeConfig { .with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true, enabled: true,
@@ -402,10 +455,28 @@ async fn gateway_truncates_deep_request_echo_for_local_openai_chat_sync_usage_im
let stored_usage = stored_usage.expect("usage should be recorded"); let stored_usage = stored_usage.expect("usage should be recorded");
assert_eq!(stored_usage.status, "completed"); assert_eq!(stored_usage.status, "completed");
assert_eq!(stored_usage.total_tokens, 5); assert_eq!(stored_usage.total_tokens, 5);
assert!(stored_usage.request_body.is_none()); let request_body = stored_usage.request_body.as_ref().unwrap();
assert_eq!(
request_body["messages"][0]["content"]
.as_str()
.unwrap()
.len(),
128 * 1024
);
assert!(
request_body["metadata"]["child"]["child"]["child"]["child"]["child"]
.get("depth")
.is_some()
);
assert!(stored_usage.request_body_ref.is_none()); assert!(stored_usage.request_body_ref.is_none());
assert!(stored_usage.request_body_state.is_none()); assert_eq!(
assert!(stored_usage.request_headers.is_none()); stored_usage.request_body_state,
Some(UsageBodyCaptureState::Inline)
);
assert_eq!(
stored_usage.request_headers.as_ref().unwrap()["authorization"],
"[redacted]"
);
gateway_handle.abort(); gateway_handle.abort();
execution_runtime_handle.abort(); execution_runtime_handle.abort();
@@ -489,10 +560,10 @@ async fn gateway_ignores_legacy_max_request_body_size_for_local_openai_chat_sync
Arc::clone(&usage_repository), Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY, DEVELOPMENT_ENCRYPTION_KEY,
) )
.with_system_config_values_for_tests([( .with_system_config_values_for_tests([
"max_request_body_size".to_string(), ("max_request_body_size".to_string(), json!(128)),
json!(128), ("request_record_level".to_string(), json!("full")),
)]), ]),
) )
.with_usage_runtime_for_tests(UsageRuntimeConfig { .with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true, enabled: true,
@@ -535,12 +606,30 @@ async fn gateway_ignores_legacy_max_request_body_size_for_local_openai_chat_sync
) )
.await; .await;
assert_eq!(stored_usage.total_tokens, 5); assert_eq!(stored_usage.total_tokens, 5);
assert!(stored_usage.request_body.is_none()); assert!(
stored_usage.request_body.as_ref().unwrap()["messages"][0]["content"]
.as_str()
.unwrap()
.len()
> 128
);
assert!(stored_usage.request_body_ref.is_none()); assert!(stored_usage.request_body_ref.is_none());
assert!(stored_usage.request_body_state.is_none()); assert_eq!(
assert!(stored_usage.provider_request_body.is_none()); stored_usage.request_body_state,
Some(UsageBodyCaptureState::Inline)
);
assert!(
stored_usage.provider_request_body.as_ref().unwrap()["messages"][0]["content"]
.as_str()
.unwrap()
.len()
> 128
);
assert!(stored_usage.provider_request_body_ref.is_none()); assert!(stored_usage.provider_request_body_ref.is_none());
assert!(stored_usage.provider_request_body_state.is_none()); assert_eq!(
stored_usage.provider_request_body_state,
Some(UsageBodyCaptureState::Inline)
);
gateway_handle.abort(); gateway_handle.abort();
execution_runtime_handle.abort(); execution_runtime_handle.abort();
@@ -551,11 +640,19 @@ async fn gateway_ignores_legacy_max_request_body_size_for_local_openai_chat_sync
fn gateway_strips_request_and_response_bodies_when_request_record_level_is_base() { fn gateway_strips_request_and_response_bodies_when_request_record_level_is_base() {
run_async_test_on_large_stack( run_async_test_on_large_stack(
"gateway_strips_request_and_response_bodies_when_request_record_level_is_base", "gateway_strips_request_and_response_bodies_when_request_record_level_is_base",
gateway_strips_request_and_response_bodies_when_request_record_level_is_base_impl(), gateway_honors_request_record_level_impl("base"),
); );
} }
async fn gateway_strips_request_and_response_bodies_when_request_record_level_is_base_impl() { #[test]
fn gateway_full_request_record_level_preserves_sync_bodies_in_admin_detail() {
run_async_test_on_large_stack(
"gateway_full_request_record_level_preserves_sync_bodies_in_admin_detail",
gateway_honors_request_record_level_impl("full"),
);
}
async fn gateway_honors_request_record_level_impl(record_level: &str) {
let usage_repository = Arc::new(InMemoryUsageReadRepository::default()); let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default()); let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
@@ -630,14 +727,14 @@ async fn gateway_strips_request_and_response_bodies_when_request_record_level_is
) )
.with_system_config_values_for_tests([( .with_system_config_values_for_tests([(
"request_record_level".to_string(), "request_record_level".to_string(),
json!("base"), json!(record_level),
)]), )]),
) )
.with_usage_runtime_for_tests(UsageRuntimeConfig { .with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true, enabled: true,
..UsageRuntimeConfig::default() ..UsageRuntimeConfig::default()
}); });
let gateway = build_router_with_state(gateway_state); let gateway = build_router_with_state(gateway_state.clone());
let (gateway_url, gateway_handle) = start_server(gateway).await; let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new() let response = reqwest::Client::new()
@@ -678,14 +775,39 @@ async fn gateway_strips_request_and_response_bodies_when_request_record_level_is
assert_eq!(stored_usage.status, "completed"); assert_eq!(stored_usage.status, "completed");
assert_eq!(stored_usage.total_tokens, 5); assert_eq!(stored_usage.total_tokens, 5);
assert_eq!(stored_usage.response_time_ms, Some(25)); assert_eq!(stored_usage.response_time_ms, Some(25));
assert!(stored_usage.request_body.is_none()); let detail = load_admin_usage_capture_detail(&gateway_state, &stored_usage.id, true).await;
assert!(stored_usage.request_body_ref.is_none()); let shallow = load_admin_usage_capture_detail(&gateway_state, &stored_usage.id, false).await;
assert!(stored_usage.provider_request_body.is_none()); for field in [
assert!(stored_usage.provider_request_body_ref.is_none()); "request_body",
assert!(stored_usage.response_body.is_none()); "provider_request_body",
assert!(stored_usage.response_body_ref.is_none()); "response_body",
assert!(stored_usage.client_response_body.is_none()); "client_response_body",
assert!(stored_usage.client_response_body_ref.is_none()); ] {
assert!(shallow[field].is_null());
let expected_captured = record_level == "full" && field != "client_response_body";
assert_eq!(
shallow[format!("has_{field}")],
expected_captured,
"availability for {field}"
);
if expected_captured {
assert!(!detail[field].is_null(), "full should expose {field}");
} else {
assert!(
detail[field].is_null(),
"uncaptured {field} must remain absent"
);
}
}
if record_level == "full" {
assert_eq!(
detail["request_body"]["messages"][0]["content"],
"request body should not be persisted"
);
assert_eq!(detail["provider_request_body"]["model"], "gpt-5-upstream");
assert_eq!(detail["response_body"], body_json);
assert!(detail["client_response_body"].is_null());
}
let stored_candidates = request_candidate_repository let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-report-sync-base-123") .list_by_request_id("trace-openai-chat-local-report-sync-base-123")
@@ -825,10 +947,16 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
); );
assert!(stored_usage.response_body.is_none()); assert!(stored_usage.response_body.is_none());
assert!(stored_usage.response_body_ref.is_none()); assert!(stored_usage.response_body_ref.is_none());
assert!(stored_usage.response_body_state.is_none()); assert_eq!(
stored_usage.response_body_state,
Some(UsageBodyCaptureState::Disabled)
);
assert!(stored_usage.client_response_body.is_none()); assert!(stored_usage.client_response_body.is_none());
assert!(stored_usage.client_response_body_ref.is_none()); assert!(stored_usage.client_response_body_ref.is_none());
assert!(stored_usage.client_response_body_state.is_none()); assert_eq!(
stored_usage.client_response_body_state,
Some(UsageBodyCaptureState::Disabled)
);
let stored_candidates = request_candidate_repository let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-report-sync-failure-123") .list_by_request_id("trace-openai-chat-local-report-sync-failure-123")
@@ -930,7 +1058,10 @@ async fn gateway_records_failed_usage_when_sync_runtime_transport_is_unavailable
assert_eq!(stored_usage.status_code, Some(503)); assert_eq!(stored_usage.status_code, Some(503));
assert!(stored_usage.response_body.is_none()); assert!(stored_usage.response_body.is_none());
assert!(stored_usage.response_body_ref.is_none()); assert!(stored_usage.response_body_ref.is_none());
assert!(stored_usage.response_body_state.is_none()); assert_eq!(
stored_usage.response_body_state,
Some(UsageBodyCaptureState::Disabled)
);
let stored_candidates = request_candidate_repository let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-transport-unavailable-123") .list_by_request_id("trace-openai-chat-local-transport-unavailable-123")
@@ -1272,7 +1403,10 @@ async fn gateway_records_failed_usage_for_claude_runtime_miss_without_execution_
); );
assert!(stored_usage.client_response_body.is_none()); assert!(stored_usage.client_response_body.is_none());
assert!(stored_usage.client_response_body_ref.is_none()); assert!(stored_usage.client_response_body_ref.is_none());
assert!(stored_usage.client_response_body_state.is_none()); assert_eq!(
stored_usage.client_response_body_state,
Some(UsageBodyCaptureState::Disabled)
);
assert!(stored_usage.error_message.is_none()); assert!(stored_usage.error_message.is_none());
let stored_candidates = request_candidate_repository let stored_candidates = request_candidate_repository
@@ -1296,11 +1430,20 @@ fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usa
{ {
run_async_test_on_large_stack( run_async_test_on_large_stack(
"gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled", "gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled",
gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl(), gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl("basic"),
);
}
#[test]
fn gateway_full_request_record_level_preserves_stream_bodies_in_admin_detail() {
run_async_test_on_large_stack(
"gateway_full_request_record_level_preserves_stream_bodies_in_admin_detail",
gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl("full"),
); );
} }
async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl( async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl(
record_level: &str,
) { ) {
let usage_repository = Arc::new(InMemoryUsageReadRepository::default()); let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default()); let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
@@ -1406,13 +1549,13 @@ async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_wh
Arc::clone(&request_candidate_repository), Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository), Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY, DEVELOPMENT_ENCRYPTION_KEY,
), ).with_system_config_values_for_tests([("request_record_level".to_string(), json!(record_level))]),
) )
.with_usage_runtime_for_tests(UsageRuntimeConfig { .with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true, enabled: true,
..UsageRuntimeConfig::default() ..UsageRuntimeConfig::default()
}); });
let gateway = build_router_with_state(gateway_state); let gateway = build_router_with_state(gateway_state.clone());
let (gateway_url, gateway_handle) = start_server(gateway).await; let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new() let response = reqwest::Client::new()
@@ -1448,6 +1591,30 @@ async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_wh
assert!(stored_usage.response_time_ms >= stored_usage.first_byte_time_ms); assert!(stored_usage.response_time_ms >= stored_usage.first_byte_time_ms);
assert!(stored_usage.is_stream); assert!(stored_usage.is_stream);
let detail = load_admin_usage_capture_detail(&gateway_state, &stored_usage.id, true).await;
for field in [
"request_body",
"provider_request_body",
"response_body",
"client_response_body",
] {
if record_level == "full" {
assert!(
!detail[field].is_null(),
"full stream should expose {field}"
);
} else {
assert!(
detail[field].is_null(),
"basic stream must not persist {field}"
);
}
}
if record_level == "full" {
assert!(detail["response_body"].to_string().contains("hello"));
assert!(detail["client_response_body"].to_string().contains("hello"));
}
let stored_candidates = request_candidate_repository let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-report-stream-123") .list_by_request_id("trace-openai-chat-local-report-stream-123")
.await .await
@@ -1585,10 +1752,10 @@ async fn gateway_ignores_legacy_max_response_body_size_for_stream_usage_impl() {
Arc::clone(&usage_repository), Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY, DEVELOPMENT_ENCRYPTION_KEY,
) )
.with_system_config_values_for_tests([( .with_system_config_values_for_tests([
"max_response_body_size".to_string(), ("max_response_body_size".to_string(), json!(128)),
json!(128), ("request_record_level".to_string(), json!("full")),
)]), ]),
) )
.with_usage_runtime_for_tests(UsageRuntimeConfig { .with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true, enabled: true,
@@ -1624,12 +1791,34 @@ async fn gateway_ignores_legacy_max_response_body_size_for_stream_usage_impl() {
) )
.await; .await;
assert_eq!(stored_usage.total_tokens, 6); assert_eq!(stored_usage.total_tokens, 6);
assert!(stored_usage.response_body.is_none()); assert!(
stored_usage
.response_body
.as_ref()
.unwrap()
.to_string()
.len()
> 128
);
assert!(stored_usage.response_body_ref.is_none()); assert!(stored_usage.response_body_ref.is_none());
assert!(stored_usage.response_body_state.is_none()); assert_eq!(
assert!(stored_usage.client_response_body.is_none()); stored_usage.response_body_state,
Some(UsageBodyCaptureState::Inline)
);
assert!(
stored_usage
.client_response_body
.as_ref()
.unwrap()
.to_string()
.len()
> 128
);
assert!(stored_usage.client_response_body_ref.is_none()); assert!(stored_usage.client_response_body_ref.is_none());
assert!(stored_usage.client_response_body_state.is_none()); assert_eq!(
stored_usage.client_response_body_state,
Some(UsageBodyCaptureState::Inline)
);
gateway_handle.abort(); gateway_handle.abort();
execution_runtime_handle.abort(); execution_runtime_handle.abort();
@@ -1903,10 +2092,16 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
Some("all_candidates_skipped") Some("all_candidates_skipped")
); );
assert!(stored_usage.error_message.is_none()); assert!(stored_usage.error_message.is_none());
assert!(stored_usage.request_headers.is_none()); assert_eq!(
stored_usage.request_headers.as_ref().unwrap()["authorization"],
"[redacted]"
);
assert!(stored_usage.request_body.is_none()); assert!(stored_usage.request_body.is_none());
assert!(stored_usage.request_body_ref.is_none()); assert!(stored_usage.request_body_ref.is_none());
assert!(stored_usage.request_body_state.is_none()); assert_eq!(
stored_usage.request_body_state,
Some(UsageBodyCaptureState::Disabled)
);
assert!(stored_usage.provider_request_body.is_none()); assert!(stored_usage.provider_request_body.is_none());
assert_eq!( assert_eq!(
stored_usage stored_usage
@@ -2169,7 +2364,10 @@ fn gateway_keeps_failed_usage_request_capture_lightweight_for_large_local_claude
) )
.await; .await;
assert_eq!(stored_usage.status, "failed"); assert_eq!(stored_usage.status, "failed");
assert!(stored_usage.request_body_state.is_none()); assert_eq!(
stored_usage.request_body_state,
Some(UsageBodyCaptureState::Disabled)
);
assert!(stored_usage.request_body.is_none()); assert!(stored_usage.request_body.is_none());
assert!(stored_usage.request_body_ref.is_none()); assert!(stored_usage.request_body_ref.is_none());
assert!(stored_usage.provider_request_body.is_none()); assert!(stored_usage.provider_request_body.is_none());
@@ -213,6 +213,13 @@ async fn gateway_background_video_task_poller_refreshes_due_openai_task_from_rep
}; };
assert_eq!(stored.status, VideoTaskStatus::Processing); assert_eq!(stored.status, VideoTaskStatus::Processing);
assert_eq!(stored.prompt.as_deref(), Some("hello"));
assert_eq!(stored.username.as_deref(), Some("video-user"));
assert_eq!(stored.api_key_name.as_deref(), Some("video-key"));
assert_eq!(stored.duration_seconds, Some(4));
assert_eq!(stored.resolution.as_deref(), Some("720p"));
assert_eq!(stored.aspect_ratio.as_deref(), Some("16:9"));
assert_eq!(stored.size.as_deref(), Some("1280x720"));
assert_eq!(stored.progress_percent, 37); assert_eq!(stored.progress_percent, 37);
assert_eq!(stored.poll_count, 1); assert_eq!(stored.poll_count, 1);
assert!( assert!(
@@ -32,6 +32,7 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
struct SeenExecutionRuntimeStreamRequest { struct SeenExecutionRuntimeStreamRequest {
method: String, method: String,
url: String, url: String,
headers: serde_json::Value,
} }
fn hash_api_key(value: &str) -> String { fn hash_api_key(value: &str) -> String {
@@ -159,6 +160,7 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
.and_then(|value| value.as_str()) .and_then(|value| value.as_str())
.unwrap_or_default() .unwrap_or_default()
.to_string(), .to_string(),
headers: payload.get("headers").cloned().unwrap_or_else(|| json!({})),
}); });
let frames = [ let frames = [
@@ -252,7 +254,10 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
updated_at_unix_secs: 456, updated_at_unix_secs: 456,
error_code: None, error_code: None,
error_message: None, error_message: None,
video_url: Some("https://cdn.example.com/video-content.mp4".to_string()), video_url: Some(
"https://cdn.example.com/video-content.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1"
.to_string(),
),
request_metadata: None, request_metadata: None,
}) })
.await .await
@@ -358,8 +363,9 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
assert_eq!(seen_stream_request.method, "GET"); assert_eq!(seen_stream_request.method, "GET");
assert_eq!( assert_eq!(
seen_stream_request.url, seen_stream_request.url,
"https://api.openai.example/v1/videos/ext-video-content-followup-123/content" "https://cdn.example.com/video-content.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1"
); );
assert!(seen_stream_request.headers.get("authorization").is_none());
assert_eq!(*decision_stream_hits.lock().expect("mutex should lock"), 0); assert_eq!(*decision_stream_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*execute_stream_hits.lock().expect("mutex should lock"), 0); assert_eq!(*execute_stream_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0); assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
+68 -1
View File
@@ -10,7 +10,8 @@ use super::redaction::{
admin_restore_secret_safe_body_rules, admin_restore_secret_safe_header_rules, admin_restore_secret_safe_body_rules, admin_restore_secret_safe_header_rules,
admin_restore_secret_safe_json, admin_restore_secret_safe_proxy, admin_restore_secret_safe_url, admin_restore_secret_safe_json, admin_restore_secret_safe_proxy, admin_restore_secret_safe_url,
admin_secret_safe_body_rules, admin_secret_safe_header_rules, admin_secret_safe_json, admin_secret_safe_body_rules, admin_secret_safe_header_rules, admin_secret_safe_json,
admin_secret_safe_proxy, admin_secret_safe_url, admin_secret_safe_proxy, admin_secret_safe_url, admin_validate_retained_body_rule_secrets,
admin_validate_retained_header_rule_secrets,
}; };
pub fn normalize_endpoint_api_format(api_format: &str) -> String { pub fn normalize_endpoint_api_format(api_format: &str) -> String {
@@ -200,6 +201,55 @@ mod endpoint_key_count_tests {
assert_eq!(active, total); assert_eq!(active, total);
} }
#[test]
fn endpoint_updates_reject_unresolved_rule_masks_before_persistence() {
let header_rules = json!([{"action": "set", "key": "x-auth", "value": "header-secret"}]);
let body_rules = json!([{"action": "set", "path": "auth.token", "value": "body-secret"}]);
let mut endpoint = sample_endpoint("chat", "openai:chat");
endpoint.header_rules = Some(header_rules.clone());
endpoint.body_rules = Some(body_rules.clone());
endpoint.config = Some(json!({"response_header_rules": header_rules}));
let moved_header =
json!([{"action": "set", "key": "x-other-auth", "value": "***", "has_value": true}]);
let cases = [
(
"header_rules",
super::AdminProviderEndpointUpdateFields {
header_rules: Some(moved_header.clone()),
..Default::default()
},
),
(
"body_rules",
super::AdminProviderEndpointUpdateFields {
body_rules: Some(
json!([{"action": "set", "path": "auth.api_key", "value": "***", "has_value": true}]),
),
..Default::default()
},
),
(
"config",
super::AdminProviderEndpointUpdateFields {
config: Some(json!({"response_header_rules": moved_header})),
..Default::default()
},
),
];
for (field, payload) in cases {
let error = super::apply_admin_provider_endpoint_update_fields(
&endpoint,
|key| key == field,
|_| false,
&payload,
)
.expect_err("unresolved masks must not overwrite saved secrets");
assert!(error.contains("无法匹配"));
assert!(!error.contains("header-secret"));
assert!(!error.contains("body-secret"));
}
}
#[test] #[test]
fn inherited_endpoint_counts_only_include_active_formats() { fn inherited_endpoint_counts_only_include_active_formats() {
let responses_endpoint = sample_endpoint("responses", "openai:responses"); let responses_endpoint = sample_endpoint("responses", "openai:responses");
@@ -352,6 +402,10 @@ where
if !header_rules.is_array() { if !header_rules.is_array() {
return Err("header_rules 必须是数组或 null".to_string()); return Err("header_rules 必须是数组或 null".to_string());
} }
admin_validate_retained_header_rule_secrets(
existing_endpoint.header_rules.as_ref(),
header_rules,
)?;
Some(admin_restore_secret_safe_header_rules( Some(admin_restore_secret_safe_header_rules(
existing_endpoint.header_rules.as_ref(), existing_endpoint.header_rules.as_ref(),
header_rules, header_rules,
@@ -369,6 +423,10 @@ where
if !body_rules.is_array() { if !body_rules.is_array() {
return Err("body_rules 必须是数组或 null".to_string()); return Err("body_rules 必须是数组或 null".to_string());
} }
admin_validate_retained_body_rule_secrets(
existing_endpoint.body_rules.as_ref(),
body_rules,
)?;
Some(admin_restore_secret_safe_body_rules( Some(admin_restore_secret_safe_body_rules(
existing_endpoint.body_rules.as_ref(), existing_endpoint.body_rules.as_ref(),
body_rules, body_rules,
@@ -407,6 +465,15 @@ where
if !config.is_object() { if !config.is_object() {
return Err("config 必须是对象或 null".to_string()); return Err("config 必须是对象或 null".to_string());
} }
if let Some(rules) = config.get("response_header_rules") {
admin_validate_retained_header_rule_secrets(
existing_endpoint
.config
.as_ref()
.and_then(|config| config.get("response_header_rules")),
rules,
)?;
}
Some(admin_restore_secret_safe_json( Some(admin_restore_secret_safe_json(
existing_endpoint.config.as_ref(), existing_endpoint.config.as_ref(),
config, config,
+454 -69
View File
@@ -1,5 +1,4 @@
use serde_json::{json, Map, Value}; use serde_json::{json, Map, Value};
use std::collections::BTreeMap;
const REDACTED_VALUE: &str = "***"; const REDACTED_VALUE: &str = "***";
const REDACTED_UPSTREAM_DIAGNOSTIC: &str = "[REDACTED upstream diagnostic]"; const REDACTED_UPSTREAM_DIAGNOSTIC: &str = "[REDACTED upstream diagnostic]";
@@ -190,6 +189,20 @@ pub fn admin_restore_secret_safe_body_rules(existing: Option<&Value>, incoming:
restore_rule_array(existing, incoming, RuleKind::Body) restore_rule_array(existing, incoming, RuleKind::Body)
} }
pub fn admin_validate_retained_header_rule_secrets(
existing: Option<&Value>,
incoming: &Value,
) -> Result<(), String> {
validate_retained_rule_secrets(existing, incoming, RuleKind::Header)
}
pub fn admin_validate_retained_body_rule_secrets(
existing: Option<&Value>,
incoming: &Value,
) -> Result<(), String> {
validate_retained_rule_secrets(existing, incoming, RuleKind::Body)
}
pub fn admin_secret_safe_url(value: Option<&str>) -> Value { pub fn admin_secret_safe_url(value: Option<&str>) -> Value {
value value
.and_then(sanitize_network_url) .and_then(sanitize_network_url)
@@ -1234,11 +1247,7 @@ fn redact_proxy_json_value_for_key(key: &str, value: &Value) -> Value {
return redact_proxy_secret_value(value); return redact_proxy_secret_value(value);
} }
if json_url_key(&compact_key) { if json_url_key(&compact_key) {
return value return redact_json_url_value(value);
.as_str()
.and_then(sanitize_network_url)
.map(Value::String)
.unwrap_or(Value::Null);
} }
if compact_key == "proxy" { if compact_key == "proxy" {
return admin_secret_safe_proxy(Some(value)); return admin_secret_safe_proxy(Some(value));
@@ -1261,11 +1270,7 @@ fn redact_json_value_for_key(key: &str, value: &Value) -> Value {
return redact_secret_value(value); return redact_secret_value(value);
} }
if json_url_key(&compact_key) { if json_url_key(&compact_key) {
return value return redact_json_url_value(value);
.as_str()
.and_then(sanitize_network_url)
.map(Value::String)
.unwrap_or(Value::Null);
} }
if compact_key == "proxy" { if compact_key == "proxy" {
return admin_secret_safe_proxy(Some(value)); return admin_secret_safe_proxy(Some(value));
@@ -1282,6 +1287,17 @@ fn redact_json_value_for_key(key: &str, value: &Value) -> Value {
redact_json_value(value) redact_json_value(value)
} }
fn redact_json_url_value(value: &Value) -> Value {
match value {
Value::String(raw) if url::Url::parse(raw).is_ok_and(|url| url.scheme() == "data") => {
value.clone()
}
Value::String(raw) => admin_secret_safe_url(Some(raw)),
Value::Array(values) => Value::Array(values.iter().map(redact_json_url_value).collect()),
_ => redact_json_value(value),
}
}
fn redact_body_rule(rule: &Value) -> Value { fn redact_body_rule(rule: &Value) -> Value {
let Some(rule) = rule.as_object() else { let Some(rule) = rule.as_object() else {
return redact_json_value(rule); return redact_json_value(rule);
@@ -1320,7 +1336,12 @@ fn redact_header_rule(rule: &Value) -> Value {
.map(|(key, value)| (key.clone(), redact_json_value_for_key(key, value))) .map(|(key, value)| (key.clone(), redact_json_value_for_key(key, value)))
.collect::<Map<_, _>>(); .collect::<Map<_, _>>();
let is_set = normalized_string_field(rule, "action").as_deref() == Some("set"); let is_set = normalized_string_field(rule, "action").as_deref() == Some("set");
if is_set { if is_set
&& rule
.get("key")
.and_then(Value::as_str)
.is_none_or(header_value_is_secret)
{
redact_rule_secret_field(rule, &mut projected, "value", "has_value"); redact_rule_secret_field(rule, &mut projected, "value", "has_value");
} }
if let Some(condition) = rule.get("condition") { if let Some(condition) = rule.get("condition") {
@@ -1374,7 +1395,14 @@ fn redact_header_values(value: &Value) -> Value {
Value::Object( Value::Object(
headers headers
.iter() .iter()
.map(|(key, value)| (key.clone(), redact_secret_value(value))) .map(|(key, value)| {
let value = if header_value_is_secret(key) {
redact_secret_value(value)
} else {
redact_json_value(value)
};
(key.clone(), value)
})
.collect(), .collect(),
) )
} }
@@ -1395,10 +1423,25 @@ fn restore_json_value(existing: Option<&Value>, incoming: &Value, key: Option<&s
return restore_header_values(existing, incoming); return restore_header_values(existing, incoming);
} }
if json_url_key(&compact_key) { if json_url_key(&compact_key) {
return incoming if let Some(incoming_url) = incoming.as_str() {
.as_str() return restore_url_value(existing, incoming_url);
.map(|incoming_url| restore_url_value(existing, incoming_url)) }
.unwrap_or_else(|| incoming.clone()); if let Some(values) = incoming.as_array() {
let existing_values = existing.and_then(Value::as_array);
return Value::Array(
values
.iter()
.enumerate()
.map(|(index, value)| {
restore_json_value(
existing_values.and_then(|values| values.get(index)),
value,
Some(key),
)
})
.collect(),
);
}
} }
if json_secret_key(&compact_key, incoming) { if json_secret_key(&compact_key, incoming) {
return restore_masked_secret(existing, incoming, true); return restore_masked_secret(existing, incoming, true);
@@ -1448,29 +1491,20 @@ fn restore_rule_array(existing: Option<&Value>, incoming: &Value, kind: RuleKind
let Some(incoming_values) = incoming.as_array() else { let Some(incoming_values) = incoming.as_array() else {
return incoming.clone(); return incoming.clone();
}; };
let existing_values = existing.and_then(Value::as_array); if unchanged_projected_rules(existing, incoming, kind) {
let incoming_identities = identity_counts(incoming_values, kind); return existing.cloned().unwrap_or_else(|| incoming.clone());
let existing_identities = existing_values }
.map(|values| identity_counts(values, kind)) let existing_values = existing
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default(); .unwrap_or_default();
Value::Array( Value::Array(
incoming_values incoming_values
.iter() .iter()
.map(|incoming_rule| { .map(|incoming_rule| {
let identity = rule_identity(incoming_rule, kind); let existing_rule =
let existing_rule = identity.as_ref().and_then(|identity| { match_existing_rule(existing_values, incoming_values, incoming_rule, kind);
(incoming_identities.get(identity) == Some(&1)
&& existing_identities.get(identity) == Some(&1))
.then(|| {
existing_values.and_then(|values| {
values.iter().find(|candidate| {
rule_identity(candidate, kind).as_ref() == Some(identity)
})
})
})
.flatten()
});
restore_rule(existing_rule, incoming_rule, kind) restore_rule(existing_rule, incoming_rule, kind)
}) })
.collect(), .collect(),
@@ -1546,7 +1580,10 @@ fn restore_condition(existing: Option<&Value>, incoming: &Value) -> Value {
continue; continue;
} }
let existing_value = existing_object.and_then(|object| object.get(key)); let existing_value = existing_object.and_then(|object| object.get(key));
let value = if key == "value" && condition_value_is_secret(incoming_object) { let value = if key == "value"
&& (condition_value_is_secret(incoming_object)
|| incoming_object.get("has_value").and_then(Value::as_bool) == Some(true))
{
let marker_set = let marker_set =
incoming_object.get("has_value").and_then(Value::as_bool) == Some(true); incoming_object.get("has_value").and_then(Value::as_bool) == Some(true);
restore_masked_secret(existing_value, incoming_value, marker_set) restore_masked_secret(existing_value, incoming_value, marker_set)
@@ -1562,29 +1599,25 @@ fn restore_condition_array(existing: Option<&Value>, incoming: &Value) -> Value
let Some(incoming_values) = incoming.as_array() else { let Some(incoming_values) = incoming.as_array() else {
return incoming.clone(); return incoming.clone();
}; };
let existing_values = existing.and_then(Value::as_array); let existing_values = existing
let incoming_counts = condition_identity_counts(incoming_values); .and_then(Value::as_array)
let existing_counts = existing_values .map(Vec::as_slice)
.map(|values| condition_identity_counts(values))
.unwrap_or_default(); .unwrap_or_default();
if projected_conditions_unchanged(existing_values, incoming_values) {
return Value::Array(existing_values.to_vec());
}
Value::Array( Value::Array(
incoming_values incoming_values
.iter() .iter()
.map(|incoming_condition| { .map(|incoming_condition| {
let identity = condition_identity(incoming_condition); let existing_condition = match_existing_entry(
let existing_condition = identity.as_ref().and_then(|identity| { existing_values,
(incoming_counts.get(identity) == Some(&1) incoming_values,
&& existing_counts.get(identity) == Some(&1)) incoming_condition,
.then(|| { condition_identity,
existing_values.and_then(|values| { redact_condition,
values.iter().find(|candidate| { );
condition_identity(candidate).as_ref() == Some(identity)
})
})
})
.flatten()
});
restore_condition(existing_condition, incoming_condition) restore_condition(existing_condition, incoming_condition)
}) })
.collect(), .collect(),
@@ -1633,20 +1666,174 @@ fn restore_url_value(existing: Option<&Value>, incoming_url: &str) -> Value {
Value::String(incoming_url.to_string()) Value::String(incoming_url.to_string())
} }
fn identity_counts(values: &[Value], kind: RuleKind) -> BTreeMap<String, usize> { fn project_rule(rule: &Value, kind: RuleKind) -> Value {
let mut counts = BTreeMap::new(); match kind {
for identity in values.iter().filter_map(|value| rule_identity(value, kind)) { RuleKind::Header => redact_header_rule(rule),
*counts.entry(identity).or_insert(0) += 1; RuleKind::Body => redact_body_rule(rule),
} }
counts
} }
fn condition_identity_counts(values: &[Value]) -> BTreeMap<String, usize> { fn unchanged_projected_rules(existing: Option<&Value>, incoming: &Value, kind: RuleKind) -> bool {
let mut counts = BTreeMap::new(); existing.and_then(Value::as_array).is_some_and(|rules| {
for identity in values.iter().filter_map(condition_identity) { Value::Array(rules.iter().map(|rule| project_rule(rule, kind)).collect()) == *incoming
*counts.entry(identity).or_insert(0) += 1; })
}
fn rule_match_shape(rule: &Value, kind: RuleKind) -> Value {
let mut projected = project_rule(rule, kind);
if let Some(object) = projected.as_object_mut() {
for field in [
"enabled",
"value",
"has_value",
"pattern",
"has_pattern",
"replacement",
"has_replacement",
] {
object.remove(field);
}
} }
counts projected
}
fn match_existing_rule<'a>(
existing: &'a [Value],
incoming: &[Value],
rule: &Value,
kind: RuleKind,
) -> Option<&'a Value> {
match_existing_entry(
existing,
incoming,
rule,
|value| rule_identity(value, kind),
|value| rule_match_shape(value, kind),
)
}
fn match_existing_entry<'a>(
existing: &'a [Value],
incoming: &[Value],
entry: &Value,
identity: impl Fn(&Value) -> Option<String>,
project: impl Fn(&Value) -> Value,
) -> Option<&'a Value> {
let entry_identity = identity(entry)?;
let same_identity = |value: &&Value| identity(value).as_ref() == Some(&entry_identity);
let candidates = existing.iter().filter(same_identity).collect::<Vec<_>>();
if candidates.len() == 1 && incoming.iter().filter(same_identity).count() == 1 {
return candidates.first().copied();
}
let projected = project(entry);
let mut matching = candidates
.into_iter()
.filter(|candidate| project(candidate) == projected);
let matched = matching.next()?;
if matching.next().is_some()
|| incoming
.iter()
.filter(same_identity)
.filter(|value| project(value) == projected)
.count()
!= 1
{
return None;
}
Some(matched)
}
fn projected_conditions_unchanged(existing: &[Value], incoming: &[Value]) -> bool {
existing.len() == incoming.len()
&& existing
.iter()
.zip(incoming)
.all(|(existing, incoming)| redact_condition(existing) == *incoming)
}
fn validate_retained_rule_secrets(
existing: Option<&Value>,
incoming: &Value,
kind: RuleKind,
) -> Result<(), String> {
let Some(incoming_values) = incoming.as_array() else {
return Ok(());
};
if unchanged_projected_rules(existing, incoming, kind) {
return Ok(());
}
let existing_values = existing
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
for rule in incoming_values {
let existing_rule = match_existing_rule(existing_values, incoming_values, rule, kind);
validate_retained_secret_fields(existing_rule, rule)?;
if let Some(condition) = rule.get("condition") {
validate_retained_condition_secrets(
existing_rule.and_then(|rule| rule.get("condition")),
condition,
)?;
}
}
Ok(())
}
fn validate_retained_secret_fields(
existing: Option<&Value>,
incoming: &Value,
) -> Result<(), String> {
for (field, marker) in [
("value", "has_value"),
("pattern", "has_pattern"),
("replacement", "has_replacement"),
] {
if incoming.get(marker).and_then(Value::as_bool) == Some(true)
&& incoming.get(field).and_then(Value::as_str) == Some(REDACTED_VALUE)
&& existing
.and_then(|value| value.get(field))
.filter(|value| secret_value_is_set(value))
.is_none()
{
return Err(
"无法匹配脱敏规则的原值,请查看原值后重新填写,避免将占位符保存为实际配置"
.to_string(),
);
}
}
Ok(())
}
fn validate_retained_condition_secrets(
existing: Option<&Value>,
incoming: &Value,
) -> Result<(), String> {
for group_key in ["all", "any"] {
if let Some(children) = incoming.get(group_key).and_then(Value::as_array) {
let existing_children = existing
.and_then(|value| value.get(group_key))
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
if projected_conditions_unchanged(existing_children, children) {
return Ok(());
}
for child in children {
let existing_child = match_existing_entry(
existing_children,
children,
child,
condition_identity,
redact_condition,
);
validate_retained_condition_secrets(existing_child, child)?;
}
return Ok(());
}
}
let existing =
existing.filter(|value| condition_identity(value) == condition_identity(incoming));
validate_retained_secret_fields(existing, incoming)
} }
fn rule_identity(value: &Value, kind: RuleKind) -> Option<String> { fn rule_identity(value: &Value, kind: RuleKind) -> Option<String> {
@@ -1676,8 +1863,18 @@ fn rule_identity(value: &Value, kind: RuleKind) -> Option<String> {
fn condition_identity(value: &Value) -> Option<String> { fn condition_identity(value: &Value) -> Option<String> {
let value = value.as_object()?; let value = value.as_object()?;
if value.contains_key("all") || value.contains_key("any") { for group_key in ["all", "any"] {
return None; if let Some(children) = value.get(group_key).and_then(Value::as_array) {
let mut identities = children
.iter()
.map(condition_identity)
.collect::<Option<Vec<_>>>()?;
identities.sort();
return Some(format!(
"condition:{group_key}:{}",
serde_json::to_string(&identities).ok()?
));
}
} }
let path = trimmed_string_field(value, "path")?; let path = trimmed_string_field(value, "path")?;
let op = normalized_string_field(value, "op")?; let op = normalized_string_field(value, "op")?;
@@ -1720,11 +1917,36 @@ fn is_rule_marker(key: &str) -> bool {
fn condition_value_is_secret(condition: &Map<String, Value>) -> bool { fn condition_value_is_secret(condition: &Map<String, Value>) -> bool {
let source = normalized_condition_source(condition.get("source").and_then(Value::as_str)); let source = normalized_condition_source(condition.get("source").and_then(Value::as_str));
source == "request_headers" let path = condition.get("path").and_then(Value::as_str);
|| condition if source == "request_headers" {
.get("path") path.is_none_or(header_value_is_secret)
.and_then(Value::as_str) } else {
.is_some_and(json_path_targets_secret) path.is_some_and(json_path_targets_secret)
}
}
fn header_value_is_secret(name: &str) -> bool {
!matches!(
name.trim().to_ascii_lowercase().as_str(),
"accept"
| "accept-encoding"
| "accept-language"
| "cache-control"
| "content-encoding"
| "content-type"
| "user-agent"
| "anthropic-version"
| "anthropic-beta"
| "openai-beta"
| "x-stainless-lang"
| "x-stainless-package-version"
| "x-stainless-os"
| "x-stainless-arch"
| "x-stainless-runtime"
| "x-stainless-runtime-version"
| "x-stainless-retry-count"
| "x-stainless-timeout"
)
} }
fn normalized_condition_source(source: Option<&str>) -> String { fn normalized_condition_source(source: Option<&str>) -> String {
@@ -2542,4 +2764,167 @@ mod tests {
assert_eq!(projected, "https://api.example/v1"); assert_eq!(projected, "https://api.example/v1");
assert_eq!(admin_secret_safe_url(Some("not a url")), json!(null)); assert_eq!(admin_secret_safe_url(Some("not a url")), json!(null));
} }
#[test]
fn public_protocol_headers_and_conditions_remain_editable() {
let rules = json!([
{"action": "set", "key": "Content-Type", "value": "application/json"},
{"action": "set", "key": "User-Agent", "value": "client/1.0"},
{"action": "set", "key": "anthropic-version", "value": "2023-06-01"},
{"action": "set", "key": "OpenAI-Beta", "value": "responses=experimental", "condition": {
"source": "request_headers", "path": "Accept", "op": "eq", "value": "text/event-stream"
}}
]);
assert_eq!(admin_secret_safe_header_rules(Some(&rules)), rules);
let mut legacy_projection = rules.clone();
legacy_projection[0]["value"] = json!("***");
legacy_projection[0]["has_value"] = json!(true);
legacy_projection[3]["condition"]["value"] = json!("***");
legacy_projection[3]["condition"]["has_value"] = json!(true);
assert_eq!(
admin_restore_secret_safe_header_rules(Some(&rules), &legacy_projection),
rules
);
}
#[test]
fn header_maps_keep_protocol_values_but_hide_credentials_and_unknown_headers() {
let projected = admin_secret_safe_json(Some(&json!({"headers": {
"Content-Type": "application/json",
"User-Agent": "client/1.0",
"Authorization": "Bearer secret",
"Cookie": "session=secret",
"x-custom-auth": "custom-secret"
}})));
assert_eq!(projected["headers"]["Content-Type"], "application/json");
assert_eq!(projected["headers"]["User-Agent"], "client/1.0");
for header in ["Authorization", "Cookie", "x-custom-auth"] {
assert_eq!(projected["headers"][header], "***");
}
}
#[test]
fn duplicate_conditional_rules_retain_secrets_when_reordered_or_disabled() {
let existing = json!([
{"action": "set", "key": "x-auth", "value": "first-secret", "condition": {
"path": "model", "op": "eq", "value": "first-model"
}},
{"action": "set", "key": "x-auth", "value": "second-secret", "condition": {
"path": "model", "op": "eq", "value": "second-model"
}}
]);
let mut incoming = admin_secret_safe_header_rules(Some(&existing));
incoming.as_array_mut().unwrap().reverse();
incoming[0]["enabled"] = json!(false);
super::admin_validate_retained_header_rule_secrets(Some(&existing), &incoming).unwrap();
let restored = admin_restore_secret_safe_header_rules(Some(&existing), &incoming);
assert_eq!(restored[0]["value"], "second-secret");
assert_eq!(restored[0]["enabled"], false);
assert_eq!(restored[1]["value"], "first-secret");
assert!(restored[0].get("has_value").is_none());
}
#[test]
fn unchanged_duplicate_body_rules_preserve_their_original_values() {
let existing = json!([
{"action": "append", "path": "auth.cookies", "value": "first-secret"},
{"action": "append", "path": "auth.cookies", "value": "second-secret"}
]);
let incoming = admin_secret_safe_body_rules(Some(&existing));
super::admin_validate_retained_body_rule_secrets(Some(&existing), &incoming).unwrap();
assert_eq!(
admin_restore_secret_safe_body_rules(Some(&existing), &incoming),
existing
);
}
#[test]
fn nested_condition_groups_preserve_secrets_after_sibling_edits_and_reordering() {
let existing = json!([{
"action": "set", "key": "x-output", "value": "header-secret",
"condition": {"all": [
{"any": [
{"path": "auth.token", "op": "eq", "value": "condition-secret"},
{"path": "model", "op": "eq", "value": "old-model"}
]},
{"path": "metadata.enabled", "op": "eq", "value": true}
]}
}]);
let mut incoming = admin_secret_safe_header_rules(Some(&existing));
incoming[0]["condition"]["all"][0]["any"][1]["value"] = json!("new-model");
incoming[0]["condition"]["all"]
.as_array_mut()
.unwrap()
.reverse();
super::admin_validate_retained_header_rule_secrets(Some(&existing), &incoming).unwrap();
let restored = admin_restore_secret_safe_header_rules(Some(&existing), &incoming);
assert_eq!(restored[0]["value"], "header-secret");
assert_eq!(
restored[0]["condition"]["all"][1]["any"][0]["value"],
"condition-secret"
);
assert_eq!(
restored[0]["condition"]["all"][1]["any"][1]["value"],
"new-model"
);
assert!(!restored.to_string().contains("has_value"));
}
#[test]
fn retained_masks_cannot_silently_overwrite_changed_or_ambiguous_rules() {
let existing = json!([
{"action": "set", "key": "x-auth", "value": "first-secret"},
{"action": "set", "key": "x-auth", "value": "second-secret"}
]);
let mut incoming = admin_secret_safe_header_rules(Some(&existing));
incoming[0]["enabled"] = json!(false);
assert!(
super::admin_validate_retained_header_rule_secrets(Some(&existing), &incoming).is_err()
);
let existing = json!([{"action": "set", "path": "auth.token", "value": "secret"}]);
let mut incoming = admin_secret_safe_body_rules(Some(&existing));
incoming[0]["path"] = json!("auth.api_key");
assert!(
super::admin_validate_retained_body_rule_secrets(Some(&existing), &incoming).is_err()
);
incoming[0]["value"] = json!("replacement-secret");
incoming[0].as_object_mut().unwrap().remove("has_value");
assert!(
super::admin_validate_retained_body_rule_secrets(Some(&existing), &incoming).is_ok()
);
}
#[test]
fn body_rule_projection_preserves_structured_image_urls_and_inline_data() {
let existing = json!([{
"action": "append", "path": "messages[0].content",
"value": {"type": "image_url", "image_url": {"url": "data:image/png;base64,aW1hZ2U=", "detail": "high"}}
}]);
let projected = admin_secret_safe_body_rules(Some(&existing));
assert_eq!(projected, existing);
assert_eq!(
admin_restore_secret_safe_body_rules(Some(&existing), &projected),
existing
);
}
#[test]
fn structured_url_values_still_hide_and_restore_network_credentials() {
let existing = json!({
"image_url": {"url": "https://user:password@example.test/image?token=secret", "detail": "auto"},
"url": ["https://example.test/file?token=secret", "data:image/png;base64,aW1hZ2U="]
});
let projected = admin_secret_safe_json(Some(&existing));
assert_eq!(projected["image_url"]["url"], "https://example.test/image");
assert_eq!(projected["image_url"]["detail"], "auto");
assert_eq!(projected["url"][0], "https://example.test/file");
assert_eq!(projected["url"][1], existing["url"][1]);
assert!(!projected.to_string().contains("secret"));
assert!(!projected.to_string().contains("password"));
assert_eq!(
admin_restore_secret_safe_json(Some(&existing), &projected),
existing
);
}
} }
File diff suppressed because it is too large Load Diff
@@ -1374,8 +1374,8 @@ WHERE u.request_id = ANY($1)
"#; "#;
const UPSERT_USAGE_ROUTING_SNAPSHOT_SQL: &str = const UPSERT_USAGE_ROUTING_SNAPSHOT_SQL: &str =
include_str!("queries/upsert_usage_routing_snapshot_sql.sql"); include_str!("queries/upsert_usage_routing_snapshot_sql.sql");
#[cfg(test)]
const UPSERT_USAGE_HTTP_AUDIT_SQL: &str = include_str!("queries/upsert_usage_http_audit_sql.sql"); const UPSERT_USAGE_HTTP_AUDIT_SQL: &str = include_str!("queries/upsert_usage_http_audit_sql.sql");
const UPSERT_USAGE_BODY_BLOB_SQL: &str = include_str!("queries/upsert_usage_body_blob_sql.sql");
const UPSERT_USAGE_SETTLEMENT_PRICING_SNAPSHOT_SQL: &str = const UPSERT_USAGE_SETTLEMENT_PRICING_SNAPSHOT_SQL: &str =
include_str!("queries/upsert_usage_settlement_pricing_snapshot_sql.sql"); include_str!("queries/upsert_usage_settlement_pricing_snapshot_sql.sql");
@@ -13362,19 +13362,36 @@ async fn sync_usage_body_blob_storage<'e, E>(
executor: E, executor: E,
request_id: &str, request_id: &str,
field: UsageBodyField, field: UsageBodyField,
_value: Option<&Value>, value: Option<&Value>,
_storage: &UsageBodyStorage, storage: &UsageBodyStorage,
_clear_existing: bool, clear_existing: bool,
) -> Result<(), DataLayerError> ) -> Result<(), DataLayerError>
where where
E: sqlx::Executor<'e, Database = Postgres>, E: sqlx::Executor<'e, Database = Postgres>,
{ {
let body_ref = usage_body_ref(request_id, field); let body_ref = usage_body_ref(request_id, field);
sqlx::query(DELETE_USAGE_BODY_BLOB_SQL) if clear_existing {
.bind(&body_ref) sqlx::query(DELETE_USAGE_BODY_BLOB_SQL)
.execute(executor) .bind(&body_ref)
.await .execute(executor)
.map_postgres_err()?; .await
.map_postgres_err()?;
} else if let Some(payload_gzip) = storage.detached_blob_bytes.as_ref() {
sqlx::query(UPSERT_USAGE_BODY_BLOB_SQL)
.bind(&body_ref)
.bind(request_id)
.bind(field.as_storage_field())
.bind(payload_gzip)
.execute(executor)
.await
.map_postgres_err()?;
} else if value.is_some() {
sqlx::query(DELETE_USAGE_BODY_BLOB_SQL)
.bind(&body_ref)
.execute(executor)
.await
.map_postgres_err()?;
}
Ok(()) Ok(())
} }
@@ -13383,43 +13400,46 @@ async fn sync_usage_http_audit_storage<'e, E>(
request_id: &str, request_id: &str,
headers: &UsageHttpAuditHeaders<'_>, headers: &UsageHttpAuditHeaders<'_>,
refs: &UsageHttpAuditRefs, refs: &UsageHttpAuditRefs,
_states: &UsageHttpAuditStates, states: &UsageHttpAuditStates,
body_capture_mode: &str, body_capture_mode: &str,
) -> Result<(), DataLayerError> ) -> Result<(), DataLayerError>
where where
E: sqlx::Executor<'e, Database = Postgres>, E: sqlx::Executor<'e, Database = Postgres>,
{ {
if headers.any_present() || refs.any_present() || body_capture_mode != "none" { if !headers.any_present()
return Err(DataLayerError::InvalidInput( && !refs.any_present()
"usage HTTP capture persistence is disabled".to_string(), && !states.any_present()
)); && body_capture_mode == "none"
{
return Ok(());
} }
sqlx::query( sqlx::query(UPSERT_USAGE_HTTP_AUDIT_SQL)
r#" .bind(request_id)
WITH deleted_audit AS ( .bind(headers.request_headers_json)
DELETE FROM usage_http_audits WHERE request_id = $1 .bind(headers.provider_request_headers_json)
) .bind(headers.response_headers_json)
UPDATE usage .bind(headers.client_response_headers_json)
SET request_headers = NULL, .bind(refs.request_body_ref.as_deref())
request_body = NULL, .bind(refs.provider_request_body_ref.as_deref())
provider_request_headers = NULL, .bind(refs.response_body_ref.as_deref())
provider_request_body = NULL, .bind(refs.client_response_body_ref.as_deref())
response_headers = NULL, .bind(usage_body_capture_state_bind_text(
response_body = NULL, states.request_body_state,
client_response_headers = NULL, ))
client_response_body = NULL, .bind(usage_body_capture_state_bind_text(
request_body_compressed = NULL, states.provider_request_body_state,
provider_request_body_compressed = NULL, ))
response_body_compressed = NULL, .bind(usage_body_capture_state_bind_text(
client_response_body_compressed = NULL states.response_body_state,
WHERE request_id = $1 ))
"#, .bind(usage_body_capture_state_bind_text(
) states.client_response_body_state,
.bind(request_id) ))
.execute(executor) .bind(body_capture_mode)
.await .execute(executor)
.map_postgres_err()?; .await
.map_postgres_err()?;
Ok(()) Ok(())
} }
@@ -187,6 +187,152 @@ async fn pending_batch_is_opt_in_and_rejects_non_pending_before_connecting() {
.contains("pending usage batch requires pending status")); .contains("pending usage batch requires pending status"));
} }
#[tokio::test]
#[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"]
async fn live_full_http_capture_round_trips_for_direct_and_batch_writes() {
let factory = PostgresPoolFactory::new(PostgresPoolConfig {
database_url: std::env::var("AETHER_TEST_DATABASE_URL").unwrap(),
min_connections: 1,
max_connections: 2,
acquire_timeout_ms: 10_000,
idle_timeout_ms: 30_000,
max_lifetime_ms: 60_000,
statement_cache_capacity: 64,
require_ssl: false,
})
.unwrap();
let repository = SqlxUsageReadRepository::new(factory.connect_lazy().unwrap());
crate::run_migrations(repository.pool()).await.unwrap();
for batch in [false, true] {
let request_id = format!("req-full-capture-{}", uuid::Uuid::new_v4().simple());
let now_unix_secs = Utc::now().timestamp() as u64;
let mut pending = fast_clear_usage_record(
&request_id,
"full-capture-test",
now_unix_secs,
false,
UsageBodyCaptureState::Inline,
None,
);
pending.request_headers =
Some(json!({"content-type": "application/json", "authorization": "Bearer private"}));
pending.request_body =
Some(json!({"messages": [{"role": "user", "content": "original request"}]}));
pending.request_body_state = Some(UsageBodyCaptureState::Inline);
pending.provider_request_body = Some(json!({"input": "provider request"}));
pending.response_body = Some(json!("pending response"));
pending.response_body_state = Some(UsageBodyCaptureState::Inline);
pending.client_response_body = Some(json!("pending client response"));
pending.client_response_body_state = Some(UsageBodyCaptureState::Inline);
if batch {
repository
.upsert_pending_many(vec![pending.clone()])
.await
.unwrap();
} else {
repository.upsert(pending.clone()).await.unwrap();
}
for (field, expected) in [
(UsageBodyField::RequestBody, pending.request_body.as_ref()),
(
UsageBodyField::ProviderRequestBody,
pending.provider_request_body.as_ref(),
),
(UsageBodyField::ResponseBody, pending.response_body.as_ref()),
(
UsageBodyField::ClientResponseBody,
pending.client_response_body.as_ref(),
),
] {
assert_eq!(
repository
.resolve_body_ref(&usage_body_ref(&request_id, field))
.await
.unwrap()
.as_ref(),
expected,
"batch={batch}, field={field:?}"
);
}
let mut terminal = fast_clear_usage_record(
&request_id,
"full-capture-test",
now_unix_secs,
true,
UsageBodyCaptureState::None,
None,
);
terminal.provider_request_body_state = None;
terminal.response_headers =
Some(json!({"content-type": "text/event-stream", "set-cookie": "private"}));
terminal.response_body = Some(json!(format!(
"data: {}\n\ndata: [DONE]\n\n",
"streamed text".repeat(8192)
)));
terminal.response_body_state = Some(UsageBodyCaptureState::Inline);
terminal.client_response_body = Some(json!({"output": "final response"}));
terminal.client_response_body_state = Some(UsageBodyCaptureState::Inline);
repository.upsert(terminal.clone()).await.unwrap();
let stored = repository
.find_by_request_id_shallow(&request_id)
.await
.unwrap()
.unwrap();
assert_eq!(
stored.request_headers,
Some(json!({"content-type": "application/json", "authorization": "[redacted]"}))
);
assert_eq!(
stored.response_headers,
Some(json!({"content-type": "text/event-stream", "set-cookie": "[redacted]"}))
);
for (field, expected) in [
(UsageBodyField::RequestBody, pending.request_body.as_ref()),
(
UsageBodyField::ProviderRequestBody,
pending.provider_request_body.as_ref(),
),
(
UsageBodyField::ResponseBody,
terminal.response_body.as_ref(),
),
(
UsageBodyField::ClientResponseBody,
terminal.client_response_body.as_ref(),
),
] {
assert_eq!(
stored.body_state(field),
Some(UsageBodyCaptureState::Reference)
);
assert_eq!(
stored.body_ref(field),
Some(usage_body_ref(&request_id, field).as_str())
);
assert_eq!(
repository
.resolve_body_ref(stored.body_ref(field).unwrap())
.await
.unwrap()
.as_ref(),
expected,
"batch={batch}, field={field:?}"
);
}
let legacy_content_present: bool = sqlx::query_scalar("SELECT request_body IS NOT NULL OR request_headers IS NOT NULL OR response_body IS NOT NULL FROM usage WHERE request_id = $1")
.bind(&request_id).fetch_one(repository.pool()).await.unwrap();
assert!(!legacy_content_present);
sqlx::query("DELETE FROM usage WHERE request_id = $1")
.bind(&request_id)
.execute(repository.pool())
.await
.unwrap();
}
}
#[tokio::test] #[tokio::test]
#[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"] #[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"]
async fn live_stale_terminal_event_is_a_full_transaction_noop() { async fn live_stale_terminal_event_is_a_full_transaction_noop() {
@@ -253,7 +399,7 @@ async fn live_stale_terminal_event_is_a_full_transaction_noop() {
.unwrap(), .unwrap(),
); );
let settlement_before = sqlx::query( let settlement_before = sqlx::query(
"SELECT billing_status, billing_total_cost_usd FROM usage_settlement_snapshots WHERE request_id = $1", "SELECT billing_status, billing_total_cost_usd::DOUBLE PRECISION AS billing_total_cost_usd FROM usage_settlement_snapshots WHERE request_id = $1",
) )
.bind(&request_id) .bind(&request_id)
.fetch_one(repository.pool()) .fetch_one(repository.pool())
@@ -318,7 +464,7 @@ async fn live_stale_terminal_event_is_a_full_transaction_noop() {
.unwrap(), .unwrap(),
); );
let settlement_after = sqlx::query( let settlement_after = sqlx::query(
"SELECT billing_status, billing_total_cost_usd FROM usage_settlement_snapshots WHERE request_id = $1", "SELECT billing_status, billing_total_cost_usd::DOUBLE PRECISION AS billing_total_cost_usd FROM usage_settlement_snapshots WHERE request_id = $1",
) )
.bind(&request_id) .bind(&request_id)
.fetch_one(repository.pool()) .fetch_one(repository.pool())
@@ -542,7 +688,7 @@ async fn live_pending_batch_persists_auxiliary_state_and_preserves_terminal_conf
.fetch_one(repository.pool()) .fetch_one(repository.pool())
.await .await
.expect("HTTP audit count should be readable"); .expect("HTTP audit count should be readable");
assert_eq!(http_count, 0); assert_eq!(http_count, 1);
let blob_count = sqlx::query_scalar::<_, i64>( let blob_count = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(*)::BIGINT FROM usage_body_blobs WHERE request_id = $1", "SELECT COUNT(*)::BIGINT FROM usage_body_blobs WHERE request_id = $1",
) )
@@ -550,7 +696,23 @@ async fn live_pending_batch_persists_auxiliary_state_and_preserves_terminal_conf
.fetch_one(repository.pool()) .fetch_one(repository.pool())
.await .await
.expect("body blob count should be readable"); .expect("body blob count should be readable");
assert_eq!(blob_count, 0); assert_eq!(blob_count, 4);
let captured = repository
.find_by_request_id_shallow(&rich_request_id)
.await
.unwrap()
.unwrap();
assert_eq!(
captured.request_headers,
Some(json!({"x-request": "[redacted]"}))
);
assert_eq!(
repository
.resolve_body_ref(captured.body_ref(UsageBodyField::RequestBody).unwrap())
.await
.unwrap(),
Some(json!({"messages": [{"role": "user", "content": "hello"}]}))
);
let routing = sqlx::query( let routing = sqlx::query(
"SELECT candidate_id, candidate_index, selected_provider_api_key_id FROM usage_routing_snapshots WHERE request_id = $1", "SELECT candidate_id, candidate_index, selected_provider_api_key_id FROM usage_routing_snapshots WHERE request_id = $1",
@@ -667,7 +829,7 @@ async fn live_pending_batch_and_terminal_upserts_count_each_provider_request_onc
let suffix = uuid::Uuid::new_v4().simple().to_string(); let suffix = uuid::Uuid::new_v4().simple().to_string();
let provider_name = format!("pending-terminal-race-provider-{suffix}"); let provider_name = format!("pending-terminal-race-provider-{suffix}");
let provider_key_id = format!("pending-terminal-race-key-{suffix}"); let provider_key_id = uuid::Uuid::new_v4().to_string();
let now_unix_secs = Utc::now().timestamp().max(0) as u64; let now_unix_secs = Utc::now().timestamp().max(0) as u64;
let request_ids = (0..REQUESTS) let request_ids = (0..REQUESTS)
.map(|index| format!("req-pending-terminal-race-{index}-{suffix}")) .map(|index| format!("req-pending-terminal-race-{index}-{suffix}"))
@@ -793,7 +955,7 @@ async fn live_first_byte_fast_path_is_atomic_and_preserves_terminal_state() {
let existing_request_id = format!("req-first-byte-existing-{suffix}"); let existing_request_id = format!("req-first-byte-existing-{suffix}");
let metadata_fill_request_id = format!("req-first-byte-metadata-fill-{suffix}"); let metadata_fill_request_id = format!("req-first-byte-metadata-fill-{suffix}");
let provider_name = format!("first-byte-fast-{suffix}"); let provider_name = format!("first-byte-fast-{suffix}");
let missing_provider_key_id = format!("key-first-byte-missing-{suffix}"); let missing_provider_key_id = uuid::Uuid::new_v4().to_string();
let now_unix_secs = Utc::now().timestamp().max(0) as u64; let now_unix_secs = Utc::now().timestamp().max(0) as u64;
let mut missing_first_byte = first_byte_usage_record( let mut missing_first_byte = first_byte_usage_record(
@@ -1064,7 +1226,7 @@ async fn live_first_byte_reads_provider_contribution_after_waiting_for_canonical
let suffix = uuid::Uuid::new_v4().simple().to_string(); let suffix = uuid::Uuid::new_v4().simple().to_string();
let request_id = format!("req-first-byte-lock-snapshot-{suffix}"); let request_id = format!("req-first-byte-lock-snapshot-{suffix}");
let provider_name = format!("first-byte-lock-snapshot-{suffix}"); let provider_name = format!("first-byte-lock-snapshot-{suffix}");
let provider_key_id = format!("key-first-byte-lock-snapshot-{suffix}"); let provider_key_id = uuid::Uuid::new_v4().to_string();
let now_unix_secs = Utc::now().timestamp().max(0) as u64; let now_unix_secs = Utc::now().timestamp().max(0) as u64;
let mut pending = first_byte_usage_record( let mut pending = first_byte_usage_record(
&request_id, &request_id,
@@ -1209,14 +1371,14 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
let request_b = format!("req-first-byte-batch-b-{suffix}"); let request_b = format!("req-first-byte-batch-b-{suffix}");
let request_missing = format!("req-first-byte-batch-missing-{suffix}"); let request_missing = format!("req-first-byte-batch-missing-{suffix}");
let request_terminal = format!("req-first-byte-batch-terminal-{suffix}"); let request_terminal = format!("req-first-byte-batch-terminal-{suffix}");
let missing_provider_key_id = format!("key-first-byte-batch-missing-{suffix}"); let missing_provider_key_id = uuid::Uuid::new_v4().to_string();
let now_unix_secs = Utc::now().timestamp().max(0) as u64; let now_unix_secs = Utc::now().timestamp().max(0) as u64;
let mut pending_a = first_byte_usage_record( let mut pending_a = first_byte_usage_record(
&request_a, &request_a,
&provider_name, &provider_name,
now_unix_secs, now_unix_secs,
Some(json!({"seed": "a"})), Some(json!({"trace_id": "seed-a"})),
); );
pending_a.status = "pending".to_string(); pending_a.status = "pending".to_string();
pending_a.first_byte_time_ms = None; pending_a.first_byte_time_ms = None;
@@ -1241,7 +1403,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
); );
terminal.is_stream = Some(true); terminal.is_stream = Some(true);
terminal.first_byte_time_ms = Some(44); terminal.first_byte_time_ms = Some(44);
terminal.request_metadata = Some(json!({"terminal": true})); terminal.request_metadata = Some(json!({"trace_id": "terminal"}));
repository repository
.upsert(pending_a) .upsert(pending_a)
@@ -1260,7 +1422,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
&request_a, &request_a,
&provider_name, &provider_name,
now_unix_secs + 1, now_unix_secs + 1,
Some(json!({"incoming": "a"})), Some(json!({"trace_id": "incoming-a"})),
); );
first_a.first_byte_time_ms = Some(30); first_a.first_byte_time_ms = Some(30);
first_a.has_format_conversion = None; first_a.has_format_conversion = None;
@@ -1272,7 +1434,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
&request_b, &request_b,
&provider_name, &provider_name,
now_unix_secs + 1, now_unix_secs + 1,
Some(json!({"incoming": "b"})), Some(json!({"trace_id": "incoming-b"})),
); );
first_b.has_format_conversion = Some(true); first_b.has_format_conversion = Some(true);
@@ -1280,7 +1442,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
&request_terminal, &request_terminal,
&provider_name, &provider_name,
now_unix_secs + 2, now_unix_secs + 2,
Some(json!({"late": true})), Some(json!({"trace_id": "late"})),
); );
late_terminal.first_byte_time_ms = Some(3); late_terminal.first_byte_time_ms = Some(3);
late_terminal.has_format_conversion = None; late_terminal.has_format_conversion = None;
@@ -1288,7 +1450,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
&request_missing, &request_missing,
&provider_name, &provider_name,
now_unix_secs + 1, now_unix_secs + 1,
Some(json!({"incoming": "missing"})), Some(json!({"trace_id": "incoming-missing"})),
); );
first_missing.provider_api_key_id = Some(missing_provider_key_id.clone()); first_missing.provider_api_key_id = Some(missing_provider_key_id.clone());
@@ -1345,7 +1507,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
row_a row_a
.try_get::<Option<serde_json::Value>, _>("request_metadata") .try_get::<Option<serde_json::Value>, _>("request_metadata")
.unwrap(), .unwrap(),
Some(json!({"seed": "a"})), Some(json!({"trace_id": "seed-a"})),
"existing metadata remains authoritative" "existing metadata remains authoritative"
); );
@@ -1364,7 +1526,7 @@ async fn live_first_byte_batch_preserves_duplicate_order_and_terminal_guards() {
row_b row_b
.try_get::<Option<serde_json::Value>, _>("request_metadata") .try_get::<Option<serde_json::Value>, _>("request_metadata")
.unwrap(), .unwrap(),
Some(json!({"incoming": "b"})) Some(json!({"trace_id": "incoming-b"}))
); );
let row_terminal = rows let row_terminal = rows
@@ -2044,7 +2206,7 @@ async fn live_provider_performance_grouping_sets_matches_separate_queries() {
} }
#[tokio::test] #[tokio::test]
#[ignore = "requires AETHER_TEST_DATABASE_URL and a populated PostgreSQL database"] #[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"]
async fn live_dashboard_daily_breakdown_uses_canonical_covering_read_path() { async fn live_dashboard_daily_breakdown_uses_canonical_covering_read_path() {
let database_url = std::env::var("AETHER_TEST_DATABASE_URL") let database_url = std::env::var("AETHER_TEST_DATABASE_URL")
.expect("AETHER_TEST_DATABASE_URL must point at the test database"); .expect("AETHER_TEST_DATABASE_URL must point at the test database");
@@ -2062,6 +2224,20 @@ async fn live_dashboard_daily_breakdown_uses_canonical_covering_read_path() {
let repository = let repository =
SqlxUsageReadRepository::new(factory.connect_lazy().expect("lazy pool should build")); SqlxUsageReadRepository::new(factory.connect_lazy().expect("lazy pool should build"));
let until = Utc::now().timestamp().max(0) as u64; let until = Utc::now().timestamp().max(0) as u64;
crate::run_migrations(repository.pool()).await.unwrap();
let request_id = format!("daily-breakdown-{}", uuid::Uuid::new_v4().simple());
let provider_name = format!("daily-provider-{}", uuid::Uuid::new_v4().simple());
repository
.upsert(fast_clear_usage_record(
&request_id,
&provider_name,
until.saturating_sub(60),
true,
UsageBodyCaptureState::None,
None,
))
.await
.unwrap();
let started = std::time::Instant::now(); let started = std::time::Instant::now();
let rows = repository let rows = repository
.list_dashboard_daily_breakdown(&UsageDashboardDailyBreakdownQuery { .list_dashboard_daily_breakdown(&UsageDashboardDailyBreakdownQuery {
@@ -2077,7 +2253,17 @@ async fn live_dashboard_daily_breakdown_uses_canonical_covering_read_path() {
started.elapsed(), started.elapsed(),
rows.len() rows.len()
); );
assert!(!rows.is_empty()); let seeded = rows
.iter()
.find(|row| row.provider == provider_name)
.unwrap();
assert_eq!(seeded.requests, 1);
assert_eq!(seeded.total_tokens, 2);
sqlx::query("DELETE FROM \"usage\" WHERE request_id = $1")
.bind(&request_id)
.execute(repository.pool())
.await
.unwrap();
} }
#[tokio::test] #[tokio::test]
@@ -81,12 +81,12 @@ fn select_video_task_full_columns() -> String {
fn select_video_task_claim_columns() -> String { fn select_video_task_claim_columns() -> String {
select_video_task_columns( select_video_task_columns(
"NULL::TEXT", "prompt",
"NULL::jsonb", "NULL::jsonb",
"NULL::INTEGER", "duration_seconds",
"NULL::TEXT", "resolution",
"NULL::TEXT", "aspect_ratio",
"NULL::TEXT", "size",
) )
} }
@@ -1176,7 +1176,7 @@ fn map_video_task_row(row: &PgRow) -> Result<StoredVideoTask, DataLayerError> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{update_if_active_sql, upsert_sql, SqlxVideoTaskRepository}; use super::{claim_due_sql, update_if_active_sql, upsert_sql, SqlxVideoTaskRepository};
use crate::{PostgresPoolConfig, PostgresPoolFactory}; use crate::{PostgresPoolConfig, PostgresPoolFactory};
use aether_data_contracts::repository::video_tasks::{ use aether_data_contracts::repository::video_tasks::{
UpsertVideoTask, VideoTaskLookupKey, VideoTaskQueryFilter, VideoTaskReadRepository, UpsertVideoTask, VideoTaskLookupKey, VideoTaskQueryFilter, VideoTaskReadRepository,
@@ -1240,6 +1240,142 @@ mod tests {
assert!(update.contains("created_at = COALESCE(created_at, TO_TIMESTAMP($34))")); assert!(update.contains("created_at = COALESCE(created_at, TO_TIMESTAMP($34))"));
} }
#[test]
fn poll_claim_returns_business_fields_required_by_identity_guards() {
let sql = claim_due_sql();
for field in [
"prompt",
"duration_seconds",
"resolution",
"aspect_ratio",
"size",
] {
assert!(
sql.contains(&format!("{field} AS {field}")),
"claim must retain {field}"
);
}
assert!(sql.contains("NULL::jsonb AS original_request_body"));
assert!(sql.contains("FOR UPDATE SKIP LOCKED"));
}
#[tokio::test]
#[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"]
async fn live_video_task_capture_claim_and_completion_preserve_business_fields() {
let database_url = std::env::var("AETHER_TEST_DATABASE_URL")
.expect("AETHER_TEST_DATABASE_URL must point at the test database");
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.connect(&database_url)
.await
.expect("test database should connect");
crate::run_migrations(&pool)
.await
.expect("test database should migrate");
sqlx::query("CREATE TEMP TABLE video_tasks (LIKE public.video_tasks INCLUDING ALL)")
.execute(&pool)
.await
.expect("isolated task table should be created");
let repository = SqlxVideoTaskRepository::new(pool);
for api_format in ["openai:video", "gemini:video"] {
let task_id = uuid::Uuid::new_v4().to_string();
let original = UpsertVideoTask {
id: task_id.clone(),
short_id: Some(uuid::Uuid::new_v4().simple().to_string()[..16].to_string()),
request_id: format!("request-{task_id}"),
user_id: None,
api_key_id: None,
username: Some("alice".to_string()),
api_key_name: Some("video-client".to_string()),
external_task_id: Some("upstream-task-1".to_string()),
provider_id: None,
endpoint_id: None,
key_id: None,
client_api_format: Some(api_format.to_string()),
provider_api_format: Some(api_format.to_string()),
format_converted: false,
model: Some("video-model".to_string()),
prompt: Some("business prompt".to_string()),
original_request_body: Some(serde_json::json!({"token": "private"})),
duration_seconds: Some(8),
resolution: Some("1080p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("1920x1080".to_string()),
status: VideoTaskStatus::Submitted,
progress_percent: 0,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: Some(10),
poll_count: 0,
max_poll_count: 360,
created_at_unix_ms: 1,
submitted_at_unix_secs: Some(1),
completed_at_unix_secs: None,
updated_at_unix_secs: 1,
error_code: None,
error_message: None,
video_url: None,
request_metadata: Some(serde_json::json!({"authorization": "private"})),
};
let stored = repository
.upsert(original.clone())
.await
.expect("task should persist");
assert_eq!(stored.prompt, original.prompt);
assert_eq!(stored.username, original.username);
assert_eq!(stored.api_key_name, original.api_key_name);
assert!(stored.original_request_body.is_none());
assert!(stored.request_metadata.is_none());
let mut claimed = repository
.claim_due(20, 50, 10)
.await
.expect("task should be claimed");
assert_eq!(claimed.len(), 1);
let mut completion: UpsertVideoTask = claimed.pop().expect("claimed task").into();
stored
.ensure_immutable_identity_matches(&completion)
.expect("claim must preserve task identity");
assert_eq!(completion.prompt, original.prompt);
let mut mismatched = completion.clone();
mismatched.duration_seconds = Some(99);
assert!(repository
.update_if_active(mismatched)
.await
.expect("guarded update should execute")
.is_none());
completion.status = VideoTaskStatus::Completed;
completion.progress_percent = 100;
completion.next_poll_at_unix_secs = None;
completion.completed_at_unix_secs = Some(21);
completion.updated_at_unix_secs = 21;
completion.video_url = Some(
"https://cdn.example.test/video.mp4?alt=media&signature=a%2Fb%2Bc%3D&part=2&part=1"
.to_string(),
);
let completed = repository
.update_if_active(completion.clone())
.await
.expect("completion should execute")
.expect("matching active task should complete");
assert_eq!(completed.video_url, completion.video_url);
let reloaded = repository
.find(VideoTaskLookupKey::Id(&task_id))
.await
.expect("task should reload")
.expect("task should exist");
assert_eq!(reloaded.status, VideoTaskStatus::Completed);
assert_eq!(reloaded.prompt, original.prompt);
assert_eq!(reloaded.video_url, completion.video_url);
assert_eq!(reloaded.duration_seconds, original.duration_seconds);
assert_eq!(reloaded.size, original.size);
assert_eq!(reloaded.username, original.username);
assert!(reloaded.request_metadata.is_none());
}
repository.pool().close().await;
}
#[tokio::test] #[tokio::test]
async fn repository_constructs_from_lazy_pool() { async fn repository_constructs_from_lazy_pool() {
let repository = SqlxVideoTaskRepository::new(build_pool()); let repository = SqlxVideoTaskRepository::new(build_pool());
@@ -268,7 +268,7 @@ pub fn strip_deprecated_usage_display_fields(mut usage: UpsertUsageRecord) -> Up
usage usage
} }
pub fn sanitize_usage_for_persistence(mut usage: UpsertUsageRecord) -> UpsertUsageRecord { fn sanitize_usage_record_metadata(mut usage: UpsertUsageRecord) -> UpsertUsageRecord {
usage = strip_deprecated_usage_display_fields(usage); usage = strip_deprecated_usage_display_fields(usage);
sanitize_usage_routing_fields(&mut usage, None); sanitize_usage_routing_fields(&mut usage, None);
usage.error_message = None; usage.error_message = None;
@@ -280,6 +280,11 @@ pub fn sanitize_usage_for_persistence(mut usage: UpsertUsageRecord) -> UpsertUsa
.map(str::to_string); .map(str::to_string);
} }
usage.request_metadata = super::sanitize_usage_request_metadata(usage.request_metadata); usage.request_metadata = super::sanitize_usage_request_metadata(usage.request_metadata);
usage
}
pub fn sanitize_usage_for_persistence(usage: UpsertUsageRecord) -> UpsertUsageRecord {
let mut usage = sanitize_usage_record_metadata(usage);
usage.request_headers = None; usage.request_headers = None;
usage.request_body = None; usage.request_body = None;
usage.request_body_ref = None; usage.request_body_ref = None;
@@ -299,40 +304,96 @@ pub fn sanitize_usage_for_persistence(mut usage: UpsertUsageRecord) -> UpsertUsa
usage usage
} }
/// Project an event onto the non-content controls accepted by auxiliary usage storage.
///
/// Explicit `none` states are retained only as tombstones for removing historical captures.
/// Every header, body, reference, and non-clear capture state is discarded.
pub fn sanitize_usage_capture_controls_for_persistence( pub fn sanitize_usage_capture_controls_for_persistence(
mut usage: UpsertUsageRecord, mut usage: UpsertUsageRecord,
) -> UpsertUsageRecord { ) -> UpsertUsageRecord {
// Routing facts are allowed in the transient event metadata for compatibility with older
// writers. Project only the known scalar fields into typed slots before the general metadata
// sanitizer drops unknown keys. This keeps snapshots useful without re-persisting arbitrary
// metadata (or any body/header material).
let metadata = usage let metadata = usage
.request_metadata .request_metadata
.as_ref() .as_ref()
.and_then(Value::as_object) .and_then(Value::as_object)
.cloned(); .cloned();
sanitize_usage_routing_fields(&mut usage, metadata.as_ref()); sanitize_usage_routing_fields(&mut usage, metadata.as_ref());
let clear_request_body = usage.request_body_state == Some(super::UsageBodyCaptureState::None); let mut usage = sanitize_usage_record_metadata(usage);
let clear_provider_request_body = for headers in [
usage.provider_request_body_state == Some(super::UsageBodyCaptureState::None); &mut usage.request_headers,
let clear_response_body = usage.response_body_state == Some(super::UsageBodyCaptureState::None); &mut usage.provider_request_headers,
let clear_client_response_body = &mut usage.response_headers,
usage.client_response_body_state == Some(super::UsageBodyCaptureState::None); &mut usage.client_response_headers,
] {
let mut usage = sanitize_usage_for_persistence(usage); *headers = sanitize_usage_headers_for_persistence(headers.take());
usage.request_body_state = clear_request_body.then_some(super::UsageBodyCaptureState::None); }
usage.provider_request_body_state = for (field, body, body_ref, state) in [
clear_provider_request_body.then_some(super::UsageBodyCaptureState::None); (
usage.response_body_state = clear_response_body.then_some(super::UsageBodyCaptureState::None); super::UsageBodyField::RequestBody,
usage.client_response_body_state = &mut usage.request_body,
clear_client_response_body.then_some(super::UsageBodyCaptureState::None); &mut usage.request_body_ref,
usage.request_body_state,
),
(
super::UsageBodyField::ProviderRequestBody,
&mut usage.provider_request_body,
&mut usage.provider_request_body_ref,
usage.provider_request_body_state,
),
(
super::UsageBodyField::ResponseBody,
&mut usage.response_body,
&mut usage.response_body_ref,
usage.response_body_state,
),
(
super::UsageBodyField::ClientResponseBody,
&mut usage.client_response_body,
&mut usage.client_response_body_ref,
usage.client_response_body_state,
),
] {
if matches!(
state,
Some(
super::UsageBodyCaptureState::None
| super::UsageBodyCaptureState::Disabled
| super::UsageBodyCaptureState::Unavailable
)
) {
*body = None;
*body_ref = None;
} else {
*body_ref = body_ref.as_deref().and_then(|value| {
super::canonical_usage_body_ref_for(value, &usage.request_id, field)
});
}
}
usage usage
} }
pub fn usage_header_value_is_sensitive(name: &str) -> bool {
![
"accept",
"accept-encoding",
"content-encoding",
"content-length",
"content-type",
"transfer-encoding",
"x-request-id",
"x-trace-id",
]
.iter()
.any(|candidate| name.trim().eq_ignore_ascii_case(candidate))
}
pub fn sanitize_usage_headers_for_persistence(value: Option<Value>) -> Option<Value> {
let Value::Object(mut headers) = value? else {
return None;
};
for (name, value) in &mut headers {
if usage_header_value_is_sensitive(name) && !value.is_null() {
*value = Value::String("[redacted]".to_string());
}
}
Some(Value::Object(headers))
}
fn sanitize_usage_routing_fields( fn sanitize_usage_routing_fields(
usage: &mut UpsertUsageRecord, usage: &mut UpsertUsageRecord,
metadata: Option<&Map<String, Value>>, metadata: Option<&Map<String, Value>>,
@@ -773,20 +834,79 @@ mod tests {
} }
#[test] #[test]
fn auxiliary_capture_projection_keeps_only_explicit_clear_tombstones() { fn auxiliary_capture_projection_preserves_captures_and_honors_disabled_states() {
let mut input = usage_with_http_capture(); let mut input = usage_with_http_capture();
input.request_body_state = Some(UsageBodyCaptureState::None); input.request_body_state = Some(UsageBodyCaptureState::None);
input.response_body_state = Some(UsageBodyCaptureState::Disabled); input.response_body_state = Some(UsageBodyCaptureState::Disabled);
let usage = sanitize_usage_capture_controls_for_persistence(input); let usage = sanitize_usage_capture_controls_for_persistence(input);
assert!(usage.request_headers.is_none()); assert_eq!(
usage.request_headers,
Some(json!({"authorization": "[redacted]"}))
);
assert!(usage.request_body.is_none()); assert!(usage.request_body.is_none());
assert!(usage.request_body_ref.is_none()); assert!(usage.request_body_ref.is_none());
assert_eq!(usage.request_body_state, Some(UsageBodyCaptureState::None)); assert_eq!(usage.request_body_state, Some(UsageBodyCaptureState::None));
assert!(usage.provider_request_body_state.is_none()); assert_eq!(
assert!(usage.response_body_state.is_none()); usage.provider_request_body,
assert!(usage.client_response_body_state.is_none()); Some(json!({"prompt": "private"}))
);
assert_eq!(
usage.provider_request_body_state,
Some(UsageBodyCaptureState::Inline)
);
assert!(usage.response_body.is_none());
assert!(usage.response_body_ref.is_none());
assert_eq!(
usage.response_body_state,
Some(UsageBodyCaptureState::Disabled)
);
assert!(usage.client_response_body.is_none());
assert_eq!(
usage.client_response_body_state,
Some(UsageBodyCaptureState::Disabled)
);
}
#[test]
fn auxiliary_capture_projection_preserves_all_body_directions_and_scopes_references() {
let mut input = usage_with_http_capture();
input.request_headers =
Some(json!({"Content-Type": "application/json", "Authorization": "Bearer secret"}));
input.request_body_state = Some(UsageBodyCaptureState::Inline);
input.client_response_body_state = Some(UsageBodyCaptureState::Inline);
input.request_body_ref = Some(super::super::usage_body_ref(
&input.request_id,
super::super::UsageBodyField::RequestBody,
));
input.provider_request_body_ref = input.request_body_ref.clone();
input.response_body_ref = Some(super::super::usage_body_ref(
"another-request",
super::super::UsageBodyField::ResponseBody,
));
let captured = sanitize_usage_capture_controls_for_persistence(input.clone());
assert_eq!(captured.request_body, input.request_body);
assert_eq!(captured.provider_request_body, input.provider_request_body);
assert_eq!(captured.response_body, input.response_body);
assert_eq!(captured.client_response_body, input.client_response_body);
assert_eq!(captured.request_body_ref, input.request_body_ref);
assert!(captured.provider_request_body_ref.is_none());
assert!(captured.response_body_ref.is_none());
assert!(captured.client_response_body_ref.is_none());
assert_eq!(
captured.request_headers,
Some(json!({"Content-Type": "application/json", "Authorization": "[redacted]"}))
);
assert_eq!(
captured.provider_request_headers,
Some(json!({"x-api-key": "[redacted]"}))
);
assert_eq!(
captured.response_headers,
Some(json!({"set-cookie": "[redacted]"}))
);
assert!(captured.error_message.is_none());
} }
#[test] #[test]
@@ -1,8 +1,6 @@
use async_trait::async_trait; use async_trait::async_trait;
use serde_json::Value; use serde_json::Value;
const SAFE_VIDEO_URL_QUERY_KEYS: &[(&str, &str)] = &[("alt", "media")];
#[derive( #[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
)] )]
@@ -221,16 +219,11 @@ impl StoredVideoTask {
} }
fn sanitize_persisted_diagnostics(&mut self) { fn sanitize_persisted_diagnostics(&mut self) {
self.prompt = None;
self.original_request_body = None; self.original_request_body = None;
self.progress_message = None; self.progress_message = None;
self.error_code = sanitize_video_task_error_code(self.error_code.take()); self.error_code = sanitize_video_task_error_code(self.error_code.take());
self.error_message = None; self.error_message = None;
self.video_url = sanitize_video_task_url( self.video_url = sanitize_video_task_url(self.video_url.take());
self.client_api_format.as_deref(),
self.provider_api_format.as_deref(),
self.video_url.take(),
);
self.request_metadata = None; self.request_metadata = None;
} }
@@ -339,18 +332,11 @@ pub struct UpsertVideoTask {
impl UpsertVideoTask { impl UpsertVideoTask {
pub fn sanitize_for_persistence(&mut self) { pub fn sanitize_for_persistence(&mut self) {
self.username = None;
self.api_key_name = None;
self.prompt = None;
self.original_request_body = None; self.original_request_body = None;
self.progress_message = None; self.progress_message = None;
self.error_code = sanitize_video_task_error_code(self.error_code.take()); self.error_code = sanitize_video_task_error_code(self.error_code.take());
self.error_message = None; self.error_message = None;
self.video_url = sanitize_video_task_url( self.video_url = sanitize_video_task_url(self.video_url.take());
self.client_api_format.as_deref(),
self.provider_api_format.as_deref(),
self.video_url.take(),
);
self.request_metadata = None; self.request_metadata = None;
} }
@@ -421,16 +407,7 @@ fn sanitize_video_task_error_code(value: Option<String>) -> Option<String> {
}) })
} }
fn sanitize_video_task_url( fn sanitize_video_task_url(value: Option<String>) -> Option<String> {
client_api_format: Option<&str>,
provider_api_format: Option<&str>,
value: Option<String>,
) -> Option<String> {
if effective_video_task_api_format(client_api_format, provider_api_format)
!= Some("gemini:video")
{
return None;
}
let mut url = url::Url::parse(value?.trim()).ok()?; let mut url = url::Url::parse(value?.trim()).ok()?;
if !matches!(url.scheme(), "http" | "https") if !matches!(url.scheme(), "http" | "https")
|| url.host_str().is_none() || url.host_str().is_none()
@@ -440,15 +417,6 @@ fn sanitize_video_task_url(
return None; return None;
} }
let query = url
.query_pairs()
.filter(|(key, value)| SAFE_VIDEO_URL_QUERY_KEYS.contains(&(key.as_ref(), value.as_ref())))
.map(|(key, value)| (key.into_owned(), value.into_owned()))
.collect::<Vec<_>>();
url.set_query(None);
if !query.is_empty() {
url.query_pairs_mut().extend_pairs(query);
}
url.set_fragment(None); url.set_fragment(None);
Some(url.into()) Some(url.into())
} }
@@ -940,20 +908,23 @@ mod tests {
assert_eq!(task.user_id.as_deref(), Some("user-1")); assert_eq!(task.user_id.as_deref(), Some("user-1"));
assert_eq!(task.api_key_id.as_deref(), Some("api-key-1")); assert_eq!(task.api_key_id.as_deref(), Some("api-key-1"));
assert_eq!(task.username, None); assert_eq!(task.username.as_deref(), Some("private-user-name"));
assert_eq!(task.api_key_name, None); assert_eq!(task.api_key_name.as_deref(), Some("private-key-name"));
assert_eq!(task.original_request_body, None); assert_eq!(task.original_request_body, None);
assert_eq!(task.progress_message, None); assert_eq!(task.progress_message, None);
assert_eq!(task.error_message, None); assert_eq!(task.error_message, None);
assert_eq!(task.error_code.as_deref(), Some("provider_error")); assert_eq!(task.error_code.as_deref(), Some("provider_error"));
assert_eq!(task.video_url, None); assert_eq!(
task.video_url.as_deref(),
Some("https://cdn.example.test/video.mp4?token=secret")
);
assert_eq!(task.request_metadata, None); assert_eq!(task.request_metadata, None);
assert_eq!(task.prompt, None); assert_eq!(task.prompt.as_deref(), Some("prompt"));
assert_eq!(task.duration_seconds, Some(4)); assert_eq!(task.duration_seconds, Some(4));
} }
#[test] #[test]
fn upsert_sanitization_keeps_only_noncredential_video_urls() { fn stored_task_preserves_prompt_and_signed_download_url() {
let mut args = base_new_args(); let mut args = base_new_args();
args.12 = Some("gemini:video".to_string()); args.12 = Some("gemini:video".to_string());
args.15 = Some("private prompt".to_string()); args.15 = Some("private prompt".to_string());
@@ -968,15 +939,15 @@ mod tests {
args.28, args.29, args.30, args.31, args.32, args.33, args.34, args.35, args.36, args.28, args.29, args.30, args.31, args.32, args.33, args.34, args.35, args.36,
) )
.expect("stored task should build"); .expect("stored task should build");
assert_eq!(task.prompt, None); assert_eq!(task.prompt.as_deref(), Some("private prompt"));
assert_eq!( assert_eq!(
task.video_url.as_deref(), task.video_url.as_deref(),
Some("https://cdn.example.test/video.mp4?alt=media") Some("https://cdn.example.test/video.mp4?key=secret&alt=media&signature=private")
); );
} }
#[test] #[test]
fn upsert_sanitization_uses_client_format_when_legacy_provider_format_is_blank() { fn stored_task_preserves_download_url_when_legacy_provider_format_is_blank() {
let mut args = base_new_args(); let mut args = base_new_args();
args.11 = Some("gemini:video".to_string()); args.11 = Some("gemini:video".to_string());
args.12 = Some(" ".to_string()); args.12 = Some(" ".to_string());
@@ -992,7 +963,32 @@ mod tests {
assert_eq!( assert_eq!(
task.video_url.as_deref(), task.video_url.as_deref(),
Some("https://cdn.example.test/video.mp4?alt=media") Some("https://cdn.example.test/video.mp4?key=secret&alt=media")
); );
} }
#[test]
fn video_url_sanitization_preserves_signed_query_encoding_and_order() {
let video_url =
"https://cdn.example.test/video.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1&name=a%20b";
assert_eq!(
super::sanitize_video_task_url(Some(format!("{video_url}#fragment"))).as_deref(),
Some(video_url)
);
}
#[test]
fn video_url_sanitization_rejects_invalid_schemes_and_embedded_credentials() {
for video_url in [
"file:///etc/passwd",
"javascript:alert(1)",
"data:video/mp4;base64,AAAA",
"https://user:password@cdn.example.test/video.mp4",
"https://user@cdn.example.test/video.mp4",
"/relative/video.mp4",
"not a url",
] {
assert!(super::sanitize_video_task_url(Some(video_url.to_string())).is_none());
}
}
} }
@@ -1,13 +1,8 @@
use std::{ use sqlx::{query, query_scalar, PgPool};
path::{Path, PathBuf},
process::{Child, Command, Stdio},
time::{Duration, Instant},
};
use sqlx::{query, query_scalar, Connection, PgConnection, PgPool};
use super::{pending_backfills, pending_backfills_from_applied, run_backfills, AppliedBackfill}; use super::{pending_backfills, pending_backfills_from_applied, run_backfills, AppliedBackfill};
use crate::lifecycle::migrate::prepare_database_for_startup; use crate::lifecycle::migrate::prepare_database_for_startup;
use crate::lifecycle::postgres_test_support::ManagedPostgresServer;
const LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_VERSION: i64 = 20260517012000; const LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_VERSION: i64 = 20260517012000;
const LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_SQL: &str = const LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_SQL: &str =
@@ -84,194 +79,6 @@ fn corrected_legacy_backfill_is_not_requeued_after_application() {
assert!(!pending_versions.contains(&LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_VERSION)); assert!(!pending_versions.contains(&LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_VERSION));
} }
#[derive(Debug)]
struct ManagedPostgresServer {
child: Option<Child>,
workdir: PathBuf,
database_url: String,
}
impl ManagedPostgresServer {
async fn try_start() -> Result<Option<Self>, Box<dyn std::error::Error>> {
let initdb_bin = std::env::var("AETHER_INITDB_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "initdb".to_string());
let postgres_bin = std::env::var("AETHER_POSTGRES_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "postgres".to_string());
if !command_exists(&initdb_bin) || !command_exists(&postgres_bin) {
eprintln!(
"skipping postgres backfill test because required binaries are unavailable: initdb={}, postgres={}",
initdb_bin, postgres_bin
);
return Ok(None);
}
match Self::start(initdb_bin, postgres_bin).await {
Ok(server) => Ok(Some(server)),
Err(err) if postgres_local_startup_unavailable(err.to_string().as_str()) => {
eprintln!(
"skipping postgres backfill test because local postgres could not start in this environment: {err}"
);
Ok(None)
}
Err(err) => Err(err),
}
}
async fn start(
initdb_bin: String,
postgres_bin: String,
) -> Result<Self, Box<dyn std::error::Error>> {
let port = reserve_local_port()?;
let workdir = std::env::temp_dir().join(format!(
"aether-backfill-tests-{}-{}",
std::process::id(),
port
));
let data_dir = workdir.join("data");
std::fs::create_dir_all(&workdir)?;
let init_output = Command::new(&initdb_bin)
.arg("-D")
.arg(&data_dir)
.arg("-U")
.arg("aether")
.arg("--auth=trust")
.arg("--encoding=UTF8")
.arg("--no-instructions")
.output()?;
if !init_output.status.success() {
return Err(std::io::Error::other(format!(
"initdb failed: {}",
String::from_utf8_lossy(&init_output.stderr)
))
.into());
}
let database_url = format!("postgres://aether@127.0.0.1:{port}/postgres");
let log_path = workdir.join("postgres.log");
let stdout = std::fs::File::create(&log_path)?;
let stderr = stdout.try_clone()?;
let mut child = Command::new(&postgres_bin)
.arg("-D")
.arg(&data_dir)
.arg("-h")
.arg("127.0.0.1")
.arg("-p")
.arg(port.to_string())
.arg("-F")
.arg("-c")
.arg("fsync=off")
.arg("-c")
.arg("synchronous_commit=off")
.arg("-c")
.arg("full_page_writes=off")
.arg("-c")
.arg("shared_buffers=8MB")
.arg("-c")
.arg("max_connections=8")
.arg("-c")
.arg("dynamic_shared_memory_type=mmap")
.arg("-c")
.arg("autovacuum=off")
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr))
.spawn()?;
if let Err(err) = wait_for_postgres(&database_url).await {
let _ = child.kill();
let _ = child.wait();
return Err(err);
}
Ok(Self {
child: Some(child),
workdir,
database_url,
})
}
fn database_url(&self) -> &str {
&self.database_url
}
fn stop(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
impl Drop for ManagedPostgresServer {
fn drop(&mut self) {
self.stop();
let _ = std::fs::remove_dir_all(&self.workdir);
}
}
fn command_exists(bin: &str) -> bool {
if bin.contains(std::path::MAIN_SEPARATOR) {
return Path::new(bin).exists();
}
let Some(paths) = std::env::var_os("PATH") else {
return false;
};
std::env::split_paths(&paths).any(|path| path.join(bin).exists())
}
fn reserve_local_port() -> Result<u16, std::io::Error> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let port = listener.local_addr()?.port();
drop(listener);
Ok(port)
}
fn postgres_shared_memory_unavailable(message: &str) -> bool {
let message = message.to_ascii_lowercase();
message.contains("shared memory")
&& (message.contains("could not create shared memory segment")
|| message.contains("shmget")
|| message.contains("no space left on device"))
}
fn postgres_local_startup_unavailable(message: &str) -> bool {
let message = message.to_ascii_lowercase();
postgres_shared_memory_unavailable(&message)
|| (message.contains("timed out waiting for local postgres")
&& (message.contains("connection refused")
|| message.contains("os error 61")
|| message.contains("os error 111")))
}
async fn wait_for_postgres(database_url: &str) -> Result<(), Box<dyn std::error::Error>> {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
match PgConnection::connect(database_url).await {
Ok(connection) => {
connection.close().await?;
return Ok(());
}
Err(_) if Instant::now() < deadline => {
tokio::time::sleep(Duration::from_millis(50)).await
}
Err(err) => {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("timed out waiting for local postgres: {err}"),
)
.into())
}
}
}
}
#[tokio::test] #[tokio::test]
async fn run_backfills_rebuilds_stats_and_records_execution() { async fn run_backfills_rebuilds_stats_and_records_execution() {
let Some(server) = ManagedPostgresServer::try_start() let Some(server) = ManagedPostgresServer::try_start()
@@ -538,9 +538,15 @@ fn imported_payload_ids(
.collect() .collect()
} }
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct DataImportOptions {
pub preserve_credentials: bool,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct DataCopyOptions { pub struct DataCopyOptions {
pub omit_request_body_details: bool, pub omit_request_body_details: bool,
pub preserve_credentials: bool,
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
@@ -572,6 +578,25 @@ fn set_supported_import_value(
} }
} }
fn apply_import_credential_policy(
table_name: &str,
object: &mut serde_json::Map<String, Value>,
target_has_column: impl Fn(&str) -> bool,
options: DataImportOptions,
) {
let normalized_table = table_name
.rsplit('.')
.next()
.unwrap_or(table_name)
.trim_matches(|character| matches!(character, '"' | '`'));
if options.preserve_credentials
&& matches!(normalized_table, "users" | "api_keys" | "management_tokens")
{
return;
}
deactivate_imported_credentials(table_name, object, target_has_column);
}
fn deactivate_imported_credentials( fn deactivate_imported_credentials(
table_name: &str, table_name: &str,
object: &mut serde_json::Map<String, Value>, object: &mut serde_json::Map<String, Value>,
@@ -1085,6 +1110,14 @@ pub async fn export_database_jsonl(
pub async fn import_database_jsonl( pub async fn import_database_jsonl(
database: SqlDatabaseConfig, database: SqlDatabaseConfig,
input: &str, input: &str,
) -> Result<usize, DataLayerError> {
import_database_jsonl_with_options(database, input, DataImportOptions::default()).await
}
pub async fn import_database_jsonl_with_options(
database: SqlDatabaseConfig,
input: &str,
options: DataImportOptions,
) -> Result<usize, DataLayerError> { ) -> Result<usize, DataLayerError> {
match database.driver { match database.driver {
#[cfg(feature = "postgres")] #[cfg(feature = "postgres")]
@@ -1092,7 +1125,7 @@ pub async fn import_database_jsonl(
let pool = let pool =
crate::driver::postgres::PostgresPoolFactory::new(database.to_postgres_config()?)? crate::driver::postgres::PostgresPoolFactory::new(database.to_postgres_config()?)?
.connect_lazy()?; .connect_lazy()?;
import_postgres_jsonl(&pool, input).await postgres::import_postgres_jsonl_with_options(&pool, input, options).await
} }
#[cfg(not(feature = "postgres"))] #[cfg(not(feature = "postgres"))]
DatabaseDriver::Postgres => Err(DataLayerError::InvalidInput( DatabaseDriver::Postgres => Err(DataLayerError::InvalidInput(
@@ -1113,7 +1146,14 @@ pub async fn copy_database_records(
if options.omit_request_body_details { if options.omit_request_body_details {
omit_request_body_details_from_records(&mut records); omit_request_body_details_from_records(&mut records);
} }
import_database_jsonl(target, &encode_jsonl(&records)?).await import_database_jsonl_with_options(
target,
&encode_jsonl(&records)?,
DataImportOptions {
preserve_credentials: options.preserve_credentials,
},
)
.await
} }
fn omit_request_body_details_from_records(records: &mut Vec<DataExportRecord>) { fn omit_request_body_details_from_records(records: &mut Vec<DataExportRecord>) {
@@ -58,14 +58,30 @@ pub async fn export_postgres_jsonl(
pub async fn import_postgres_jsonl( pub async fn import_postgres_jsonl(
pool: &crate::driver::postgres::PostgresPool, pool: &crate::driver::postgres::PostgresPool,
input: &str, input: &str,
) -> Result<usize, DataLayerError> {
import_postgres_jsonl_with_options(pool, input, DataImportOptions::default()).await
}
pub(super) async fn import_postgres_jsonl_with_options(
pool: &crate::driver::postgres::PostgresPool,
input: &str,
options: DataImportOptions,
) -> Result<usize, DataLayerError> { ) -> Result<usize, DataLayerError> {
let plan = build_import_plan(input)?; let plan = build_import_plan(input)?;
import_postgres_plan(pool, &plan).await import_postgres_plan_with_options(pool, &plan, options).await
} }
pub async fn import_postgres_plan( pub async fn import_postgres_plan(
pool: &crate::driver::postgres::PostgresPool, pool: &crate::driver::postgres::PostgresPool,
plan: &DataImportPlan, plan: &DataImportPlan,
) -> Result<usize, DataLayerError> {
import_postgres_plan_with_options(pool, plan, DataImportOptions::default()).await
}
async fn import_postgres_plan_with_options(
pool: &crate::driver::postgres::PostgresPool,
plan: &DataImportPlan,
options: DataImportOptions,
) -> Result<usize, DataLayerError> { ) -> Result<usize, DataLayerError> {
let identity_scope = IdentityImportScope::from_plan(plan)?; let identity_scope = IdentityImportScope::from_plan(plan)?;
let mut tx = pool.begin().await.map_sql_err()?; let mut tx = pool.begin().await.map_sql_err()?;
@@ -75,7 +91,7 @@ pub async fn import_postgres_plan(
for domain in &plan.manifest.domains { for domain in &plan.manifest.domains {
if *domain == ExportDomain::Auxiliary { if *domain == ExportDomain::Auxiliary {
for row in plan.rows(*domain) { for row in plan.rows(*domain) {
import_postgres_auxiliary_row(&mut tx, row, &mut column_cache).await?; import_postgres_auxiliary_row(&mut tx, row, &mut column_cache, options).await?;
imported = imported.saturating_add(1); imported = imported.saturating_add(1);
} }
continue; continue;
@@ -110,6 +126,7 @@ pub async fn import_postgres_plan(
*domain, *domain,
row, row,
&target_columns, &target_columns,
options,
) )
.await?; .await?;
imported = imported.saturating_add(1); imported = imported.saturating_add(1);
@@ -587,11 +604,15 @@ async fn import_postgres_row(
domain: ExportDomain, domain: ExportDomain,
row: &ExportRow, row: &ExportRow,
target_columns: &PostgresImportColumns, target_columns: &PostgresImportColumns,
options: DataImportOptions,
) -> Result<(), DataLayerError> { ) -> Result<(), DataLayerError> {
let mut object = normalize_postgres_import_payload(table_name, domain, row, target_columns)?; let mut object = normalize_postgres_import_payload(table_name, domain, row, target_columns)?;
deactivate_imported_credentials(table_name, &mut object, |column_name| { apply_import_credential_policy(
target_columns.contains_key(column_name) table_name,
}); &mut object,
|column_name| target_columns.contains_key(column_name),
options,
);
let columns = object.keys().map(String::as_str).collect::<Vec<_>>(); let columns = object.keys().map(String::as_str).collect::<Vec<_>>();
let column_sql = columns let column_sql = columns
@@ -839,6 +860,7 @@ async fn import_postgres_billing_row(
payload, payload,
}, },
&target_columns, &target_columns,
DataImportOptions::default(),
) )
.await .await
} }
@@ -847,6 +869,7 @@ async fn import_postgres_auxiliary_row(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
row: &ExportRow, row: &ExportRow,
column_cache: &mut BTreeMap<String, PostgresImportColumns>, column_cache: &mut BTreeMap<String, PostgresImportColumns>,
options: DataImportOptions,
) -> Result<(), DataLayerError> { ) -> Result<(), DataLayerError> {
let (table_name, payload) = domain_payload_table(row, "auxiliary", None)?; let (table_name, payload) = domain_payload_table(row, "auxiliary", None)?;
let table = auxiliary_table(&table_name)?; let table = auxiliary_table(&table_name)?;
@@ -862,6 +885,7 @@ async fn import_postgres_auxiliary_row(
payload, payload,
}, },
&target_columns, &target_columns,
options,
) )
.await .await
} }
@@ -897,6 +921,7 @@ async fn import_postgres_wallet_row(
payload, payload,
}, },
&target_columns, &target_columns,
DataImportOptions::default(),
) )
.await .await
} }
@@ -3,11 +3,12 @@ use std::collections::{BTreeMap, BTreeSet};
use serde_json::{json, Value}; use serde_json::{json, Value};
use super::{ use super::{
build_import_plan, deactivate_imported_credentials, decode_jsonl, decode_jsonl_with_limits, apply_import_credential_policy, build_import_plan, deactivate_imported_credentials,
encode_jsonl, export_postgres_core_jsonl, normalize_imported_binary, decode_jsonl, decode_jsonl_with_limits, encode_jsonl, export_postgres_core_jsonl,
normalize_imported_integer_timestamp, normalize_postgres_import_payload, normalize_imported_binary, normalize_imported_integer_timestamp,
postgres_bytea_json_value, postgres_core_export_domains, DataExportManifest, DataExportRecord, normalize_postgres_import_payload, postgres_bytea_json_value, postgres_core_export_domains,
ExportDomain, ExportRow, PostgresImportColumn, DataExportManifest, DataExportRecord, DataImportOptions, ExportDomain, ExportRow,
PostgresImportColumn,
}; };
use crate::driver::postgres::{PostgresPoolConfig, PostgresPoolFactory}; use crate::driver::postgres::{PostgresPoolConfig, PostgresPoolFactory};
use crate::lifecycle::migrate::run_migrations as run_postgres_migrations; use crate::lifecycle::migrate::run_migrations as run_postgres_migrations;
@@ -416,6 +417,159 @@ fn imported_proxy_nodes_receive_a_new_offline_tunnel_generation() {
); );
} }
#[test]
fn trusted_import_preserves_stable_credentials_only_when_explicitly_requested() {
assert!(!DataImportOptions::default().preserve_credentials);
for (table, payload) in [
("users", json!({"password_hash": "$2b$12$trusted-hash"})),
(
"public.\"api_keys\"",
json!({
"key_hash": "trusted-key-hash", "key_encrypted": "trusted-ciphertext",
"status": "active", "is_active": true, "is_locked": false,
}),
),
(
"management_tokens",
json!({"token_hash": "trusted-token-hash", "is_active": true}),
),
] {
for preserve_credentials in [false, true] {
let mut object = payload.as_object().unwrap().clone();
apply_import_credential_policy(
table,
&mut object,
|_| true,
DataImportOptions {
preserve_credentials,
},
);
if preserve_credentials {
assert_eq!(&object, payload.as_object().unwrap());
} else {
assert_ne!(&object, payload.as_object().unwrap());
}
}
}
}
#[test]
fn trusted_import_still_revokes_imported_sessions_and_live_tunnels() {
let options = DataImportOptions {
preserve_credentials: true,
};
let mut session = json!({
"refresh_token_hash": "old-session", "prev_refresh_token_hash": "older-session",
"revoked_at": null, "revoke_reason": null,
})
.as_object()
.unwrap()
.clone();
apply_import_credential_policy("public.user_sessions", &mut session, |_| true, options);
assert_ne!(session["refresh_token_hash"], json!("old-session"));
assert_eq!(session["prev_refresh_token_hash"], Value::Null);
assert_eq!(
session["revoke_reason"],
json!("imported_credentials_revoked")
);
let mut node = json!({
"tunnel_generation": "old-generation", "tunnel_connected": true,
"status": "online", "active_connections": 10,
})
.as_object()
.unwrap()
.clone();
apply_import_credential_policy("proxy_nodes", &mut node, |_| true, options);
assert_ne!(node["tunnel_generation"], json!("old-generation"));
assert_eq!(node["tunnel_connected"], json!(false));
assert_eq!(node["status"], json!("offline"));
assert_eq!(node["active_connections"], json!(0));
}
#[tokio::test]
#[ignore = "requires AETHER_TEST_POSTGRES_URL and PostgreSQL migrations"]
async fn live_import_credential_policy_round_trips_through_postgres() {
let pool = PostgresPoolFactory::new(PostgresPoolConfig {
database_url: std::env::var("AETHER_TEST_POSTGRES_URL").unwrap(),
..Default::default()
})
.unwrap()
.connect_lazy()
.unwrap();
run_postgres_migrations(&pool).await.unwrap();
for preserve_credentials in [false, true] {
let user_id = uuid::Uuid::new_v4().to_string();
let key_id = uuid::Uuid::new_v4().to_string();
let password_hash = "$2b$12$trusted-import-hash";
let key_hash = format!("trusted-{key_id}");
sqlx::query("INSERT INTO users (id, username, password_hash, auth_source, email_verified) VALUES ($1, $2, $3, 'local', FALSE)")
.bind(&user_id).bind(format!("import-{}", &user_id[..8])).bind(password_hash)
.execute(&pool).await.unwrap();
sqlx::query("INSERT INTO api_keys (id, user_id, key_hash, key_encrypted, name) VALUES ($1, $2, $3, 'trusted-ciphertext', 'Import probe')")
.bind(&key_id).bind(&user_id).bind(&key_hash).execute(&pool).await.unwrap();
let user: Value = sqlx::query_scalar("SELECT to_jsonb(users) FROM users WHERE id = $1")
.bind(&user_id)
.fetch_one(&pool)
.await
.unwrap();
let key: Value =
sqlx::query_scalar("SELECT to_jsonb(api_keys) FROM api_keys WHERE id = $1")
.bind(&key_id)
.fetch_one(&pool)
.await
.unwrap();
let input = encode_jsonl(&[
DataExportRecord::manifest(DataExportManifest::new(
1_788_739_200,
Some(DatabaseDriver::Postgres),
vec![ExportDomain::Users, ExportDomain::ApiKeys],
)),
DataExportRecord::row(ExportDomain::Users, &user_id, user),
DataExportRecord::row(ExportDomain::ApiKeys, &key_id, key),
])
.unwrap();
super::postgres::import_postgres_jsonl_with_options(
&pool,
&input,
DataImportOptions {
preserve_credentials,
},
)
.await
.unwrap();
let imported_password: String =
sqlx::query_scalar("SELECT password_hash FROM users WHERE id = $1")
.bind(&user_id)
.fetch_one(&pool)
.await
.unwrap();
let imported_key: (String, Option<String>, bool) =
sqlx::query_as("SELECT key_hash, key_encrypted, is_active FROM api_keys WHERE id = $1")
.bind(&key_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(imported_password == password_hash, preserve_credentials);
assert_eq!(imported_key.0 == key_hash, preserve_credentials);
assert_eq!(
imported_key.1.as_deref(),
preserve_credentials.then_some("trusted-ciphertext")
);
assert_eq!(imported_key.2, preserve_credentials);
sqlx::query("DELETE FROM api_keys WHERE id = $1")
.bind(&key_id)
.execute(&pool)
.await
.unwrap();
sqlx::query("DELETE FROM users WHERE id = $1")
.bind(&user_id)
.execute(&pool)
.await
.unwrap();
}
}
fn postgres_column(data_type: &str, udt_name: &str) -> PostgresImportColumn { fn postgres_column(data_type: &str, udt_name: &str) -> PostgresImportColumn {
PostgresImportColumn { PostgresImportColumn {
data_type: data_type.to_ascii_lowercase(), data_type: data_type.to_ascii_lowercase(),
@@ -1,9 +1,7 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::fs; use std::fs;
use std::path::{Path, PathBuf}; use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use sqlx::{ use sqlx::{
migrate::{AppliedMigration, Migrate}, migrate::{AppliedMigration, Migrate},
@@ -18,6 +16,8 @@ use aether_data_contracts::repository::{
}, },
}; };
use crate::lifecycle::postgres_test_support::ManagedPostgresServer;
use super::{ use super::{
postgres::{all_up_migrations, pending_migrations_from_applied, POSTGRES_MIGRATOR}, postgres::{all_up_migrations, pending_migrations_from_applied, POSTGRES_MIGRATOR},
prepare_database_for_startup, prepare_database_for_startup,
@@ -27,146 +27,6 @@ use crate::lifecycle::bootstrap::postgres::{
EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION, EMPTY_DATABASE_SNAPSHOT_SQL, EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION, EMPTY_DATABASE_SNAPSHOT_SQL,
}; };
#[derive(Debug)]
struct ManagedPostgresServer {
child: Option<Child>,
workdir: PathBuf,
database_url: String,
}
impl ManagedPostgresServer {
async fn try_start() -> Result<Option<Self>, Box<dyn std::error::Error>> {
let required = local_postgres_tests_required();
let initdb_bin = std::env::var("AETHER_INITDB_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "initdb".to_string());
let postgres_bin = std::env::var("AETHER_POSTGRES_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "postgres".to_string());
if !command_exists(&initdb_bin) || !command_exists(&postgres_bin) {
let message = format!(
"required postgres integration test binaries are unavailable: initdb={initdb_bin}, postgres={postgres_bin}"
);
if required {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, message).into());
}
eprintln!("skipping postgres integration test because {message}");
return Ok(None);
}
match Self::start(initdb_bin, postgres_bin).await {
Ok(server) => Ok(Some(server)),
Err(err)
if !required && postgres_local_startup_unavailable(err.to_string().as_str()) =>
{
eprintln!(
"skipping postgres integration test because local postgres could not start in this environment: {err}"
);
Ok(None)
}
Err(err) => Err(err),
}
}
async fn start(
initdb_bin: String,
postgres_bin: String,
) -> Result<Self, Box<dyn std::error::Error>> {
let port = reserve_local_port()?;
let workdir = std::env::temp_dir().join(format!(
"aether-migrate-tests-{}-{}",
std::process::id(),
port
));
let data_dir = workdir.join("data");
std::fs::create_dir_all(&workdir)?;
let init_output = Command::new(&initdb_bin)
.arg("-D")
.arg(&data_dir)
.arg("-U")
.arg("aether")
.arg("--auth=trust")
.arg("--encoding=UTF8")
.arg("--no-instructions")
.output()?;
if !init_output.status.success() {
return Err(std::io::Error::other(format!(
"initdb failed: {}",
String::from_utf8_lossy(&init_output.stderr)
))
.into());
}
let database_url = format!("postgres://aether@127.0.0.1:{port}/postgres");
let log_path = workdir.join("postgres.log");
let stdout = std::fs::File::create(&log_path)?;
let stderr = stdout.try_clone()?;
let mut child = Command::new(&postgres_bin)
.arg("-D")
.arg(&data_dir)
.arg("-h")
.arg("127.0.0.1")
.arg("-p")
.arg(port.to_string())
.arg("-k")
.arg(&workdir)
.arg("-F")
.arg("-c")
.arg("fsync=off")
.arg("-c")
.arg("synchronous_commit=off")
.arg("-c")
.arg("full_page_writes=off")
.arg("-c")
.arg("shared_buffers=8MB")
.arg("-c")
.arg("max_connections=8")
.arg("-c")
.arg("dynamic_shared_memory_type=mmap")
.arg("-c")
.arg("autovacuum=off")
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr))
.spawn()?;
if let Err(err) = wait_for_postgres(&database_url).await {
let _ = child.kill();
let exit_status = child
.wait()
.map(|status| status.to_string())
.unwrap_or_else(|wait_err| format!("unavailable ({wait_err})"));
let logs = fs::read_to_string(&log_path)
.unwrap_or_else(|read_err| format!("<failed to read postgres log: {read_err}>"));
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("{err}; postgres exit status: {exit_status}; logs:\n{logs}"),
)
.into());
}
Ok(Self {
child: Some(child),
workdir,
database_url,
})
}
fn database_url(&self) -> &str {
&self.database_url
}
fn stop(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
/// A clean PostgreSQL database is bootstrapped from the schema snapshot first; /// A clean PostgreSQL database is bootstrapped from the schema snapshot first;
/// migrations after the privacy/security frontier are intentionally left /// migrations after the privacy/security frontier are intentionally left
/// pending so their data-preserving changes still execute. Exercise the same /// pending so their data-preserving changes still execute. Exercise the same
@@ -191,83 +51,6 @@ async fn prepare_and_apply_clean_postgres_database(pool: &PgPool) {
); );
} }
fn local_postgres_tests_required() -> bool {
// CI can opt into failing when the isolated local PostgreSQL fixture is unavailable.
std::env::var("AETHER_REQUIRE_LOCAL_POSTGRES_TESTS")
.ok()
.is_some_and(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
}
impl Drop for ManagedPostgresServer {
fn drop(&mut self) {
self.stop();
let _ = std::fs::remove_dir_all(&self.workdir);
}
}
fn command_exists(bin: &str) -> bool {
if bin.contains(std::path::MAIN_SEPARATOR) {
return Path::new(bin).exists();
}
let Some(paths) = std::env::var_os("PATH") else {
return false;
};
std::env::split_paths(&paths).any(|path| path.join(bin).exists())
}
fn reserve_local_port() -> Result<u16, std::io::Error> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let port = listener.local_addr()?.port();
drop(listener);
Ok(port)
}
fn postgres_shared_memory_unavailable(message: &str) -> bool {
let message = message.to_ascii_lowercase();
message.contains("shared memory")
&& (message.contains("could not create shared memory segment")
|| message.contains("shmget")
|| message.contains("no space left on device"))
}
fn postgres_local_startup_unavailable(message: &str) -> bool {
let message = message.to_ascii_lowercase();
postgres_shared_memory_unavailable(&message)
|| (message.contains("timed out waiting for local postgres")
&& (message.contains("connection refused")
|| message.contains("os error 61")
|| message.contains("os error 111")))
}
async fn wait_for_postgres(database_url: &str) -> Result<(), Box<dyn std::error::Error>> {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
match PgConnection::connect(database_url).await {
Ok(connection) => {
connection.close().await?;
return Ok(());
}
Err(_) if Instant::now() < deadline => {
tokio::time::sleep(Duration::from_millis(50)).await
}
Err(err) => {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("timed out waiting for local postgres: {err}"),
)
.into())
}
}
}
}
async fn table_exists(pool: &PgPool, table_name: &str) -> Result<bool, sqlx::Error> { async fn table_exists(pool: &PgPool, table_name: &str) -> Result<bool, sqlx::Error> {
query_scalar::<_, bool>("SELECT to_regclass($1) IS NOT NULL") query_scalar::<_, bool>("SELECT to_regclass($1) IS NOT NULL")
.bind(format!("public.{table_name}")) .bind(format!("public.{table_name}"))
@@ -8,3 +8,5 @@ pub mod backfill;
pub(crate) mod bootstrap; pub(crate) mod bootstrap;
pub mod export; pub mod export;
pub mod migrate; pub mod migrate;
#[cfg(all(test, feature = "postgres"))]
mod postgres_test_support;
@@ -0,0 +1,276 @@
use std::{
path::{Path, PathBuf},
process::{Child, Command, Stdio},
time::{Duration, Instant},
};
use sqlx::{Connection, PgConnection};
#[derive(Debug)]
pub(super) struct ManagedPostgresServer {
child: Option<Child>,
pg_ctl_bin: PathBuf,
workdir: PathBuf,
data_dir: PathBuf,
database_url: String,
}
impl ManagedPostgresServer {
pub(super) async fn try_start() -> Result<Option<Self>, Box<dyn std::error::Error>> {
let required = local_postgres_tests_required();
let initdb_bin = configured_binary("AETHER_INITDB_BIN", "initdb");
let postgres_bin = configured_binary("AETHER_POSTGRES_BIN", "postgres");
let pg_ctl_bin = std::env::var("AETHER_PG_CTL_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| {
PathBuf::from(&postgres_bin).with_file_name(if cfg!(windows) {
"pg_ctl.exe"
} else {
"pg_ctl"
})
});
if !command_exists(Path::new(&initdb_bin))
|| !command_exists(Path::new(&postgres_bin))
|| !command_exists(&pg_ctl_bin)
{
let message = format!(
"required postgres integration test binaries are unavailable: initdb={initdb_bin}, postgres={postgres_bin}, pg_ctl={}",
pg_ctl_bin.display()
);
if required {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, message).into());
}
eprintln!("skipping postgres integration test because {message}");
return Ok(None);
}
match Self::start(initdb_bin, postgres_bin, pg_ctl_bin).await {
Ok(server) => Ok(Some(server)),
Err(error) if !required && postgres_local_startup_unavailable(&error.to_string()) => {
eprintln!(
"skipping postgres integration test because local postgres could not start in this environment: {error}"
);
Ok(None)
}
Err(error) => Err(error),
}
}
async fn start(
initdb_bin: String,
postgres_bin: String,
pg_ctl_bin: PathBuf,
) -> Result<Self, Box<dyn std::error::Error>> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let port = listener.local_addr()?.port();
drop(listener);
let workdir = std::env::temp_dir().join(format!(
"aether-lifecycle-tests-{}-{port}",
std::process::id()
));
std::fs::create_dir(&workdir)?;
let mut server = Self {
child: None,
pg_ctl_bin,
data_dir: workdir.join("data"),
workdir,
database_url: format!("postgres://aether@127.0.0.1:{port}/postgres"),
};
let init_output = Command::new(&initdb_bin)
.arg("-D")
.arg(&server.data_dir)
.args([
"-U",
"aether",
"--auth=trust",
"--encoding=UTF8",
"--no-instructions",
])
.output()?;
if !init_output.status.success() {
return Err(std::io::Error::other(format!(
"initdb failed: {}",
String::from_utf8_lossy(&init_output.stderr)
))
.into());
}
let log_path = server.workdir.join("postgres.log");
let stdout = std::fs::File::create(&log_path)?;
let stderr = stdout.try_clone()?;
server.child = Some(
Command::new(&postgres_bin)
.arg("-D")
.arg(&server.data_dir)
.args(["-h", "127.0.0.1", "-p"])
.arg(port.to_string())
.arg("-F")
.args(["-c", "unix_socket_directories="])
.args(["-c", "fsync=off"])
.args(["-c", "synchronous_commit=off"])
.args(["-c", "full_page_writes=off"])
.args(["-c", "shared_buffers=8MB"])
.args(["-c", "max_connections=8"])
.args(["-c", "dynamic_shared_memory_type=mmap"])
.args(["-c", "autovacuum=off"])
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr))
.spawn()?,
);
if let Err(error) = wait_for_postgres(&server.database_url).await {
let logs = std::fs::read_to_string(&log_path).unwrap_or_default();
server.stop()?;
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("{error}; postgres logs:\n{logs}"),
)
.into());
}
Ok(server)
}
pub(super) fn database_url(&self) -> &str {
&self.database_url
}
fn stop(&mut self) -> Result<(), std::io::Error> {
let Some(child) = self.child.as_mut() else {
return Ok(());
};
if child.try_wait()?.is_some() {
self.child = None;
return Ok(());
}
let output = Command::new(&self.pg_ctl_bin)
.arg("-D")
.arg(&self.data_dir)
.args(["stop", "-m", "fast", "-w", "-t", "10"])
.output()?;
if !output.status.success() && child.try_wait()?.is_none() {
return Err(std::io::Error::other(format!(
"pg_ctl stop failed for {}: {}{}",
self.data_dir.display(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
)));
}
child.wait()?;
self.child = None;
Ok(())
}
}
impl Drop for ManagedPostgresServer {
fn drop(&mut self) {
match self.stop() {
Ok(()) => {
let _ = std::fs::remove_dir_all(&self.workdir);
}
Err(error) => {
eprintln!(
"failed to stop managed postgres; preserving {}: {error}",
self.workdir.display(),
);
}
}
}
}
fn configured_binary(variable: &str, default: &str) -> String {
std::env::var(variable)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| default.to_string())
}
fn command_exists(binary: &Path) -> bool {
if binary.is_absolute() || binary.components().count() > 1 {
return binary.is_file();
}
std::env::var_os("PATH")
.is_some_and(|paths| std::env::split_paths(&paths).any(|path| path.join(binary).is_file()))
}
fn local_postgres_tests_required() -> bool {
std::env::var("AETHER_REQUIRE_LOCAL_POSTGRES_TESTS")
.ok()
.is_some_and(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
}
fn postgres_local_startup_unavailable(message: &str) -> bool {
let message = message.to_ascii_lowercase();
(message.contains("shared memory")
&& (message.contains("could not create shared memory segment")
|| message.contains("shmget")
|| message.contains("no space left on device")))
|| (message.contains("timed out waiting for local postgres")
&& (message.contains("connection refused")
|| message.contains("os error 61")
|| message.contains("os error 111")))
}
async fn wait_for_postgres(database_url: &str) -> Result<(), Box<dyn std::error::Error>> {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
match PgConnection::connect(database_url).await {
Ok(connection) => {
connection.close().await?;
return Ok(());
}
Err(_) if Instant::now() < deadline => {
tokio::time::sleep(Duration::from_millis(50)).await;
}
Err(error) => {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("timed out waiting for local postgres: {error}"),
)
.into());
}
}
}
}
#[tokio::test]
async fn managed_postgres_stops_cleanly_with_open_connections() {
let Some(mut server) = ManagedPostgresServer::try_start().await.unwrap() else {
return;
};
let connection = PgConnection::connect(server.database_url()).await.unwrap();
let workdir = server.workdir.clone();
assert!(server.data_dir.join("postmaster.pid").exists());
server.stop().unwrap();
assert!(server.child.is_none());
assert!(!server.data_dir.join("postmaster.pid").exists());
server.stop().unwrap();
drop(connection);
drop(server);
assert!(!workdir.exists());
}
#[tokio::test]
async fn failed_postgres_stop_retains_ownership_for_retry() {
let Some(mut server) = ManagedPostgresServer::try_start().await.unwrap() else {
return;
};
let pg_ctl_bin = server.pg_ctl_bin.clone();
let workdir = server.workdir.clone();
server.pg_ctl_bin = workdir.join("missing-pg-ctl");
assert!(server.stop().is_err());
assert!(server.child.as_mut().unwrap().try_wait().unwrap().is_none());
assert!(server.data_dir.exists());
server.pg_ctl_bin = pg_ctl_bin;
server.stop().unwrap();
drop(server);
assert!(!workdir.exists());
}
@@ -2752,6 +2752,40 @@ fn hydrate_client_family(item: &mut StoredRequestUsageAudit) {
} }
} }
fn merge_usage_body_capture(
incoming_body: Option<Value>,
incoming_ref: Option<String>,
incoming_state: Option<UsageBodyCaptureState>,
existing: Option<&StoredRequestUsageAudit>,
field: UsageBodyField,
) -> (Option<Value>, Option<String>, Option<UsageBodyCaptureState>) {
if matches!(
incoming_state,
Some(
UsageBodyCaptureState::None
| UsageBodyCaptureState::Disabled
| UsageBodyCaptureState::Unavailable
)
) {
return (None, None, incoming_state);
}
if incoming_body.is_some() {
return (
incoming_body,
None,
incoming_state.or(Some(UsageBodyCaptureState::Inline)),
);
}
if incoming_ref.is_some() {
return (None, incoming_ref, Some(UsageBodyCaptureState::Reference));
}
(
existing.and_then(|item| item.body_value(field).cloned()),
existing.and_then(|item| item.body_ref(field).map(ToOwned::to_owned)),
incoming_state.or_else(|| existing.and_then(|item| item.body_state(field))),
)
}
fn request_body_capture_replaces_derived_facts( fn request_body_capture_replaces_derived_facts(
request_body: Option<&Value>, request_body: Option<&Value>,
request_body_state: Option<UsageBodyCaptureState>, request_body_state: Option<UsageBodyCaptureState>,
@@ -2883,36 +2917,47 @@ impl UsageWriteRepository for InMemoryUsageReadRepository {
return Ok(existing.clone()); return Ok(existing.clone());
} }
} }
let capture_usage = sanitize_usage_capture_controls_for_persistence(capture_usage); let mut capture_usage = sanitize_usage_capture_controls_for_persistence(capture_usage);
if let Some(existing) = by_request_id.get_mut(&usage.request_id) { if let Some(existing) = by_request_id.get_mut(&usage.request_id) {
existing.request_headers = None;
existing.request_body = None;
existing.request_body_ref = None;
existing.request_body_state = None;
existing.provider_request_headers = None;
existing.provider_request_body = None;
existing.provider_request_body_ref = None;
existing.provider_request_body_state = None;
existing.response_headers = None;
existing.response_body = None;
existing.response_body_ref = None;
existing.response_body_state = None;
existing.client_response_headers = None;
existing.client_response_body = None;
existing.client_response_body_ref = None;
existing.client_response_body_state = None;
existing.request_metadata = existing.request_metadata =
sanitize_usage_request_metadata(existing.request_metadata.take()); sanitize_usage_request_metadata(existing.request_metadata.take());
} }
{ {
let mut detached_bodies = self.detached_bodies.write().expect("usage repository lock"); let mut detached_bodies = self.detached_bodies.write().expect("usage repository lock");
for field in [ for (field, state, body) in [
UsageBodyField::RequestBody, (
UsageBodyField::ProviderRequestBody, UsageBodyField::RequestBody,
UsageBodyField::ResponseBody, capture_usage.request_body_state,
UsageBodyField::ClientResponseBody, &capture_usage.request_body,
),
(
UsageBodyField::ProviderRequestBody,
capture_usage.provider_request_body_state,
&capture_usage.provider_request_body,
),
(
UsageBodyField::ResponseBody,
capture_usage.response_body_state,
&capture_usage.response_body,
),
(
UsageBodyField::ClientResponseBody,
capture_usage.client_response_body_state,
&capture_usage.client_response_body,
),
] { ] {
detached_bodies.remove(&usage_body_ref(&usage.request_id, field)); if body.is_some()
|| matches!(
state,
Some(
UsageBodyCaptureState::None
| UsageBodyCaptureState::Disabled
| UsageBodyCaptureState::Unavailable
)
)
{
detached_bodies.remove(&usage_body_ref(&usage.request_id, field));
}
} }
} }
@@ -2977,10 +3022,36 @@ impl UsageWriteRepository for InMemoryUsageReadRepository {
} }
}); });
let request_metadata = sanitize_memory_request_metadata(request_metadata); let request_metadata = sanitize_memory_request_metadata(request_metadata);
let request_body_ref = None; let (request_body, request_body_ref, request_body_state) = merge_usage_body_capture(
let provider_request_body_ref = None; capture_usage.request_body.take(),
let response_body_ref = None; capture_usage.request_body_ref.take(),
let client_response_body_ref = None; capture_usage.request_body_state,
existing.as_ref(),
UsageBodyField::RequestBody,
);
let (provider_request_body, provider_request_body_ref, provider_request_body_state) =
merge_usage_body_capture(
capture_usage.provider_request_body.take(),
capture_usage.provider_request_body_ref.take(),
capture_usage.provider_request_body_state,
existing.as_ref(),
UsageBodyField::ProviderRequestBody,
);
let (response_body, response_body_ref, response_body_state) = merge_usage_body_capture(
capture_usage.response_body.take(),
capture_usage.response_body_ref.take(),
capture_usage.response_body_state,
existing.as_ref(),
UsageBodyField::ResponseBody,
);
let (client_response_body, client_response_body_ref, client_response_body_state) =
merge_usage_body_capture(
capture_usage.client_response_body.take(),
capture_usage.client_response_body_ref.take(),
capture_usage.client_response_body_state,
existing.as_ref(),
UsageBodyField::ClientResponseBody,
);
let stored = StoredRequestUsageAudit { let stored = StoredRequestUsageAudit {
id: existing id: existing
.as_ref() .as_ref()
@@ -3089,22 +3160,38 @@ impl UsageWriteRepository for InMemoryUsageReadRepository {
), ),
status: usage.status, status: usage.status,
billing_status: usage.billing_status, billing_status: usage.billing_status,
request_headers: None, request_headers: capture_usage.request_headers.or_else(|| {
request_body: None, existing
.as_ref()
.and_then(|item| item.request_headers.clone())
}),
request_body,
request_body_ref, request_body_ref,
request_body_state: capture_usage.request_body_state, request_body_state,
provider_request_headers: None, provider_request_headers: capture_usage.provider_request_headers.or_else(|| {
provider_request_body: None, existing
.as_ref()
.and_then(|item| item.provider_request_headers.clone())
}),
provider_request_body,
provider_request_body_ref, provider_request_body_ref,
provider_request_body_state: capture_usage.provider_request_body_state, provider_request_body_state,
response_headers: None, response_headers: capture_usage.response_headers.or_else(|| {
response_body: None, existing
.as_ref()
.and_then(|item| item.response_headers.clone())
}),
response_body,
response_body_ref, response_body_ref,
response_body_state: capture_usage.response_body_state, response_body_state,
client_response_headers: None, client_response_headers: capture_usage.client_response_headers.or_else(|| {
client_response_body: None, existing
.as_ref()
.and_then(|item| item.client_response_headers.clone())
}),
client_response_body,
client_response_body_ref, client_response_body_ref,
client_response_body_state: capture_usage.client_response_body_state, client_response_body_state,
candidate_id: if replace_routing_snapshot { candidate_id: if replace_routing_snapshot {
capture_usage.candidate_id capture_usage.candidate_id
} else { } else {
@@ -136,6 +136,76 @@ fn sample_upsert_usage_record(request_id: &str) -> UpsertUsageRecord {
} }
} }
#[tokio::test]
async fn upsert_preserves_full_http_captures_across_lifecycle_updates() {
let repository = InMemoryUsageReadRepository::default();
let mut pending = sample_upsert_usage_record("req-full-capture");
pending.request_headers =
Some(json!({"content-type": "application/json", "authorization": "Bearer private"}));
pending.request_body =
Some(json!({"messages": [{"role": "user", "content": "original request"}]}));
pending.provider_request_body = Some(json!({"input": "provider request"}));
pending.request_body_state = Some(UsageBodyCaptureState::Inline);
pending.provider_request_body_state = Some(UsageBodyCaptureState::Inline);
let stored_pending = repository.upsert(pending.clone()).await.unwrap();
assert_eq!(stored_pending.request_body, pending.request_body);
assert_eq!(
stored_pending.provider_request_body,
pending.provider_request_body
);
assert_eq!(
stored_pending.request_headers,
Some(json!({"content-type": "application/json", "authorization": "[redacted]"}))
);
let mut streaming = sample_upsert_usage_record(&pending.request_id);
streaming.status = "streaming".to_string();
streaming.updated_at_unix_secs += 1;
let stored_streaming = repository.upsert(streaming).await.unwrap();
assert_eq!(stored_streaming.request_body, pending.request_body);
assert_eq!(
stored_streaming.provider_request_body,
pending.provider_request_body
);
let mut terminal = sample_upsert_usage_record(&pending.request_id);
terminal.status = "completed".to_string();
terminal.updated_at_unix_secs += 2;
terminal.finalized_at_unix_secs = Some(terminal.updated_at_unix_secs);
terminal.response_headers =
Some(json!({"content-type": "text/event-stream", "set-cookie": "private"}));
terminal.response_body = Some(json!("data: upstream response\n\ndata: [DONE]\n\n"));
terminal.client_response_body =
Some(json!({"choices": [{"message": {"content": "client response"}}]}));
let stored_terminal = repository.upsert(terminal.clone()).await.unwrap();
assert_eq!(stored_terminal.request_body, pending.request_body);
assert_eq!(
stored_terminal.provider_request_body,
pending.provider_request_body
);
assert_eq!(stored_terminal.response_body, terminal.response_body);
assert_eq!(
stored_terminal.client_response_body,
terminal.client_response_body
);
assert_eq!(
stored_terminal.response_headers,
Some(json!({"content-type": "text/event-stream", "set-cookie": "[redacted]"}))
);
let found = repository
.find_by_request_id(&pending.request_id)
.await
.unwrap()
.unwrap();
assert_eq!(found.request_body, pending.request_body);
assert_eq!(found.response_body, terminal.response_body);
assert_eq!(
repository.upsert(pending).await.unwrap().response_body,
terminal.response_body
);
}
#[tokio::test] #[tokio::test]
async fn upsert_uses_typed_provider_capture_as_the_fast_fact_snapshot() { async fn upsert_uses_typed_provider_capture_as_the_fast_fact_snapshot() {
for (name, state, incoming_tier, expected_tier) in [ for (name, state, incoming_tier, expected_tier) in [
@@ -244,9 +244,15 @@ impl GenericProviderOAuthAdapter {
fn resolve_client_secret(&self, configured: Option<String>) -> Result<String, OAuthError> { fn resolve_client_secret(&self, configured: Option<String>) -> Result<String, OAuthError> {
let configured = configured.or_else(|| { let configured = configured.or_else(|| {
(self.template.provider_type == "antigravity" let default_template = template_for_provider_type(self.template.provider_type)?;
&& self.client_id() == self.template.client_id) if self.client_id() != default_template.client_id {
.then(|| "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".to_string()) return None;
}
match self.template.provider_type {
"gemini_cli" => Some("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".to_string()),
"antigravity" => Some("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".to_string()),
_ => None,
}
}); });
required_client_secret( required_client_secret(
self.template self.template
@@ -973,54 +979,78 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn antigravity_default_credentials_support_authorization_and_refresh() { async fn google_default_credentials_support_authorization_and_refresh() {
let mut template = template_for_provider_type("antigravity").expect("template"); for provider_type in ["gemini_cli", "antigravity"] {
template.client_id_env = None; let mut template = template_for_provider_type(provider_type).expect("template");
template.client_secret_env = Some("AETHER_TEST_UNUSED_ANTIGRAVITY_SECRET"); template.client_id_env = None;
let adapter = GenericProviderOAuthAdapter::new(template); template.client_secret_env = Some("AETHER_TEST_UNUSED_ANTIGRAVITY_SECRET");
let ctx = transport_context("antigravity"); let adapter = GenericProviderOAuthAdapter::new(template);
adapter let ctx = transport_context(provider_type);
.build_authorize_url(&ctx, "state", None) adapter
.expect("authorize"); .build_authorize_url(&ctx, "state", None)
let seen_request = Arc::new(Mutex::new(None)); .expect("authorize");
let executor = StaticExecutor { let seen_request = Arc::new(Mutex::new(None));
seen_request: Arc::clone(&seen_request), let executor = StaticExecutor {
response_payload: json!({"access_token": "new-token", "expires_in": 3600}), seen_request: Arc::clone(&seen_request),
}; response_payload: json!({"access_token": "new-token", "expires_in": 3600}),
adapter };
.refresh(&executor, &ctx, &oauth_account("antigravity")) adapter
.await .refresh(&executor, &ctx, &oauth_account(provider_type))
.expect("refresh"); .await
let seen = seen_request.lock().expect("lock").clone().expect("request"); .expect("refresh");
let body = seen.body_bytes.expect("body"); let seen = seen_request.lock().expect("lock").clone().expect("request");
let fields = url::form_urlencoded::parse(&body) let body = seen.body_bytes.expect("body");
.into_owned() let fields = url::form_urlencoded::parse(&body)
.collect::<BTreeMap<_, _>>(); .into_owned()
assert_eq!(fields["client_id"], template.client_id); .collect::<BTreeMap<_, _>>();
assert_eq!( assert_eq!(fields["client_id"], template.client_id);
fields["client_secret"], assert_eq!(
adapter.resolve_client_secret(None).expect("default") fields["client_secret"],
); adapter.resolve_client_secret(None).expect("default")
assert_eq!(fields["refresh_token"], "old-refresh-token"); );
assert!(!format!("{adapter:?}").contains(&fields["client_secret"])); assert_eq!(fields["refresh_token"], "old-refresh-token");
assert!(!format!("{adapter:?}").contains(&fields["client_secret"]));
}
} }
#[test] #[test]
fn antigravity_custom_client_requires_its_own_secret() { fn google_custom_client_requires_its_own_secret() {
let adapter = GenericProviderOAuthAdapter::for_provider_type("antigravity") for provider_type in ["gemini_cli", "antigravity"] {
.expect("adapter") let adapter = GenericProviderOAuthAdapter::for_provider_type(provider_type)
.with_oauth_credentials_for_tests("custom-client", "custom-secret"); .expect("adapter")
assert!(adapter.resolve_client_secret(None).is_err()); .with_oauth_credentials_for_tests("custom-client", "custom-secret");
assert_eq!( assert!(adapter.resolve_client_secret(None).is_err());
adapter assert_eq!(
.resolve_client_secret(Some("custom-secret".to_string())) adapter
.expect("configured secret"), .resolve_client_secret(Some("custom-secret".to_string()))
"custom-secret" .expect("configured secret"),
); "custom-secret"
assert_eq!( );
adapter.client_secret().expect("override"), assert_eq!(
Some("custom-secret".to_string()) adapter.client_secret().expect("override"),
); Some("custom-secret".to_string())
);
}
}
#[test]
fn google_configured_secret_overrides_the_native_app_default() {
for provider_type in ["gemini_cli", "antigravity"] {
let adapter =
GenericProviderOAuthAdapter::for_provider_type(provider_type).expect("adapter");
assert_eq!(
adapter
.resolve_client_secret(Some("configured-secret".to_string()))
.unwrap(),
"configured-secret"
);
let mut template = template_for_provider_type(provider_type).expect("template");
template.client_id = "custom-template-client";
template.client_id_env = None;
assert!(GenericProviderOAuthAdapter::new(template)
.resolve_client_secret(None)
.is_err());
}
} }
#[test] #[test]
+111 -16
View File
@@ -11,6 +11,7 @@ use crate::wait_until;
pub struct ManagedPostgresServer { pub struct ManagedPostgresServer {
child: Option<Child>, child: Option<Child>,
postgres_bin: String, postgres_bin: String,
pg_ctl_bin: PathBuf,
port: u16, port: u16,
workdir: PathBuf, workdir: PathBuf,
data_dir: PathBuf, data_dir: PathBuf,
@@ -26,7 +27,7 @@ impl ManagedPostgresServer {
port port
)); ));
let data_dir = workdir.join("data"); let data_dir = workdir.join("data");
std::fs::create_dir_all(&workdir)?; std::fs::create_dir(&workdir)?;
let initdb_bin = std::env::var("AETHER_INITDB_BIN") let initdb_bin = std::env::var("AETHER_INITDB_BIN")
.ok() .ok()
@@ -36,10 +37,31 @@ impl ManagedPostgresServer {
.ok() .ok()
.filter(|value| !value.trim().is_empty()) .filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "postgres".to_string()); .unwrap_or_else(|| "postgres".to_string());
let pg_ctl_bin = std::env::var("AETHER_PG_CTL_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| {
PathBuf::from(&postgres_bin).with_file_name(if cfg!(windows) {
"pg_ctl.exe"
} else {
"pg_ctl"
})
});
let database_url = format!("postgres://aether@127.0.0.1:{port}/postgres");
let mut server = Self {
child: None,
postgres_bin,
pg_ctl_bin,
port,
workdir,
data_dir,
database_url,
};
let init_output = Command::new(&initdb_bin) let init_output = Command::new(&initdb_bin)
.arg("-D") .arg("-D")
.arg(&data_dir) .arg(&server.data_dir)
.arg("-U") .arg("-U")
.arg("aether") .arg("aether")
.arg("--auth=trust") .arg("--auth=trust")
@@ -54,15 +76,6 @@ impl ManagedPostgresServer {
.into()); .into());
} }
let database_url = format!("postgres://aether@127.0.0.1:{port}/postgres");
let mut server = Self {
child: None,
postgres_bin,
port,
workdir,
data_dir,
database_url,
};
server.restart().await?; server.restart().await?;
Ok(server) Ok(server)
} }
@@ -76,10 +89,29 @@ impl ManagedPostgresServer {
} }
pub fn stop(&mut self) -> Result<(), std::io::Error> { pub fn stop(&mut self) -> Result<(), std::io::Error> {
if let Some(mut child) = self.child.take() { let Some(child) = self.child.as_mut() else {
let _ = child.kill(); return Ok(());
let _ = child.wait(); };
if child.try_wait()?.is_some() {
self.child = None;
return Ok(());
} }
let output = Command::new(&self.pg_ctl_bin)
.arg("-D")
.arg(&self.data_dir)
.args(["stop", "-m", "fast", "-w", "-t", "10"])
.output()?;
if !output.status.success() && child.try_wait()?.is_none() {
return Err(std::io::Error::other(format!(
"pg_ctl stop failed for {}: {}{}",
self.data_dir.display(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
)));
}
child.wait()?;
self.child = None;
Ok(()) Ok(())
} }
@@ -139,8 +171,17 @@ impl ManagedPostgresServer {
impl Drop for ManagedPostgresServer { impl Drop for ManagedPostgresServer {
fn drop(&mut self) { fn drop(&mut self) {
let _ = self.stop(); match self.stop() {
let _ = std::fs::remove_dir_all(&self.workdir); Ok(()) => {
let _ = std::fs::remove_dir_all(&self.workdir);
}
Err(error) => {
eprintln!(
"failed to stop managed postgres; preserving {}: {error}",
self.workdir.display(),
);
}
}
} }
} }
@@ -170,3 +211,57 @@ fn reserve_local_port() -> Result<u16, std::io::Error> {
drop(listener); drop(listener);
Ok(port) Ok(port)
} }
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
#[ignore = "requires local initdb, postgres, and pg_ctl binaries"]
async fn live_managed_postgres_restarts_cleanly_with_open_connections() {
let mut server = ManagedPostgresServer::start().await.unwrap();
let workdir = server.workdir.clone();
let mut connection = PgConnection::connect(server.database_url()).await.unwrap();
sqlx::query("CREATE TABLE restart_probe (value INTEGER NOT NULL)")
.execute(&mut connection)
.await
.unwrap();
sqlx::query("INSERT INTO restart_probe VALUES (42)")
.execute(&mut connection)
.await
.unwrap();
for _iteration in 0..4 {
server.stop().unwrap();
server.stop().unwrap();
assert!(server.child.is_none());
assert!(!server.data_dir.join("postmaster.pid").exists());
assert!(server.data_dir.exists());
server.restart().await.unwrap();
connection = PgConnection::connect(server.database_url()).await.unwrap();
let value: i32 = sqlx::query_scalar("SELECT value FROM restart_probe")
.fetch_one(&mut connection)
.await
.unwrap();
assert_eq!(value, 42);
}
drop(server);
assert!(!workdir.exists());
}
#[tokio::test]
#[ignore = "requires local initdb, postgres, and pg_ctl binaries"]
async fn live_failed_postgres_stop_can_be_retried_without_losing_ownership() {
let mut server = ManagedPostgresServer::start().await.unwrap();
let pg_ctl_bin = server.pg_ctl_bin.clone();
server.pg_ctl_bin = server.workdir.join("missing-pg-ctl");
assert!(server.stop().is_err());
assert!(server.child.as_mut().unwrap().try_wait().unwrap().is_none());
assert!(server.data_dir.exists());
server.pg_ctl_bin = pg_ctl_bin;
server.stop().unwrap();
assert!(server.child.is_none());
assert!(!server.data_dir.join("postmaster.pid").exists());
}
}
+2 -37
View File
@@ -2726,25 +2726,8 @@ fn headers_to_json(headers: &BTreeMap<String, String>) -> Option<Value> {
const REDACTED_USAGE_VALUE: &str = "[redacted]"; const REDACTED_USAGE_VALUE: &str = "[redacted]";
/// Only headers whose values are protocol metadata are persisted verbatim.
/// Unknown headers are treated as credentials because providers commonly use
/// custom `X-*` names for authentication.
const SAFE_USAGE_HEADER_VALUE_NAMES: &[&str] = &[
"accept",
"accept-encoding",
"content-encoding",
"content-length",
"content-type",
"transfer-encoding",
"x-request-id",
"x-trace-id",
];
fn is_sensitive_header(name: &str) -> bool { fn is_sensitive_header(name: &str) -> bool {
let trimmed = name.trim(); aether_data_contracts::repository::usage::usage_header_value_is_sensitive(name)
!SAFE_USAGE_HEADER_VALUE_NAMES
.iter()
.any(|candidate| trimmed.eq_ignore_ascii_case(candidate))
} }
fn mask_header_value(name: &str, value: &str) -> String { fn mask_header_value(name: &str, value: &str) -> String {
@@ -2761,25 +2744,7 @@ fn mask_sensitive_header_value(_value: &str) -> String {
/// Non-object values cannot be established as a valid header map and are /// Non-object values cannot be established as a valid header map and are
/// discarded instead of being persisted verbatim. /// discarded instead of being persisted verbatim.
fn mask_sensitive_headers_in_json_value(value: Option<Value>) -> Option<Value> { fn mask_sensitive_headers_in_json_value(value: Option<Value>) -> Option<Value> {
let mut value = value?; aether_data_contracts::repository::usage::sanitize_usage_headers_for_persistence(value)
let Value::Object(map) = &mut value else {
return None;
};
for (key, val) in map.iter_mut() {
if !is_sensitive_header(key) {
continue;
}
match val {
Value::String(text) => {
*text = mask_sensitive_header_value(text);
}
Value::Null => {}
other => {
*other = Value::String(mask_sensitive_header_value(&other.to_string()));
}
}
}
Some(value)
} }
fn mask_sensitive_body_fields(mut value: Value) -> Value { fn mask_sensitive_body_fields(mut value: Value) -> Value {
+61 -4
View File
@@ -4,6 +4,7 @@ use aether_data_contracts::repository::video_tasks::{
}; };
use serde_json::{json, Map, Value}; use serde_json::{json, Map, Value};
use crate::transport::{gemini_response_video_url, gemini_video_metadata};
use crate::types::sanitize_video_task_error_code; use crate::types::sanitize_video_task_error_code;
use crate::{ use crate::{
build_video_follow_up_report_context, current_unix_timestamp_secs, gemini_metadata_video_url, build_video_follow_up_report_context, current_unix_timestamp_secs, gemini_metadata_video_url,
@@ -112,7 +113,16 @@ impl GeminiVideoTaskSeed {
self.error_code = None; self.error_code = None;
self.error_message = None; self.error_message = None;
} }
self.metadata = json!({}); self.metadata = if error.is_none() {
gemini_video_metadata(
provider_body
.get("response")
.and_then(gemini_response_video_url)
.as_deref(),
)
} else {
json!({})
};
return; return;
} }
@@ -365,8 +375,8 @@ mod tests {
use serde_json::json; use serde_json::json;
use crate::{ use crate::{
GeminiVideoTaskSeed, LocalVideoTaskPersistence, LocalVideoTaskStatus, GeminiVideoTaskSeed, LocalVideoTaskPersistence, LocalVideoTaskSnapshot,
LocalVideoTaskTransport, LocalVideoTaskStatus, LocalVideoTaskTransport,
}; };
use super::map_gemini_stored_task_to_read_response; use super::map_gemini_stored_task_to_read_response;
@@ -484,7 +494,54 @@ mod tests {
assert!(record.request_metadata.is_none()); assert!(record.request_metadata.is_none());
assert_eq!( assert_eq!(
record.video_url.as_deref(), record.video_url.as_deref(),
Some("https://files.example/video.mp4?alt=media") Some("https://files.example/video.mp4?alt=media&token=sensitive")
); );
assert_eq!(record.prompt.as_deref(), Some("business prompt"));
let transport = seed.transport.clone();
let mut snapshot = LocalVideoTaskSnapshot::Gemini(seed);
snapshot.apply_provider_body(json!({
"done": true,
"debug": "private-provider-debug",
"response": {
"provider_token": "private-provider-token",
"generateVideoResponse": {
"generatedSamples": [{
"video": {
"uri": "https://files.example/video.mp4?alt=media&token=signed%2Bvalue",
"debug": "private-video-debug"
}
}]
}
}
}).as_object().expect("provider body"));
assert!(!serde_json::to_string(&snapshot)
.expect("serialize snapshot")
.contains("private-"));
snapshot.sanitize_persisted_diagnostics();
let stored = snapshot.to_upsert_record().into_stored();
assert_eq!(stored.status, VideoTaskStatus::Completed);
assert_eq!(stored.prompt.as_deref(), Some("business prompt"));
assert_eq!(
stored.video_url.as_deref(),
Some("https://files.example/video.mp4?alt=media&token=signed%2Bvalue")
);
assert!(stored.request_metadata.is_none());
assert!(stored.original_request_body.is_none());
let mut restored =
LocalVideoTaskSnapshot::from_stored_task_with_transport(&stored, transport)
.expect("stored Gemini task should reconstruct");
restored.sanitize_persisted_diagnostics();
let restored_record = restored.to_upsert_record();
assert_eq!(restored_record.video_url, stored.video_url);
assert_eq!(restored_record.prompt, stored.prompt);
let response = restored.read_response();
assert_eq!(response.body_json["done"], true);
assert!(response
.body_json
.to_string()
.contains("/v1beta/files/aev_"));
assert!(!response.body_json.to_string().contains("signed%2Bvalue"));
} }
} }
+28 -4
View File
@@ -610,7 +610,7 @@ impl OpenAiVideoTaskSeed {
updated_at_unix_secs: self.completed_at_unix_secs.unwrap_or(now_unix_secs), updated_at_unix_secs: self.completed_at_unix_secs.unwrap_or(now_unix_secs),
error_code: self.error_code.clone(), error_code: self.error_code.clone(),
error_message: None, error_message: None,
video_url: None, video_url: self.video_url.clone(),
request_metadata: None, request_metadata: None,
}; };
record.sanitize_for_persistence(); record.sanitize_for_persistence();
@@ -626,8 +626,8 @@ mod tests {
use serde_json::json; use serde_json::json;
use crate::{ use crate::{
LocalVideoTaskPersistence, LocalVideoTaskStatus, LocalVideoTaskTransport, LocalVideoTaskContentAction, LocalVideoTaskPersistence, LocalVideoTaskSnapshot,
OpenAiVideoTaskSeed, LocalVideoTaskStatus, LocalVideoTaskTransport, OpenAiVideoTaskSeed,
}; };
use super::map_openai_stored_task_to_read_response; use super::map_openai_stored_task_to_read_response;
@@ -751,9 +751,33 @@ mod tests {
assert!(record.original_request_body.is_none()); assert!(record.original_request_body.is_none());
assert!(record.progress_message.is_none()); assert!(record.progress_message.is_none());
assert!(record.error_message.is_none()); assert!(record.error_message.is_none());
assert!(record.video_url.is_none()); assert_eq!(record.video_url, seed.video_url);
assert_eq!(record.prompt, seed.prompt);
assert!(record.request_metadata.is_none()); assert!(record.request_metadata.is_none());
assert_eq!(record.duration_seconds, Some(4)); assert_eq!(record.duration_seconds, Some(4));
assert_eq!(record.size.as_deref(), Some("1280x720")); assert_eq!(record.size.as_deref(), Some("1280x720"));
let mut stored = record.into_stored();
stored.status = VideoTaskStatus::Completed;
let snapshot =
LocalVideoTaskSnapshot::from_stored_task_with_transport(&stored, seed.transport)
.expect("stored task should reconstruct with current transport");
let LocalVideoTaskSnapshot::OpenAi(restored) = snapshot else {
panic!("expected OpenAI snapshot");
};
assert_eq!(restored.prompt, stored.prompt);
assert_eq!(restored.to_upsert_record().video_url, stored.video_url);
let Some(LocalVideoTaskContentAction::StreamPlan(plan)) =
restored.build_content_stream_action(None, "trace-download")
else {
panic!("completed stored task should stream content");
};
assert_eq!(Some(plan.url.as_str()), stored.video_url.as_deref());
assert!(plan.headers.is_empty());
let response = map_openai_stored_task_to_read_response(stored.clone());
assert_eq!(
response.body_json["video_url"].as_str(),
stored.video_url.as_deref()
);
} }
} }
@@ -1,6 +1,7 @@
use aether_data_contracts::repository::video_tasks::{StoredVideoTask, UpsertVideoTask}; use aether_data_contracts::repository::video_tasks::{StoredVideoTask, UpsertVideoTask};
use serde_json::{json, Map, Value}; use serde_json::{json, Map, Value};
use crate::transport::{gemini_metadata_video_url, gemini_video_metadata};
use crate::types::sanitize_video_task_error_code; use crate::types::sanitize_video_task_error_code;
use crate::{ use crate::{
local_status_from_stored, non_empty_owned, request_body_string, GeminiVideoTaskSeed, local_status_from_stored, non_empty_owned, request_body_string, GeminiVideoTaskSeed,
@@ -104,7 +105,7 @@ impl LocalVideoTaskSnapshot {
progress_percent: task.progress_percent, progress_percent: task.progress_percent,
error_code: task.error_code.clone(), error_code: task.error_code.clone(),
error_message: task.error_message.clone(), error_message: task.error_message.clone(),
metadata: Value::Object(Map::new()), metadata: gemini_video_metadata(task.video_url.as_deref()),
persistence, persistence,
transport, transport,
})) }))
@@ -128,7 +129,8 @@ impl LocalVideoTaskSnapshot {
let previous_error_code = seed.error_code.clone(); let previous_error_code = seed.error_code.clone();
let error_code = let error_code =
sanitized_error_code_for_status(seed.status, seed.error_code.take()); sanitized_error_code_for_status(seed.status, seed.error_code.take());
let safe_metadata = Value::Object(Map::new()); let safe_metadata =
gemini_video_metadata(gemini_metadata_video_url(&seed.metadata).as_deref());
let changed = previous_error_code != error_code let changed = previous_error_code != error_code
|| seed.error_message.is_some() || seed.error_message.is_some()
|| seed.metadata != safe_metadata; || seed.metadata != safe_metadata;
@@ -450,6 +450,49 @@ mod tests {
}) })
} }
#[test]
fn completed_gemini_download_survives_encrypted_file_reload() {
let path = temp_store_path("completed-download");
let video_url = "https://cdn.example.test/video.mp4?signature=a%2Fb%2Bc%3D";
let mut snapshot = sensitive_gemini_snapshot();
snapshot.apply_provider_body(
json!({
"done": true,
"debug": "private-debug",
"response": {
"generateVideoResponse": {
"generatedSamples": [{"video": {"uri": video_url}}]
}
}
})
.as_object()
.expect("provider body"),
);
let store = FileVideoTaskStore::new(&path, DEVELOPMENT_ENCRYPTION_KEY).expect("store");
store.insert(snapshot);
drop(store);
let bytes = std::fs::read_to_string(&path).expect("encrypted store file");
assert!(bytes.starts_with(ENCRYPTED_VIDEO_TASK_STORE_PREFIX));
assert!(!bytes.contains(video_url));
assert!(!bytes.contains("transport-key-required-for-resume"));
let restored =
FileVideoTaskStore::new(&path, DEVELOPMENT_ENCRYPTION_KEY).expect("restored store");
let task = restored
.clone_gemini("task-sensitive")
.expect("completed Gemini task");
let record = task.to_upsert_record();
assert_eq!(
record.status,
aether_data_contracts::repository::video_tasks::VideoTaskStatus::Completed
);
assert_eq!(record.video_url.as_deref(), Some(video_url));
assert_eq!(record.prompt.as_deref(), Some("create a video"));
assert!(!task.metadata.to_string().contains("private-debug"));
assert!(record.request_metadata.is_none());
drop(restored);
cleanup_store_path(&path);
}
#[test] #[test]
fn loading_encrypted_store_rewrites_legacy_provider_diagnostics() { fn loading_encrypted_store_rewrites_legacy_provider_diagnostics() {
let path = temp_store_path("diagnostic-migration"); let path = temp_store_path("diagnostic-migration");
@@ -1,4 +1,4 @@
use serde_json::Value; use serde_json::{json, Value};
use crate::LocalVideoTaskStatus; use crate::LocalVideoTaskStatus;
@@ -20,9 +20,12 @@ pub fn parse_video_content_variant(query_string: Option<&str>) -> Option<&'stati
} }
pub fn gemini_metadata_video_url(metadata: &Value) -> Option<String> { pub fn gemini_metadata_video_url(metadata: &Value) -> Option<String> {
metadata metadata.get("response").and_then(gemini_response_video_url)
.get("response") }
.and_then(|value| value.get("generateVideoResponse"))
pub(crate) fn gemini_response_video_url(response: &Value) -> Option<String> {
response
.get("generateVideoResponse")
.and_then(|value| value.get("generatedSamples")) .and_then(|value| value.get("generatedSamples"))
.and_then(Value::as_array) .and_then(Value::as_array)
.and_then(|value| value.first()) .and_then(|value| value.first())
@@ -32,6 +35,19 @@ pub fn gemini_metadata_video_url(metadata: &Value) -> Option<String> {
.map(str::to_string) .map(str::to_string)
} }
pub(crate) fn gemini_video_metadata(video_url: Option<&str>) -> Value {
match video_url {
Some(video_url) => json!({
"response": {
"generateVideoResponse": {
"generatedSamples": [{"video": {"uri": video_url}}]
}
}
}),
None => json!({}),
}
}
pub fn map_openai_task_status(status: LocalVideoTaskStatus) -> &'static str { pub fn map_openai_task_status(status: LocalVideoTaskStatus) -> &'static str {
match status { match status {
LocalVideoTaskStatus::Submitted | LocalVideoTaskStatus::Queued => "queued", LocalVideoTaskStatus::Submitted | LocalVideoTaskStatus::Queued => "queued",
@@ -98,10 +98,15 @@ impl LocalVideoTaskPersistence {
api_key_name: task.api_key_name.clone(), api_key_name: task.api_key_name.clone(),
client_api_format, client_api_format,
provider_api_format, provider_api_format,
original_request_body: task original_request_body: task.original_request_body.clone().unwrap_or_else(|| {
.original_request_body serde_json::json!({
.clone() "prompt": task.prompt,
.unwrap_or_else(|| Value::Object(Map::new())), "seconds": task.duration_seconds,
"resolution": task.resolution,
"aspect_ratio": task.aspect_ratio,
"size": task.size,
})
}),
format_converted: task.format_converted, format_converted: task.format_converted,
}) })
} }
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,98 @@
# 安全加固兼容性复核(2026-09-07)
后续扩展到全仓的检查范围、新发现、逐文件清单及限制见 `security-hardening-full-audit.md``security-hardening-audit-manifest.tsv`
## 范围
本轮针对 `579f2c7cc`(2026-09-04)及其后续修复进行复核。该提交涉及 1019 个文件,混合了安全边界、订阅计费、数据持久化和运行时变更,不适合整体 revert。
本轮重点检查管理端脱敏与编辑往返、Provider OAuth 默认配置、已有 DNS/代理兼容性修复、管理员错误诊断、SMTP/LDAP/OAuth 配置、导入导出及请求大小限制。以下是已确认的问题及处理,不代表对全部文件逐行审计或对生产环境的完整验证。
## 新增:管理员按需查看规则原值
- 端点管理的请求/响应规则工具栏增加“查看原值”。只读弹窗展示服务端已保存的请求头、请求体和响应头规则,不覆盖未保存的草稿。
- 新接口:`GET /api/admin/endpoints/{endpoint_id}/rules/reveal`
- 仅管理员可访问;管理令牌需要 `admin:endpoints_manage:admin`,只读和普通写权限不足。
- 返回 `Cache-Control: no-store``Pragma: no-cache`,记录 `admin_endpoint_rules_revealed` 审计事件。响应仅包含三组规则,不附带端点的其他配置或凭据。
- 关闭弹窗、切换端点或卸载组件时取消请求并清除明文;旧请求的迟到结果不会重新显示。
## 本轮修复的六类误伤
| 问题 | 影响 | 处理 |
| --- | --- | --- |
| 普通协议头也全部脱敏 | `Content-Type``User-Agent`、版本/客户端信息及相应条件无法正常查看;配置导出也受影响 | 恢复明确的常用非凭据头的显示;认证头、Cookie 和未知自定义头继续默认隐藏。旧版本返回的保留标记仍可正常保存 |
| 重复规则无法恢复原值 | 同一头或路径配置多条条件规则后,保存或重排可能将原值变成 `***` | 在相同操作和目标范围内按可区分的规则结构匹配;未修改的重复规则保留原顺序与原值。无法唯一定位的脱敏编辑明确报错,禁止静默写入占位符 |
| 嵌套条件组丢失原值 | 多层 `all`/`any` 在修改其他条件或重排后,内部脱敏值无法恢复 | 增加递归条件组匹配、未修改条件组往返保留和保留值校验 |
| URL 类型判断过宽 | `image_url` 等对象/数组被变成 `null`,内嵌 `data:` 图片 URL 也受影响,继续编辑可能破坏请求规则 | 保留结构化 URL 和内嵌数据;递归处理网络 URL,继续隐藏并在保存时恢复网络凭据 |
| 保存后的编辑状态没有同步 | 后台返回脱敏数据后,界面仍保留提交前明文,持续显示“未保存” | 使用保存响应重新建立编辑基准;保存期间的新编辑不被覆盖,关闭后清除草稿,忽略跨弹窗的旧响应 |
| Gemini CLI 默认 OAuth 被误移除 | 未额外配置环境变量时,原有默认客户端不能授权或刷新 | 恢复加固前内置 native-app 客户端的配套默认值;显式配置优先,自定义客户端 ID 仍必须提供自己的凭据,调试输出继续脱敏 |
明确可直接显示的头包括 `Accept`/编码/语言、`Content-Type`/编码、`Cache-Control``User-Agent`、Anthropic/OpenAI 协议版本与 beta 标记,以及明确列出的 `x-stainless-*` 客户端运行时字段;不是按前缀放行任意自定义头。
## 补充修复:显式 `full` 请求记录被禁用
原则:安全处理应约束未授权访问、未配置时的默认行为和真实凭据泄露,不应静默覆盖管理员明确启用的功能。
本次确认不是单纯的前端隐藏,而是同一链路上的多重清空:
- `request_record_level=full` 在运行时被强制解析为 `basic`
- 独立 HTTP 审计存储的输入投影清除了所有请求头、正文、正文引用和采集状态。
- 内存仓库每次更新都丢弃正文与引用;PostgreSQL 的单条写入把正文 blob 同步改成删除,并拒绝 HTTP 审计内容。
处理:恢复显式 `full`(含旧配置名 `request_log_level`)的采集与独立存储,保留当前配置名优先;恢复请求、上游请求、上游响应和客户端响应各方向已有的采集内容,流转状态更新不再无条件删除它们。PostgreSQL 正文仍写入原有压缩 blob 表,HTTP 头与引用仍写入独立审计表,不重新塞回计费主表。
保留:缺失、无效配置和读取配置失败时默认 `basic``basic` 不记录正文;OAuth 令牌交换等内部凭据请求不采集正文;认证头及未知自定义头继续脱敏;正文引用仍校验所属请求和字段;管理端权限、审计及无缓存策略不变。普通同格式非流式响应仍只保存一份原有响应体,前端沿用回退显示,不制造重复的客户端副本。
新增回归覆盖配置解析、内存生命周期、流式/非流式管理端正文读取、旧正文大小配置不覆盖 `full`、PostgreSQL 单条/批量写入与正文读回、前端按需加载与无正文时的展示。真实数据库测试使用本机临时隔离 PostgreSQL,不访问现有业务数据库。
此修复只能恢复之后新采集的记录;此前未采集或已删除的正文无法由代码补回。运行时沿用原有的 30 秒采集策略缓存,刚切换记录级别时需等待缓存刷新。
扩大执行原有、默认忽略的 PostgreSQL 测试时,另发现 5 个旧用例的测试数据/类型与当前 schema 不兼容:4 个使用超过 `varchar(36)` 限制的 Provider Key ID1 个直接按 `f64` 解码 `NUMERIC` 列。它们并非本次正文修复引入,也不作为本次已通过项;未修改相关业务逻辑或测试夹具。
## 继续复核:视频任务链路的四类回归
继续沿“业务字段被当作诊断数据清空”“读取投影与更新条件不一致”检查,另确认以下四类问题,均可定位到本次加固新增的清空或校验条件,不是用户配置错误:
| 问题 | 实际影响 | 本轮修复 |
| --- | --- | --- |
| 任务业务信息被清空 | 提示词、用户名和客户端 Key 名称丢失,管理页显示空白或 `Unknown`;提示词被写成 `NULL` 还与 PostgreSQL 的非空约束冲突,可能直接阻止任务入库 | 保留这些明确用于任务展示的业务字段,不再作为敏感诊断一律删除 |
| 下载地址被清空或破坏签名 | OpenAI 任务的 `video_url` 被无条件丢弃;Gemini 地址仅保留 `alt=media`,使下载签名、有效期等参数失效 | 保留 HTTP(S) 产物地址及完整查询串,不重排重复参数、不重编码签名;继续拒绝非法协议和 URL userinfo,移除 fragment |
| Gemini 完成结果和重载丢失 | 完成轮询直接清空元数据;数据库重建与加密文件加载再次清空结果 URI,出现任务已完成但无法取得视频 | 仅保留下载所需的最小结果 URI,不保留上游完整响应;从数据库的业务字段重建必要的提示词、尺寸信息和结果 URI,保证再次保存不会丢失 |
| PostgreSQL 轮询更新条件自相矛盾 | 领取任务时不读出时长、分辨率、宽高比和尺寸,而加固后的更新要求这些不可变字段逐项相等,导致正常轮询更新被拒绝、任务停留在处理中 | 领取时读回必要业务字段,保留原有归属/身份校验和并发领取锁,不靠放宽更新条件绕过问题 |
保留原有播放/下载接口与鉴权语义:签名产物链接是有权限用户访问任务结果所需的业务数据,不等同于可随意删除的调试凭据。OpenAI 直接下载不附带 Provider 认证头;Gemini 仍通过网关文件接口访问,并校验上游地址同源后才附加 Provider Key。私网/保留地址拦截、跨域凭据隔离、任务归属校验及管理操作审计不撤销。
任务数据库仍不保存完整请求体、原始错误消息或含认证头的传输快照;本地文件仍使用原有认证加密格式,调试输出继续脱敏。回归覆盖任务保存后重读、签名参数顺序与编码、加密文件重载、管理列表/详情、带审计的下载,以及轮询前后提示词和尺寸不丢失。
真实 PostgreSQL 回归使用临时 Unix socket 实例,并从正式迁移后的 schema 复制会话隔离的任务表。OpenAI/Gemini 两种任务均验证了入库、领取、正常完成更新、拒绝不匹配的尺寸更新和重新查询签名地址;不连接现有业务数据库。
这四类是本轮已经确认的新增回归,不代表对混合提交全部 1019 个文件给出“零遗漏”保证。原有用户策略字段停用早于本次加固,不据此回退;后台任务和缓存中的裁剪未发现足以确认本次功能回归的调用链,不做猜测性恢复。数据库仍保存的历史值可以重新读取;已经写空、未入库或已过期的资源不能凭本次源码修复重建。
## 已存在的后续修复:保留,不重复撤销
- `522b97905`:已移除普通 Provider 代理的 DNS 地址过滤和相关白名单设置。
- `696273122`:已恢复 Antigravity 默认 native-app OAuth 配套凭据;本轮将同类遗漏补到 Gemini CLI。
- `a26680f46`:已恢复旧 SMTP 密码迁移。
- `062e111c0`:已恢复管理员查看上游错误诊断的能力,同时保留普通用户侧脱敏。
- `14f96c9fa` 等:已修复脱敏后的密钥健康状态摘要,不将其退回旧实现。
## 保留的安全边界
- 管理员/普通用户与管理令牌的权限隔离;凭据查看接口的审计和无缓存要求。
- 凭据加密存储及凭据与目标/身份绑定;未知头和真实敏感字段的默认脱敏。
- 登录 OAuth、支付、隧道中继等独立网络边界,TLS 校验和隧道防重放。
- 有界请求/响应缓冲、备份恢复模式、导入校验、计费与配额一致性;本轮没有因“安全加固”标签撤回这些功能。
## 回归覆盖与使用限制
回归用例覆盖规则投影/恢复、重复与嵌套规则、占位符拒绝、结构化 URL、保存期间的并发编辑、弹窗关闭/切换时的迟到响应、接口权限/审计/无缓存,以及 Gemini CLI 和 Antigravity 的默认配置与显式覆盖。
早期兼容性修复的阶段性验证记录(最终全量结果见 `security-hardening-full-audit.md`):
- 视频数据契约 12 项、视频核心 33 项(含加密文件重载)、内存视频仓库 11 项、PostgreSQL 视频仓库 9 项、Provider 视频传输 9 项均通过。
- 新增的真实 PostgreSQL 回归单独执行通过,同时覆盖 OpenAI 和 Gemini;临时数据库实例已停止并清理。
- 网关 `video` 101 项、`async_task::` 20 项、`usage` 302 项、端点规则原值查看 3 项、管理令牌权限 28 项均通过;筛选结果可能重叠,不合计为独立用例总数。
- 前端 199 个测试文件、1411 项测试通过;补齐保存测试的类型夹具后,该文件 4 项测试及 ESLint 再次通过;Rust 格式检查和 `git diff --check` 通过。
- 阶段性检查曾发现 461 条既有类型诊断,与未修改 HEAD 的同依赖基线一致。2026-09-07 的“全部修复”续轮已修正这些诊断,并将 `npm run type-check` 接入 `vue-tsc -b --force --pretty false`;当前真实全量类型检查为 0 错误。最新全量测试、构建及剩余验证边界以 `security-hardening-full-audit.md` 为准,未关闭严格模式或排除测试。
上述修复与回归不涉及生产服务器部署或现有业务数据库更改;源码提交、推送不代表生产环境已经部署或完成验证。若历史版本已经把真实值覆盖为字面量 `***`,查看接口不能重建丢失的原值,需重新填写或从可靠备份恢复。
@@ -0,0 +1,134 @@
# 安全加固全仓兼容性复核(2026-09-07)
## 范围与方法
- 加固基线:`579f2c7cc`2026-09-04);复核时 HEAD`a5c3699ae`。保留已有未提交的兼容性修复,不整体回退该混合提交,也不撤销后续修复。
- 历史提交涉及 1019 个文件,其中 907 个路径仍存在,112 个已在后续重构中移除。逐路径清单见 `security-hardening-audit-manifest.tsv`;已移除的旧数据库适配器、未发布迁移等不凭历史清单重新引入。
- 对复核时 Git 跟踪的 2843 个 Rust、TypeScript、Vue、JavaScript、Python、SQL 和 Shell 源文件进行清单、读取与模式扫描,并检查加固差异、后续修订及关键生产调用链。正文清空、无条件禁用、脱敏、URL 拒绝、权限与数据归属等是重点搜索模式。
- 清单中的 `heuristic_added_risk_lines` 是启发式候选行数,含初始化代码和内联测试,**不是 Bug 数、漏洞数或逐行人工审计完成标记**。全仓扫描、模块复核、回归测试是不同的覆盖层次,不等于人工精读所有源文件,也不承诺零遗漏。
- 判断标准:恢复管理员明确启用的功能和必要业务数据;保留未授权访问阻断、默认保护、凭据隔离和有界资源使用。不因一条旧测试要求“所有内容都为空”,就把正常功能重新禁用。
之前的规则、OAuth、`full` 记录及视频四类修复详见 `security-hardening-compatibility-review.md`。本报告只把本次扩展检查的新发现单独计数。
## 本次新增确认并修复的两类加固回归
### 1. 正文的“压缩保留”被替换成删除
涉及 `crates/aether-data/adapters/postgres/src/usage/cleanup.rs`
原本分为详细正文、压缩正文、请求头、计费记录等独立保留周期。加固后,详细正文清理不再压缩迁移,而是删除正文、独立 blob 与审计引用;选择条件还包含已经独立存储的正文。因此采用默认 7/30 天设置时,明确启用 `full` 后保存的正文也会在约 7 天后提前失去,而不是按压缩正文的 30 天保留。
修复内容:
- 详细正文到期只处理主表中的旧内联/压缩数据,将四个方向的正文迁入独立 gzip blob,并保留 HTTP 审计引用;不选择已经迁出的正文反复处理。
- 旧 metadata 中的正文引用按当前请求和字段校验后迁入审计表,删除旧 metadata 键,但不删除实际正文。无效、跨请求及跨字段引用不恢复。
- 每条迁移在事务中锁定并重读来源行。正文写入或引用更新失败时回滚,不先清空原值;其他 metadata 内容保留。
- 预览统计与实际清理选择条件一致。压缩正文到期仍删除;明确选择“立即清理正文”的操作仍执行删除,未把隐私清理功能改成永远保留。
真实 PostgreSQL 回归使用正式迁移后的 schema 与会话隔离的临时表,覆盖四个方向的内联 JSON、旧 gzip、独立 blob、旧引用迁移、外部请求引用拒绝、7/30 天区间、过期删除、重复清理幂等、显式即时删除和中途写入失败后的事务回滚。不会读取或修改业务数据库。
同时修复了该迁移路径原有的 SQL 错误:`usage` 表没有 `updated_at` 列,旧更新语句却写入它,真实 schema 下会导致迁移失败。这个错误在加固前已经存在,**不混算成第三类加固新增回归**。
### 2. 合法支付会话链接的 fragment 被误拒绝
涉及 `frontend/src/utils/paymentUrl.ts`,调用方包括钱包充值和订阅购买。
加固把包含 `#fragment` 的 HTTPS 支付链接一律拒绝。支付会话链接可能依赖该片段,不能将其等同于脚本协议或 URL 内嵌凭据。现在保留原有片段、查询参数及编码,同时继续拒绝非 HTTPS、相对地址、反斜杠和 URL 用户名/密码。
对应工具函数新增会话片段和编码保留测试,危险协议及凭据注入用例继续执行。服务器端支付 API 基址的同源、TLS 和无片段限制不变;没有放宽 webhook 验签、支付金额、订单归属或实际支付状态校验。此次不进行真实扣款测试。
## 按模块的复核边界
| 模块 | 复核内容与处理 |
| --- | --- |
| 公共入口、认证、会话、管理令牌 | 核对路由分类、认证缓存刷新、跨节点撤权和令牌权限目录;保留身份头防伪、用户/管理员隔离及敏感动作的权限要求。 |
| 端点配置及编辑往返 | 保留前序规则原值查看、重复/嵌套规则恢复、占位符拒绝和并发编辑保护,避免只恢复显示而破坏再次保存。 |
| 普通 Provider 连接与代理 | 对照后续 DNS/FakeIP/SOCKS 修复,不重新引入已移除的普通 Provider 地址过滤;凭据专用连接与普通业务代理分开判断。 |
| OAuth、SMTP、LDAP 与密钥 | 检查默认客户端、显式覆盖、凭据迁移与目标绑定;保留 Gemini CLI/Antigravity 兼容性恢复及既有 SMTP 修复,不撤销传输凭据隔离。 |
| 请求记录、候选诊断及保留策略 | 核对配置读取、采集、投影、单条/批量写入、读取和清理链路;修复本报告的提前删除,保留显式 `full` 及必要诊断,不把原始正文重新塞回计费主表。 |
| 视频与异步任务 | 保留前序提示词/展示名、签名产物 URL、Gemini 结果 URI、数据库领取字段修复;身份不可变条件、任务归属、加密文件和下载凭据隔离不撤销。 |
| 模型、调度、配额和池状态 | 对照模型获取、手动模型、健康摘要、候选选择、额度预留及状态流转;不回退混入该提交的订阅/计费功能。 |
| 钱包、订阅、兑换与支付 | 检查业务 URL 到前端跳转链路并修复 fragment 误拦;交易归属、金额、并发更新、幂等及回调校验维持原边界。 |
| 流式、WebSocket、格式转换 | 检查请求限制、认证头转发、取消/重定向、会话延续和正文捕获之间的关系;修正两处与恢复后的日志语义冲突的旧断言,不退回禁用功能的实现。 |
| 系统导入、导出、备份和恢复 | 区分交互式导入、恢复备份、回滚模式;保留加密备份、用途绑定、导入锁和凭据保留规则,不把安全导出误当作完整灾备。 |
| 数据契约、PostgreSQL 和迁移 | 检查脱敏投影与业务字段消费者是否矛盾;旧驱动不复活,使用真实迁移 schema 验证正文及视频的关键写入链路。 |
| 管理前端与外链 | 复核脱敏状态、权限显示、URL/导航校验、支付调用方,并执行全部前端测试与真实项目类型检查。 |
| 安装、更新、容器和发布流程 | 执行归档、链接、目录、写入目标、来源信任、compose 参数及发布供应链的隔离 Shell 测试;不运行实际部署。 |
## “全部修复”续轮完成项
以下问题已按根因处理,不再作为上一轮的未解决清单。历史类型问题、测试夹具与辅助器缺陷不统一归因为本次加固,功能恢复也不以撤销所有保护为代价。
### 1. 真实前端类型检查及相关运行时缺陷
- 修复原先 116 个文件中的 461 条类型诊断,将 `npm run type-check` 改为 `vue-tsc -b --force --pretty false`,确实检查引用的应用和工具项目,不再依赖顶层空项目的成功退出。
- 按现有接口契约补齐响应泛型、可空字段、配置 schema、图表时间轴、用户角色、事件参数及测试夹具。保留严格检查、ES2021、未知数据边界与动态配置扩展字段,未通过 `any`、忽略诊断、排除测试或降低配置绕过错误。
- 修复请求缓存和登录校验在 fetcher 同步抛错后不能正确清理 in-flight 状态的问题;失败后可按原有退避策略重试,仍隔离不同身份和旧请求。
- 修复 Provider 余额重试的迟到响应覆盖新加载结果,以及卸载后更新状态的问题;新增回归同时覆盖合法的零余额与签到失败值,不因真假值判断隐藏正常数据。
- 修复钱包模板中的刷新调用,并在用户密钥删除、路由配置保存等异步流程中固定操作目标,避免确认期间切换页面后作用到另一个对象。
### 2. 手动保留天数取了更激进的截止点
`usage_cleanup_window_with_override` 原来使用 `max` 合并截止时间,与界面“在策略内取更保守时间点”的设计相反:删除条件是记录时间早于截止点,选更晚的时间会扩大删除范围。
现改为每一保留层级分别取 `min`。详细正文、压缩正文、请求头、完整日志均不短于既有策略,也不短于本次手动指定天数。测试覆盖 0、5、30、180、400 天及“选中记录是原策略子集”的关系。显式立即清理模式不受此改动影响。
### 3. 可信原始数据库恢复可显式保留凭据
原始 JSONL 导入及数据库复制曾无条件替换密码哈希、API Key 和管理令牌,导致可信恢复/迁移也不能继续使用原凭据。新增 CLI `--preserve-credentials`,仅由操作者显式选择,不能由导入文件中的字段自行启用:
```sh
aether-gateway --database-driver postgres --postgres-url "$TARGET_DATABASE_URL" import --input /path/to/trusted.jsonl --preserve-credentials
aether-gateway copy --source-driver postgres --source-url "$SOURCE_DATABASE_URL" --target-driver postgres --target-url "$TARGET_DATABASE_URL" --preserve-credentials
```
- 不加该参数时仍默认撤销导入的身份凭据,并在操作前给出警告;原有库函数入口也保持该默认值。
- 显式保留时,只保留导入文件中用户密码、API Key 和管理令牌原有字段/状态;不会把原本停用的凭据重新启用。
- 导入的登录会话仍撤销,代理隧道仍重置在线状态和代际。外键、身份归属、OAuth 绑定一致性、输入大小及事务校验不变。
- 仅适用于可信且由操作者控制的备份/来源。目标实例须使用与来源兼容的加密配置;此开关不会解密、重新加密或重建已经丢失的值。复制链接的 TLS 默认保护不变。
- 这是原始数据库工具的选项,不改变 HTTP 配置导入权限,也不替代已有的加密备份恢复模式。操作示例会写入指定目标,执行前须自行核对目标并备份;本轮仅在临时隔离库中验证。
真实 PostgreSQL 回归分别验证默认撤销与显式保留后的密码哈希、密钥哈希、密文和启用状态;单元回归另覆盖管理令牌、会话及隧道边界。
### 4. 数据库回归夹具与测试资源回收
- 修正历史测试的超长 ID、NUMERIC/f64 解码、依赖预填充数据库的统计夹具,以及使用不在持久化契约内的 metadata 字段。使用真实 UUID、显式 SQL 类型转换、自建自清理数据和合法 trace 字段,未放宽生产 schema 或脱敏规则来迁就测试。
- 保留现有 usage 导出时间戳以秒计的兼容契约,未因旧字段名包含 `_ms` 就改变导入导出单位。
- 临时 PostgreSQL 辅助器改为 `pg_ctl stop -m fast` 并等待子进程退出,再清理自有目录;关闭失败时保留进程与目录的所有权,允许重试,不强杀后直接删数据。
- 新增两个真实 PostgreSQL 回归,覆盖打开连接下四次停止/重启、数据保留、重复停止、关闭失败重试及析构清理。专项运行前后共享内存段清单未新增残留;未清除其他业务或历史进程的 IPC 资源。
- 续轮全量回归进一步定位到迁移、回填测试各自复制的旧辅助器,同样在强杀后泄漏资源,导致后续网关测试无法初始化数据库。两份实现已合并到仅测试使用的 `postgres_test_support.rs`,采用相同的正常关闭与失败重试规则,另补两个回归;没有只清理环境而保留泄漏代码。
- 用 `AETHER_REQUIRE_LOCAL_POSTGRES_TESTS=1` 强制迁移、回填和共用辅助器的真实数据库用例执行,生命周期筛选结果为 63 通过、0 失败、1 忽略;该忽略项为已在隔离数据库中单独通过的导入测试。此次完整生命周期运行前后 IPC 清单一致。
- 辅助器支持 `AETHER_PG_CTL_BIN`,默认查找 `AETHER_POSTGRES_BIN` 同目录下的 `pg_ctl`;初始化失败也回收本次拥有的工作目录。
## 验证结果
| 验证层 | 结果 |
| --- | --- |
| Rust 工作区所有 library 与 binary 测试目标 | 最终 `cargo test --workspace --lib --bins --locked -- --test-threads=4`:65 个目标,8944 通过、0 失败、19 默认忽略;其中 41 个 library 目标 8645 通过,24 个 binary 目标 299 通过。设置 `AETHER_REQUIRE_LOCAL_POSTGRES_TESTS=1`,迁移/回填测试不能因环境问题静默跳过。 |
| 最终网关全量 | 随工作区 feature 合并执行:5124 通过,0 失败,含此前因数据库初始化失败的两项跨节点认证回归。使用仓库既有的 `RUST_MIN_STACK=16777216` 测试配置;不直接运行遗漏该配置的产物,也不改生产配置规避问题。 |
| PostgreSQL 适配器全量含真实数据库回归 | `AETHER_TEST_DATABASE_URL=<隔离库> cargo test -p aether-data-postgres --lib --locked -- --include-ignored --test-threads=1`:232 通过,0 失败,0 忽略,包含原先默认忽略的 16 项。 |
| 原始数据库导入/导出 | 设置独立的 `AETHER_TEST_POSTGRES_URL` 并执行 `lifecycle::export::tests --include-ignored`:18 通过,0 失败,0 忽略,包含迁移后数据库读取及两种凭据策略的真实往返。 |
| 临时 PostgreSQL 关闭/重启 | `aether-testkit --features postgres` 的两个真实数据库专项回归均通过;四次打开连接下重启、关闭失败重试、目录与共享内存回收均已验证。 |
| WebSocket 真实网关集成 | 最终重新执行 11 通过、0 失败,含跨连接继续对话、连续计费、长连接撤权、认证头隔离、未知字段透传、断开结算、额度重试和 PII 恢复。使用临时 PostgreSQL 与模拟上游。 |
| 可执行入口 | 299 项通过中包括网关主入口 61、隧道 185、备份恢复 CLI 10、两种 WebSocket 探针 3/4,以及压力测试种子/探针、模拟上游等入口测试。没有启动这些工具的实际生产操作。 |
| 额外入口与身份隔离集成 | `aether-data --test public_entrypoints` 3 项、`aether-gateway --test admin_unsigned_identity_headers` 1 项均重新执行通过。 |
| 前端全量与构建 | 最终 `npm run test:run`:200 个测试文件、1419 项测试通过;`npm run build:with-typecheck` 构建通过。 |
| 类型与格式 | `npm run type-check` 实际运行 `vue-tsc -b --force --pretty false`:0 错误,原先 461 条诊断均已修正;142 个改动/新增前端源码文件 ESLint 为 0 错误、0 警告。`cargo fmt --all -- --check``git diff --check` 通过。 |
| 提交前 Clippy | 本地按 `.github/workflows/rust-ci.yml` 的 Gateway、Data、其余工作区三组范围执行,均使用 `--locked``-D warnings`,全部通过;未关闭或放宽 lint 规则。 |
| 安装/发布脚本 | 10 份隔离 Shell 测试全部通过;`python3 tests/compose_database_config_test.py` 通过,只渲染配置,不启动容器。未实际安装、升级或部署。 |
工作区默认忽略的 19 项全部另行显式执行通过:PostgreSQL 适配器 16 项、真实导入 1 项、测试辅助器 2 项。默认命令中的忽略不计作通过;工作区 feature、单包测试及过滤测试的范围不同且存在重叠,上表不累加成独立用例总数。此前的规则、完整正文单条/批量持久化和视频真实数据库回归结果保留在兼容性复核报告中。
中间轮次曾因旧测试辅助器留下的 IPC 残留导致 2 项网关测试和随后 10 项 WebSocket 测试初始化失败;没有把这些命令记为成功。现已修正三处辅助器的关闭路径,并只清理本轮确认创建、无连接且创建进程已退出的 11 个段,未清理此前已有的 21 个段,也未修改内核共享内存限制。
最终工作区、WebSocket、公开入口和身份隔离测试顺序运行前后的 IPC 清单一致,均为此前已有的 21 个段,无新增残留。单独用于数据库适配器及导入往返的临时 PostgreSQL 已正常停止并删除本次自有目录。
脚本范围:`tests/deploy_state_safety_test.sh``tests/install_archive_safety_test.sh``tests/install_container_runtime_security_test.sh``tests/install_current_release_link_test.sh``tests/install_local_bundle_safety_test.sh``tests/install_privileged_write_safety_test.sh``tests/install_source_trust_test.sh``tests/release_supply_chain_test.sh``tests/tunnel_installer_config_security_test.sh``tests/update_compose_safety_test.sh`
## 交付限制
本轮已确认、可复现且可在本仓库修复的剩余问题均已处理,未保留已确认却未修复的本轮源码问题;这一结论不等于“所有代码及所有部署环境绝无未知缺陷”。
这是本地源码、静态检查与隔离回归,不是对生产配置、第三方账户或所有部署组合的认证。上述验证未执行服务器部署或改动现有业务数据库;后续源码提交、推送不代表已经部署或完成生产环境验证。
已经被旧代码写空、删除、未采集的正文或视频字段不能由修复自动重建;仍在数据库中的旧内联/压缩正文可在修正后的保留链路中迁移。真实第三方 OAuth、支付、对象存储和远程隧道服务未使用生产凭据进行端到端验证。
+1 -1
View File
@@ -16,7 +16,7 @@
"test:ui": "node --experimental-require-module --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs --ui", "test:ui": "node --experimental-require-module --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs --ui",
"test:run": "node --experimental-require-module --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run", "test:run": "node --experimental-require-module --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run",
"lint": "eslint . --fix", "lint": "eslint . --fix",
"type-check": "vue-tsc --noEmit", "type-check": "vue-tsc -b --force --pretty false",
"version": "git describe --tags --always" "version": "git describe --tags --always"
}, },
"dependencies": { "dependencies": {
+17 -21
View File
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { AxiosAdapter, AxiosInstance, InternalAxiosRequestConfig } from 'axios' import type { AxiosAdapter, InternalAxiosRequestConfig } from 'axios'
import apiClient, { import apiClient, {
AUTH_SESSION_SIGNAL_KEY, AUTH_SESSION_SIGNAL_KEY,
@@ -8,10 +8,6 @@ import apiClient, {
} from '@/api/client' } from '@/api/client'
import { cache, cachedRequest } from '@/utils/cache' import { cache, cachedRequest } from '@/utils/cache'
type TestableApiClient = typeof apiClient & {
client: AxiosInstance
}
describe('apiClient auth state change event', () => { describe('apiClient auth state change event', () => {
beforeEach(() => { beforeEach(() => {
localStorage.clear() localStorage.clear()
@@ -55,10 +51,10 @@ describe('apiClient auth state change event', () => {
}) })
it('restores a session through the refresh cookie and stores the result in memory only', async () => { it('restores a session through the refresh cookie and stores the result in memory only', async () => {
const rawClient = apiClient as TestableApiClient const rawClient = apiClient['client']
const previousAdapter = rawClient.client.defaults.adapter const previousAdapter = rawClient.defaults.adapter
rawClient.client.defaults.adapter = (async (config: InternalAxiosRequestConfig) => ({ rawClient.defaults.adapter = (async (config: InternalAxiosRequestConfig) => ({
data: { access_token: 'restored-access-token' }, data: { access_token: 'restored-access-token' },
status: 200, status: 200,
statusText: 'OK', statusText: 'OK',
@@ -72,16 +68,16 @@ describe('apiClient auth state change event', () => {
expect(localStorage.getItem('access_token')).toBeNull() expect(localStorage.getItem('access_token')).toBeNull()
expect(sessionStorage.getItem('access_token')).toBeNull() expect(sessionStorage.getItem('access_token')).toBeNull()
} finally { } finally {
rawClient.client.defaults.adapter = previousAdapter rawClient.defaults.adapter = previousAdapter
} }
}) })
it('does not resurrect a session when logout wins an in-flight restore', async () => { it('does not resurrect a session when logout wins an in-flight restore', async () => {
const rawClient = apiClient as TestableApiClient const rawClient = apiClient['client']
const previousAdapter = rawClient.client.defaults.adapter const previousAdapter = rawClient.defaults.adapter
let resolveRefresh!: (response: Awaited<ReturnType<AxiosAdapter>>) => void let resolveRefresh!: (response: Awaited<ReturnType<AxiosAdapter>>) => void
rawClient.client.defaults.adapter = (() => new Promise((resolve) => { rawClient.defaults.adapter = (() => new Promise((resolve) => {
resolveRefresh = resolve resolveRefresh = resolve
})) as AxiosAdapter })) as AxiosAdapter
@@ -100,7 +96,7 @@ describe('apiClient auth state change event', () => {
await expect(restore).rejects.toThrow('Auth state changed') await expect(restore).rejects.toThrow('Auth state changed')
expect(apiClient.getToken()).toBeNull() expect(apiClient.getToken()).toBeNull()
} finally { } finally {
rawClient.client.defaults.adapter = previousAdapter rawClient.defaults.adapter = previousAdapter
} }
}) })
@@ -141,11 +137,11 @@ describe('apiClient auth state change event', () => {
}) })
it('sends auth refresh without a request body', async () => { it('sends auth refresh without a request body', async () => {
const rawClient = apiClient as TestableApiClient const rawClient = apiClient['client']
const previousAdapter = rawClient.client.defaults.adapter const previousAdapter = rawClient.defaults.adapter
const requests: InternalAxiosRequestConfig[] = [] const requests: InternalAxiosRequestConfig[] = []
rawClient.client.defaults.adapter = (async (config: InternalAxiosRequestConfig) => { rawClient.defaults.adapter = (async (config: InternalAxiosRequestConfig) => {
requests.push(config) requests.push(config)
return { return {
data: { access_token: 'new-access-token' }, data: { access_token: 'new-access-token' },
@@ -165,16 +161,16 @@ describe('apiClient auth state change event', () => {
expect(requests[0].method).toBe('post') expect(requests[0].method).toBe('post')
expect(requests[0].data).toBeUndefined() expect(requests[0].data).toBeUndefined()
} finally { } finally {
rawClient.client.defaults.adapter = previousAdapter rawClient.defaults.adapter = previousAdapter
} }
}) })
it('authenticates protected gateway operational requests', async () => { it('authenticates protected gateway operational requests', async () => {
const rawClient = apiClient as TestableApiClient const rawClient = apiClient['client']
const previousAdapter = rawClient.client.defaults.adapter const previousAdapter = rawClient.defaults.adapter
const requests: InternalAxiosRequestConfig[] = [] const requests: InternalAxiosRequestConfig[] = []
rawClient.client.defaults.adapter = (async (config: InternalAxiosRequestConfig) => { rawClient.defaults.adapter = (async (config: InternalAxiosRequestConfig) => {
requests.push(config) requests.push(config)
return { return {
data: '', data: '',
@@ -193,7 +189,7 @@ describe('apiClient auth state change event', () => {
expect(requests[0].headers.Authorization).toBe('Bearer operational-access-token') expect(requests[0].headers.Authorization).toBe('Bearer operational-access-token')
expect(requests[0].headers['X-Client-Device-Id']).toBeTruthy() expect(requests[0].headers['X-Client-Device-Id']).toBeTruthy()
} finally { } finally {
rawClient.client.defaults.adapter = previousAdapter rawClient.defaults.adapter = previousAdapter
} }
}) })
}) })
@@ -0,0 +1,26 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { revealEndpointRules } from '../endpoints/endpoints'
const client = vi.hoisted(() => ({ get: vi.fn() }))
vi.mock('../client', () => ({ default: client }))
beforeEach(() => {
client.get.mockReset().mockResolvedValue({ data: { header_rules: [], body_rules: [], response_header_rules: [] } })
})
describe('endpoint rule reveal API', () => {
it('uses the scoped route and forwards cancellation', async () => {
const controller = new AbortController()
await revealEndpointRules('endpoint/with?reserved', controller.signal)
expect(client.get).toHaveBeenCalledWith(
'/api/admin/endpoints/endpoint%2Fwith%3Freserved/rules/reveal',
{ signal: controller.signal },
)
})
it('fetches each reveal without retaining a cached response', async () => {
await revealEndpointRules('endpoint-1')
await revealEndpointRules('endpoint-1')
expect(client.get).toHaveBeenCalledTimes(2)
})
})
+27 -5
View File
@@ -215,7 +215,18 @@ export const adminWalletApi = {
credited_at: string | null credited_at: string | null
} }
}> { }> {
const response = await apiClient.post(`/api/admin/wallets/${walletId}/recharge`, payload) const response = await apiClient.post<{
wallet: AdminWallet
payment_order: {
id: string
order_no: string
amount_usd: number
payment_method: string
status: string
created_at: string
credited_at: string | null
}
}>(`/api/admin/wallets/${walletId}/recharge`, payload)
return response.data return response.data
}, },
@@ -223,7 +234,10 @@ export const adminWalletApi = {
wallet: AdminWallet wallet: AdminWallet
transaction: WalletTransaction transaction: WalletTransaction
}> { }> {
const response = await apiClient.post(`/api/admin/wallets/${walletId}/adjust`, payload) const response = await apiClient.post<{
wallet: AdminWallet
transaction: WalletTransaction
}>(`/api/admin/wallets/${walletId}/adjust`, payload)
return response.data return response.data
}, },
@@ -232,7 +246,11 @@ export const adminWalletApi = {
refund: RefundRequest refund: RefundRequest
transaction: WalletTransaction transaction: WalletTransaction
}> { }> {
const response = await apiClient.post( const response = await apiClient.post<{
wallet: AdminWallet
refund: RefundRequest
transaction: WalletTransaction
}>(
`/api/admin/wallets/${walletId}/refunds/${refundId}/process`, `/api/admin/wallets/${walletId}/refunds/${refundId}/process`,
{} {}
) )
@@ -244,7 +262,11 @@ export const adminWalletApi = {
refund: RefundRequest refund: RefundRequest
transaction: WalletTransaction | null transaction: WalletTransaction | null
}> { }> {
const response = await apiClient.post( const response = await apiClient.post<{
wallet: AdminWallet
refund: RefundRequest
transaction: WalletTransaction | null
}>(
`/api/admin/wallets/${walletId}/refunds/${refundId}/fail`, `/api/admin/wallets/${walletId}/refunds/${refundId}/fail`,
payload payload
) )
@@ -256,7 +278,7 @@ export const adminWalletApi = {
refundId: string, refundId: string,
payload: RefundCompleteRequest payload: RefundCompleteRequest
): Promise<{ refund: RefundRequest }> { ): Promise<{ refund: RefundRequest }> {
const response = await apiClient.post( const response = await apiClient.post<{ refund: RefundRequest }>(
`/api/admin/wallets/${walletId}/refunds/${refundId}/complete`, `/api/admin/wallets/${walletId}/refunds/${refundId}/complete`,
payload payload
) )
+7 -2
View File
@@ -8,6 +8,11 @@ import type { ApiKeyInstallSession, InstallSessionTargetSystem, InstallTargetCli
const SYSTEM_DATA_IMPORT_TIMEOUT_MS = 10 * 60 * 1000 const SYSTEM_DATA_IMPORT_TIMEOUT_MS = 10 * 60 * 1000
const ALL_SYSTEM_CONFIGS_CACHE_KEY = 'admin:system:configs' const ALL_SYSTEM_CONFIGS_CACHE_KEY = 'admin:system:configs'
export interface AdminTimeSeriesPoint extends Record<string, unknown> {
date: string
total_cost: number
}
export interface AdminSystemConfigItem { export interface AdminSystemConfigItem {
key: string key: string
value: unknown value: unknown
@@ -1594,12 +1599,12 @@ export const adminApi = {
provider_name?: string provider_name?: string
}, },
options?: AdminAnalyticsRequestOptions options?: AdminAnalyticsRequestOptions
): Promise<Array<Record<string, unknown>>> { ): Promise<AdminTimeSeriesPoint[]> {
const cacheKey = buildCacheKey('admin:stats:time-series', params) const cacheKey = buildCacheKey('admin:stats:time-series', params)
return cachedRequest( return cachedRequest(
cacheKey, cacheKey,
async () => { async () => {
const response = await apiClient.get<Array<Record<string, unknown>>>('/api/admin/stats/time-series', { params }) const response = await apiClient.get<AdminTimeSeriesPoint[]>('/api/admin/stats/time-series', { params })
return response.data return response.data
}, },
options?.skipCache ? 0 : 20 * 1000 options?.skipCache ? 0 : 20 * 1000
+10 -10
View File
@@ -57,61 +57,61 @@ export const announcementApi = {
limit?: number limit?: number
offset?: number offset?: number
}): Promise<AnnouncementListResponse> { }): Promise<AnnouncementListResponse> {
const response = await apiClient.get('/api/announcements', { params }) const response = await apiClient.get<AnnouncementListResponse>('/api/announcements', { params })
return response.data return response.data
}, },
// 获取当前有效的公告 // 获取当前有效的公告
async getActiveAnnouncements(): Promise<AnnouncementListResponse> { async getActiveAnnouncements(): Promise<AnnouncementListResponse> {
const response = await apiClient.get('/api/announcements/active') const response = await apiClient.get<AnnouncementListResponse>('/api/announcements/active')
return response.data return response.data
}, },
// 获取单个公告 // 获取单个公告
async getAnnouncement(id: string): Promise<Announcement> { async getAnnouncement(id: string): Promise<Announcement> {
const response = await apiClient.get(`/api/announcements/${id}`) const response = await apiClient.get<Announcement>(`/api/announcements/${id}`)
return response.data return response.data
}, },
// 标记公告为已读 // 标记公告为已读
async markAsRead(id: string): Promise<{ message: string }> { async markAsRead(id: string): Promise<{ message: string }> {
const response = await apiClient.patch(`/api/announcements/${id}/read-status`) const response = await apiClient.patch<{ message: string }>(`/api/announcements/${id}/read-status`)
return response.data return response.data
}, },
// 标记所有公告为已读 // 标记所有公告为已读
async markAllAsRead(): Promise<{ message: string }> { async markAllAsRead(): Promise<{ message: string }> {
const response = await apiClient.post('/api/announcements/read-all') const response = await apiClient.post<{ message: string }>('/api/announcements/read-all')
return response.data return response.data
}, },
// 获取未读公告数量 // 获取未读公告数量
async getUnreadCount(): Promise<{ unread_count: number }> { async getUnreadCount(): Promise<{ unread_count: number }> {
const response = await apiClient.get('/api/announcements/users/me/unread-count') const response = await apiClient.get<{ unread_count: number }>('/api/announcements/users/me/unread-count')
return response.data return response.data
}, },
async getRequiredUnreadAnnouncements(): Promise<AnnouncementListResponse> { async getRequiredUnreadAnnouncements(): Promise<AnnouncementListResponse> {
const response = await apiClient.get('/api/announcements/users/me/required-unread') const response = await apiClient.get<AnnouncementListResponse>('/api/announcements/users/me/required-unread')
return response.data return response.data
}, },
// 管理员方法 // 管理员方法
// 创建公告 // 创建公告
async createAnnouncement(data: CreateAnnouncementRequest): Promise<{ id: string; title: string; message: string }> { async createAnnouncement(data: CreateAnnouncementRequest): Promise<{ id: string; title: string; message: string }> {
const response = await apiClient.post('/api/announcements', data) const response = await apiClient.post<{ id: string; title: string; message: string }>('/api/announcements', data)
return response.data return response.data
}, },
// 更新公告 // 更新公告
async updateAnnouncement(id: string, data: UpdateAnnouncementRequest): Promise<{ message: string }> { async updateAnnouncement(id: string, data: UpdateAnnouncementRequest): Promise<{ message: string }> {
const response = await apiClient.put(`/api/announcements/${id}`, data) const response = await apiClient.put<{ message: string }>(`/api/announcements/${id}`, data)
return response.data return response.data
}, },
// 删除公告 // 删除公告
async deleteAnnouncement(id: string): Promise<{ message: string }> { async deleteAnnouncement(id: string): Promise<{ message: string }> {
const response = await apiClient.delete(`/api/announcements/${id}`) const response = await apiClient.delete<{ message: string }>(`/api/announcements/${id}`)
return response.data return response.data
} }
} }
+6 -6
View File
@@ -180,27 +180,27 @@ export const asyncTasksApi = {
const query = searchParams.toString() const query = searchParams.toString()
const url = query ? `/api/admin/tasks?${query}` : '/api/admin/tasks' const url = query ? `/api/admin/tasks?${query}` : '/api/admin/tasks'
const response = await apiClient.get(url) const response = await apiClient.get<AsyncTaskListResponse>(url)
return response.data return response.data
}, },
async getStats(): Promise<AsyncTaskStatsResponse> { async getStats(): Promise<AsyncTaskStatsResponse> {
const response = await apiClient.get('/api/admin/tasks/stats') const response = await apiClient.get<AsyncTaskStatsResponse>('/api/admin/tasks/stats')
return response.data return response.data
}, },
async getDetail(taskId: string): Promise<AsyncTaskDetail> { async getDetail(taskId: string): Promise<AsyncTaskDetail> {
const response = await apiClient.get(`/api/admin/tasks/${taskId}`) const response = await apiClient.get<AsyncTaskDetail>(`/api/admin/tasks/${taskId}`)
return response.data return response.data
}, },
async getEvents(taskId: string): Promise<{ items: AsyncTaskEvent[] }> { async getEvents(taskId: string): Promise<{ items: AsyncTaskEvent[] }> {
const response = await apiClient.get(`/api/admin/tasks/${taskId}/events`) const response = await apiClient.get<{ items: AsyncTaskEvent[] }>(`/api/admin/tasks/${taskId}/events`)
return response.data return response.data
}, },
async cancel(taskId: string): Promise<{ id: string; status: string; message: string }> { async cancel(taskId: string): Promise<{ id: string; status: string; message: string }> {
const response = await apiClient.post(`/api/admin/tasks/${taskId}/cancel`) const response = await apiClient.post<{ id: string; status: string; message: string }>(`/api/admin/tasks/${taskId}/cancel`)
return response.data return response.data
}, },
@@ -213,7 +213,7 @@ export const asyncTasksApi = {
}, },
async trigger(taskKey: string, payload: Record<string, unknown> = {}): Promise<{ run_id: string; status: string }> { async trigger(taskKey: string, payload: Record<string, unknown> = {}): Promise<{ run_id: string; status: string }> {
const response = await apiClient.post(`/api/admin/tasks/${taskKey}/trigger`, payload) const response = await apiClient.post<{ run_id: string; status: string }>(`/api/admin/tasks/${taskKey}/trigger`, payload)
return response.data return response.data
}, },
} }
+10 -4
View File
@@ -57,13 +57,13 @@ export const auditApi = {
limit?: number limit?: number
offset?: number offset?: number
}): Promise<AuditLogsResponse> { }): Promise<AuditLogsResponse> {
const response = await apiClient.get('/api/monitoring/my-audit-logs', { params: filters }) const response = await apiClient.get<Record<string, unknown>>('/api/monitoring/my-audit-logs', { params: filters })
return normalizeAuditResponse(response.data) return normalizeAuditResponse(response.data)
}, },
// 获取所有审计日志 (管理员) // 获取所有审计日志 (管理员)
async getAuditLogs(filters?: AuditFilters): Promise<AuditLogsResponse> { async getAuditLogs(filters?: AuditFilters): Promise<AuditLogsResponse> {
const response = await apiClient.get('/api/admin/monitoring/audit-logs', { params: filters }) const response = await apiClient.get<Record<string, unknown>>('/api/admin/monitoring/audit-logs', { params: filters })
return normalizeAuditResponse(response.data) return normalizeAuditResponse(response.data)
}, },
@@ -72,7 +72,10 @@ export const auditApi = {
activities: AuditLog[] activities: AuditLog[]
count: number count: number
}> { }> {
const response = await apiClient.get('/api/admin/monitoring/suspicious-activities', { const response = await apiClient.get<{
activities: AuditLog[]
count: number
}>('/api/admin/monitoring/suspicious-activities', {
params: { hours, limit } params: { hours, limit }
}) })
return response.data return response.data
@@ -83,7 +86,10 @@ export const auditApi = {
analysis: Record<string, unknown> analysis: Record<string, unknown>
recommendations: string[] recommendations: string[]
}> { }> {
const response = await apiClient.get(`/api/admin/monitoring/user-behavior/${userId}`, { const response = await apiClient.get<{
analysis: Record<string, unknown>
recommendations: string[]
}>(`/api/admin/monitoring/user-behavior/${userId}`, {
params: { days } params: { days }
}) })
return response.data return response.data
+21 -15
View File
@@ -89,7 +89,7 @@ export const cacheApi = {
* *
*/ */
async getStats(): Promise<CacheStats> { async getStats(): Promise<CacheStats> {
const response = await api.get('/api/admin/monitoring/cache/stats') const response = await api.get<{ data: CacheStats }>('/api/admin/monitoring/cache/stats')
return response.data.data return response.data.data
}, },
@@ -97,7 +97,7 @@ export const cacheApi = {
* *
*/ */
async getConfig(): Promise<CacheConfig> { async getConfig(): Promise<CacheConfig> {
const response = await api.get('/api/admin/monitoring/cache/config') const response = await api.get<{ data: CacheConfig }>('/api/admin/monitoring/cache/config')
return response.data.data return response.data.data
}, },
@@ -107,7 +107,7 @@ export const cacheApi = {
* @param userIdentifier User UUIDAPI Key ID * @param userIdentifier User UUIDAPI Key ID
*/ */
async getUserAffinity(userIdentifier: string): Promise<UserAffinity[] | null> { async getUserAffinity(userIdentifier: string): Promise<UserAffinity[] | null> {
const response = await api.get(`/api/admin/monitoring/cache/affinity/${userIdentifier}`) const response = await api.get<{ status: string; affinities: UserAffinity[] }>(`/api/admin/monitoring/cache/affinity/${userIdentifier}`)
if (response.data.status === 'not_found') { if (response.data.status === 'not_found') {
return null return null
} }
@@ -151,7 +151,7 @@ export const cacheApi = {
* *
*/ */
async clearAllCache(): Promise<{ count: number }> { async clearAllCache(): Promise<{ count: number }> {
const response = await api.delete('/api/admin/monitoring/cache') const response = await api.delete<{ count: number }>('/api/admin/monitoring/cache')
return response.data return response.data
}, },
@@ -159,7 +159,7 @@ export const cacheApi = {
* Provider的所有缓存 * Provider的所有缓存
*/ */
async clearProviderCache(providerId: string): Promise<{ count: number; provider_id: string }> { async clearProviderCache(providerId: string): Promise<{ count: number; provider_id: string }> {
const response = await api.delete(`/api/admin/monitoring/cache/providers/${providerId}`) const response = await api.delete<{ count: number; provider_id: string }>(`/api/admin/monitoring/cache/providers/${providerId}`)
return response.data return response.data
}, },
@@ -167,7 +167,13 @@ export const cacheApi = {
* *
*/ */
async listAffinities(keyword?: string): Promise<AffinityListResponse> { async listAffinities(keyword?: string): Promise<AffinityListResponse> {
const response = await api.get('/api/admin/monitoring/cache/affinities', { const response = await api.get<{
data?: {
items?: UserAffinity[]
meta?: { total?: number }
matched_user_id?: string | null
}
}>('/api/admin/monitoring/cache/affinities', {
params: keyword ? { keyword } : undefined params: keyword ? { keyword } : undefined
}) })
const data = response.data.data ?? {} const data = response.data.data ?? {}
@@ -212,7 +218,7 @@ export const redisCacheApi = {
* Redis * Redis
*/ */
async getCategories(): Promise<RedisCacheCategoriesResponse> { async getCategories(): Promise<RedisCacheCategoriesResponse> {
const response = await api.get('/api/admin/monitoring/cache/redis-keys') const response = await api.get<{ data: RedisCacheCategoriesResponse }>('/api/admin/monitoring/cache/redis-keys')
return response.data.data return response.data.data
}, },
@@ -220,7 +226,7 @@ export const redisCacheApi = {
* Redis * Redis
*/ */
async clearCategory(category: string): Promise<{ status: string; message: string; category: string; deleted_count: number }> { async clearCategory(category: string): Promise<{ status: string; message: string; category: string; deleted_count: number }> {
const response = await api.delete(`/api/admin/monitoring/cache/redis-keys/${category}`) const response = await api.delete<{ status: string; message: string; category: string; deleted_count: number }>(`/api/admin/monitoring/cache/redis-keys/${category}`)
return response.data return response.data
} }
} }
@@ -308,7 +314,7 @@ export const cacheAnalysisApi = {
api_key_id?: string api_key_id?: string
hours?: number hours?: number
}): Promise<TTLAnalysisResponse> { }): Promise<TTLAnalysisResponse> {
const response = await api.get('/api/admin/usage/cache-affinity/ttl-analysis', { params }) const response = await api.get<TTLAnalysisResponse>('/api/admin/usage/cache-affinity/ttl-analysis', { params })
return response.data return response.data
}, },
@@ -320,7 +326,7 @@ export const cacheAnalysisApi = {
api_key_id?: string api_key_id?: string
hours?: number hours?: number
}): Promise<CacheHitAnalysisResponse> { }): Promise<CacheHitAnalysisResponse> {
const response = await api.get('/api/admin/usage/cache-affinity/hit-analysis', { params }) const response = await api.get<CacheHitAnalysisResponse>('/api/admin/usage/cache-affinity/hit-analysis', { params })
return response.data return response.data
}, },
@@ -339,7 +345,7 @@ export const cacheAnalysisApi = {
return cachedRequest( return cachedRequest(
cacheKey, cacheKey,
async () => { async () => {
const response = await api.get('/api/admin/usage/cache-affinity/interval-timeline', { params }) const response = await api.get<IntervalTimelineResponse>('/api/admin/usage/cache-affinity/interval-timeline', { params })
return response.data return response.data
}, },
30000 30000
@@ -408,7 +414,7 @@ export const modelMappingCacheApi = {
* *
*/ */
async getStats(): Promise<ModelMappingCacheStats> { async getStats(): Promise<ModelMappingCacheStats> {
const response = await api.get('/api/admin/monitoring/cache/model-mapping/stats') const response = await api.get<{ data: ModelMappingCacheStats }>('/api/admin/monitoring/cache/model-mapping/stats')
return response.data.data return response.data.data
}, },
@@ -416,7 +422,7 @@ export const modelMappingCacheApi = {
* *
*/ */
async clearAll(): Promise<ClearModelMappingCacheResponse> { async clearAll(): Promise<ClearModelMappingCacheResponse> {
const response = await api.delete('/api/admin/monitoring/cache/model-mapping') const response = await api.delete<ClearModelMappingCacheResponse>('/api/admin/monitoring/cache/model-mapping')
return response.data return response.data
}, },
@@ -424,7 +430,7 @@ export const modelMappingCacheApi = {
* *
*/ */
async clearByName(modelName: string): Promise<ClearModelMappingCacheResponse> { async clearByName(modelName: string): Promise<ClearModelMappingCacheResponse> {
const response = await api.delete(`/api/admin/monitoring/cache/model-mapping/${encodeURIComponent(modelName)}`) const response = await api.delete<ClearModelMappingCacheResponse>(`/api/admin/monitoring/cache/model-mapping/${encodeURIComponent(modelName)}`)
return response.data return response.data
}, },
@@ -432,7 +438,7 @@ export const modelMappingCacheApi = {
* Provider GlobalModel * Provider GlobalModel
*/ */
async clearProviderModel(providerId: string, globalModelId: string): Promise<ClearModelMappingCacheResponse> { async clearProviderModel(providerId: string, globalModelId: string): Promise<ClearModelMappingCacheResponse> {
const response = await api.delete(`/api/admin/monitoring/cache/model-mapping/provider/${providerId}/${globalModelId}`) const response = await api.delete<ClearModelMappingCacheResponse>(`/api/admin/monitoring/cache/model-mapping/provider/${providerId}/${globalModelId}`)
return response.data return response.data
} }
} }
+1 -1
View File
@@ -224,7 +224,7 @@ class ApiClient {
/** /**
* *
*/ */
private async handleResponseError(error: unknown): Promise<never> { private async handleResponseError(error: unknown): Promise<AxiosResponse> {
// 请求被取消 // 请求被取消
if (axios.isCancel(error)) { if (axios.isCancel(error)) {
return Promise.reject(error) return Promise.reject(error)
+8 -8
View File
@@ -278,14 +278,14 @@ export interface RequestDetail {
end_to_end_first_byte_time_ms?: number | null end_to_end_first_byte_time_ms?: number | null
created_at: string created_at: string
updated_at?: string | null updated_at?: string | null
request_headers?: Record<string, unknown> request_headers?: Record<string, unknown> | null
request_body?: Record<string, unknown> request_body?: Record<string, unknown> | null
provider_request_headers?: Record<string, unknown> provider_request_headers?: Record<string, unknown> | null
provider_request_body?: Record<string, unknown> provider_request_body?: Record<string, unknown> | null
response_headers?: Record<string, unknown> response_headers?: Record<string, unknown> | null
client_response_headers?: Record<string, unknown> client_response_headers?: Record<string, unknown> | null
response_body?: Record<string, unknown> response_body?: Record<string, unknown> | null
client_response_body?: Record<string, unknown> client_response_body?: Record<string, unknown> | null
has_request_body?: boolean has_request_body?: boolean
has_provider_request_body?: boolean has_provider_request_body?: boolean
has_response_body?: boolean has_response_body?: boolean
+16 -4
View File
@@ -17,7 +17,13 @@ export async function toggleAdaptiveMode(
rpm_limit: number | null rpm_limit: number | null
effective_limit: number | null effective_limit: number | null
}> { }> {
const response = await client.patch(`/api/admin/adaptive/keys/${keyId}/mode`, data) const response = await client.patch<{
message: string
key_id: string
is_adaptive: boolean
rpm_limit: number | null
effective_limit: number | null
}>(`/api/admin/adaptive/keys/${keyId}/mode`, data)
return response.data return response.data
} }
@@ -34,7 +40,13 @@ export async function setRpmLimit(
rpm_limit: number rpm_limit: number
previous_mode: string previous_mode: string
}> { }> {
const response = await client.patch(`/api/admin/adaptive/keys/${keyId}/limit`, null, { const response = await client.patch<{
message: string
key_id: string
is_adaptive: boolean
rpm_limit: number
previous_mode: string
}>(`/api/admin/adaptive/keys/${keyId}/limit`, null, {
params: { limit } params: { limit }
}) })
return response.data return response.data
@@ -44,7 +56,7 @@ export async function setRpmLimit(
* Key * Key
*/ */
export async function getAdaptiveStats(keyId: string): Promise<AdaptiveStatsResponse> { export async function getAdaptiveStats(keyId: string): Promise<AdaptiveStatsResponse> {
const response = await client.get(`/api/admin/adaptive/keys/${keyId}/stats`) const response = await client.get<AdaptiveStatsResponse>(`/api/admin/adaptive/keys/${keyId}/stats`)
return response.data return response.data
} }
@@ -52,6 +64,6 @@ export async function getAdaptiveStats(keyId: string): Promise<AdaptiveStatsResp
* Key * Key
*/ */
export async function resetAdaptiveLearning(keyId: string): Promise<{ message: string; key_id: string }> { export async function resetAdaptiveLearning(keyId: string): Promise<{ message: string; key_id: string }> {
const response = await client.delete(`/api/admin/adaptive/keys/${keyId}/learning`) const response = await client.delete<{ message: string; key_id: string }>(`/api/admin/adaptive/keys/${keyId}/learning`)
return response.data return response.data
} }
+20 -6
View File
@@ -1,11 +1,25 @@
import client from '../client' import client from '../client'
import type { ProviderEndpoint, ProxyConfig, HeaderRule, BodyRule, FormatAcceptanceConfig } from './types' import type { ProviderEndpoint, ProxyConfig, HeaderRule, BodyRule, FormatAcceptanceConfig } from './types'
export interface ProviderEndpointRules {
header_rules: HeaderRule[]
body_rules: BodyRule[]
response_header_rules: HeaderRule[]
}
export async function revealEndpointRules(endpointId: string, signal?: AbortSignal): Promise<ProviderEndpointRules> {
const response = await client.get<ProviderEndpointRules>(
`/api/admin/endpoints/${encodeURIComponent(endpointId)}/rules/reveal`,
{ signal },
)
return response.data
}
/** /**
* Provider Endpoints * Provider Endpoints
*/ */
export async function getProviderEndpoints(providerId: string): Promise<ProviderEndpoint[]> { export async function getProviderEndpoints(providerId: string): Promise<ProviderEndpoint[]> {
const response = await client.get(`/api/admin/endpoints/providers/${providerId}/endpoints`) const response = await client.get<ProviderEndpoint[]>(`/api/admin/endpoints/providers/${providerId}/endpoints`)
return response.data return response.data
} }
@@ -13,7 +27,7 @@ export async function getProviderEndpoints(providerId: string): Promise<Provider
* Endpoint * Endpoint
*/ */
export async function getEndpoint(endpointId: string): Promise<ProviderEndpoint> { export async function getEndpoint(endpointId: string): Promise<ProviderEndpoint> {
const response = await client.get(`/api/admin/endpoints/${endpointId}`) const response = await client.get<ProviderEndpoint>(`/api/admin/endpoints/${endpointId}`)
return response.data return response.data
} }
@@ -36,7 +50,7 @@ export async function createEndpoint(
format_acceptance_config?: FormatAcceptanceConfig | null format_acceptance_config?: FormatAcceptanceConfig | null
} }
): Promise<ProviderEndpoint> { ): Promise<ProviderEndpoint> {
const response = await client.post(`/api/admin/endpoints/providers/${providerId}/endpoints`, data) const response = await client.post<ProviderEndpoint>(`/api/admin/endpoints/providers/${providerId}/endpoints`, data)
return response.data return response.data
} }
@@ -57,7 +71,7 @@ export async function updateEndpoint(
format_acceptance_config: FormatAcceptanceConfig | null format_acceptance_config: FormatAcceptanceConfig | null
}> }>
): Promise<ProviderEndpoint> { ): Promise<ProviderEndpoint> {
const response = await client.put(`/api/admin/endpoints/${endpointId}`, data) const response = await client.put<ProviderEndpoint>(`/api/admin/endpoints/${endpointId}`, data)
return response.data return response.data
} }
@@ -65,7 +79,7 @@ export async function updateEndpoint(
* Endpoint * Endpoint
*/ */
export async function deleteEndpoint(endpointId: string): Promise<{ message: string; affected_keys_count: number }> { export async function deleteEndpoint(endpointId: string): Promise<{ message: string; affected_keys_count: number }> {
const response = await client.delete(`/api/admin/endpoints/${endpointId}`) const response = await client.delete<{ message: string; affected_keys_count: number }>(`/api/admin/endpoints/${endpointId}`)
return response.data return response.data
} }
@@ -75,6 +89,6 @@ export async function deleteEndpoint(endpointId: string): Promise<{ message: str
export async function getDefaultBodyRules(apiFormat: string, providerType?: string): Promise<{ api_format: string; body_rules: BodyRule[] }> { export async function getDefaultBodyRules(apiFormat: string, providerType?: string): Promise<{ api_format: string; body_rules: BodyRule[] }> {
const params: Record<string, string> = {} const params: Record<string, string> = {}
if (providerType) params.provider_type = providerType if (providerType) params.provider_type = providerType
const response = await client.get(`/api/admin/endpoints/defaults/${apiFormat}/body-rules`, { params }) const response = await client.get<{ api_format: string; body_rules: BodyRule[] }>(`/api/admin/endpoints/defaults/${apiFormat}/body-rules`, { params })
return response.data return response.data
} }
+19 -8
View File
@@ -12,6 +12,7 @@ import type {
// 重新导出路由相关类型供外部使用 // 重新导出路由相关类型供外部使用
export type { export type {
GlobalModelResponse,
RoutingKeyInfo, RoutingKeyInfo,
RoutingEndpointInfo, RoutingEndpointInfo,
RoutingModelMapping, RoutingModelMapping,
@@ -37,7 +38,7 @@ export async function getGlobalModels(params?: {
return cachedRequest( return cachedRequest(
key, key,
async () => { async () => {
const response = await client.get('/api/admin/models/global', { params }) const response = await client.get<GlobalModelListResponse>('/api/admin/models/global', { params })
return response.data return response.data
}, },
cacheTtlMs, cacheTtlMs,
@@ -49,7 +50,7 @@ export async function getGlobalModels(params?: {
*/ */
export async function getGlobalModel(id: string): Promise<GlobalModelWithStats> { export async function getGlobalModel(id: string): Promise<GlobalModelWithStats> {
return dedupedRequest(`global-models:detail:${id}`, async () => { return dedupedRequest(`global-models:detail:${id}`, async () => {
const response = await client.get(`/api/admin/models/global/${id}`) const response = await client.get<GlobalModelWithStats>(`/api/admin/models/global/${id}`)
return response.data return response.data
}) })
} }
@@ -58,7 +59,7 @@ export async function getGlobalModel(id: string): Promise<GlobalModelWithStats>
* GlobalModel * GlobalModel
*/ */
export async function createGlobalModel(data: GlobalModelCreate): Promise<GlobalModelResponse> { export async function createGlobalModel(data: GlobalModelCreate): Promise<GlobalModelResponse> {
const response = await client.post('/api/admin/models/global', data) const response = await client.post<GlobalModelResponse>('/api/admin/models/global', data)
return response.data return response.data
} }
@@ -69,7 +70,7 @@ export async function updateGlobalModel(
id: string, id: string,
data: GlobalModelUpdate data: GlobalModelUpdate
): Promise<GlobalModelResponse> { ): Promise<GlobalModelResponse> {
const response = await client.patch(`/api/admin/models/global/${id}`, data) const response = await client.patch<GlobalModelResponse>(`/api/admin/models/global/${id}`, data)
return response.data return response.data
} }
@@ -89,7 +90,7 @@ export async function deleteGlobalModel(
export async function batchDeleteGlobalModels( export async function batchDeleteGlobalModels(
ids: string[] ids: string[]
): Promise<{ success_count: number; failed: Array<{ id: string; error: string }> }> { ): Promise<{ success_count: number; failed: Array<{ id: string; error: string }> }> {
const response = await client.post('/api/admin/models/global/batch-delete', { ids }) const response = await client.post<{ success_count: number; failed: Array<{ id: string; error: string }> }>('/api/admin/models/global/batch-delete', { ids })
return response.data return response.data
} }
@@ -113,7 +114,17 @@ export async function batchAssignToProviders(
error: string error: string
}> }>
}> { }> {
const response = await client.post( const response = await client.post<{
success: Array<{
provider_id: string
provider_name: string
model_id?: string
}>
errors: Array<{
provider_id: string
error: string
}>
}>(
`/api/admin/models/global/${globalModelId}/assign-to-providers`, `/api/admin/models/global/${globalModelId}/assign-to-providers`,
data data
) )
@@ -128,7 +139,7 @@ export async function getGlobalModelProviders(globalModelId: string): Promise<{
total: number total: number
}> { }> {
return dedupedRequest(`global-models:providers:${globalModelId}`, async () => { return dedupedRequest(`global-models:providers:${globalModelId}`, async () => {
const response = await client.get( const response = await client.get<{ providers: ModelCatalogProviderDetail[]; total: number }>(
`/api/admin/models/global/${globalModelId}/providers` `/api/admin/models/global/${globalModelId}/providers`
) )
return response.data return response.data
@@ -141,7 +152,7 @@ export async function getGlobalModelProviders(globalModelId: string): Promise<{
export async function getGlobalModelRoutingPreview( export async function getGlobalModelRoutingPreview(
globalModelId: string globalModelId: string
): Promise<ModelRoutingPreviewResponse> { ): Promise<ModelRoutingPreviewResponse> {
const response = await client.get( const response = await client.get<ModelRoutingPreviewResponse>(
`/api/admin/models/global/${globalModelId}/routing` `/api/admin/models/global/${globalModelId}/routing`
) )
return response.data return response.data
+28 -12
View File
@@ -14,7 +14,7 @@ import type {
* *
*/ */
export async function getHealthSummary(): Promise<HealthSummary> { export async function getHealthSummary(): Promise<HealthSummary> {
const response = await client.get('/api/admin/endpoints/health/summary') const response = await client.get<HealthSummary>('/api/admin/endpoints/health/summary')
return response.data return response.data
} }
@@ -22,7 +22,7 @@ export async function getHealthSummary(): Promise<HealthSummary> {
* Endpoint * Endpoint
*/ */
export async function getEndpointHealth(endpointId: string): Promise<HealthStatus> { export async function getEndpointHealth(endpointId: string): Promise<HealthStatus> {
const response = await client.get(`/api/admin/endpoints/health/endpoint/${endpointId}`) const response = await client.get<HealthStatus>(`/api/admin/endpoints/health/endpoint/${endpointId}`)
return response.data return response.data
} }
@@ -30,7 +30,7 @@ export async function getEndpointHealth(endpointId: string): Promise<HealthStatu
* Key * Key
*/ */
export async function getKeyHealth(keyId: string): Promise<HealthStatus> { export async function getKeyHealth(keyId: string): Promise<HealthStatus> {
const response = await client.get(`/api/admin/endpoints/health/key/${keyId}`) const response = await client.get<HealthStatus>(`/api/admin/endpoints/health/key/${keyId}`)
return response.data return response.data
} }
@@ -48,7 +48,15 @@ export async function recoverKeyHealth(keyId: string, apiFormat?: string): Promi
is_active: boolean is_active: boolean
} }
}> { }> {
const response = await client.patch(`/api/admin/endpoints/health/keys/${keyId}`, null, { const response = await client.patch<{
message: string
details: {
api_format?: string
health_score: number
circuit_breaker_open: boolean
is_active: boolean
}
}>(`/api/admin/endpoints/health/keys/${keyId}`, null, {
params: apiFormat ? { api_format: apiFormat } : undefined params: apiFormat ? { api_format: apiFormat } : undefined
}) })
return response.data return response.data
@@ -66,7 +74,15 @@ export async function recoverAllKeysHealth(): Promise<{
endpoint_id: string endpoint_id: string
}> }>
}> { }> {
const response = await client.patch('/api/admin/endpoints/health/keys') const response = await client.patch<{
message: string
recovered_count: number
recovered_keys: Array<{
key_id: string
key_name: string
endpoint_id: string
}>
}>('/api/admin/endpoints/health/keys')
return response.data return response.data
} }
@@ -77,7 +93,7 @@ export async function getEndpointStatusMonitor(params?: {
lookback_hours?: number lookback_hours?: number
per_format_limit?: number per_format_limit?: number
}): Promise<EndpointStatusMonitorResponse> { }): Promise<EndpointStatusMonitorResponse> {
const response = await client.get('/api/admin/endpoints/health/api-formats', { const response = await client.get<EndpointStatusMonitorResponse>('/api/admin/endpoints/health/api-formats', {
params params
}) })
return response.data return response.data
@@ -90,7 +106,7 @@ export async function getPublicEndpointStatusMonitor(params?: {
lookback_hours?: number lookback_hours?: number
per_format_limit?: number per_format_limit?: number
}): Promise<PublicEndpointStatusMonitorResponse> { }): Promise<PublicEndpointStatusMonitorResponse> {
const response = await client.get('/api/public/health/api-formats', { const response = await client.get<PublicEndpointStatusMonitorResponse>('/api/public/health/api-formats', {
params params
}) })
return response.data return response.data
@@ -104,7 +120,7 @@ export async function getModelStatusMonitor(params?: {
model_limit?: number model_limit?: number
per_model_limit?: number per_model_limit?: number
}): Promise<ModelStatusMonitorResponse> { }): Promise<ModelStatusMonitorResponse> {
const response = await client.get('/api/admin/endpoints/health/models', { const response = await client.get<ModelStatusMonitorResponse>('/api/admin/endpoints/health/models', {
params params
}) })
return response.data return response.data
@@ -118,7 +134,7 @@ export async function getPublicModelStatusMonitor(params?: {
model_limit?: number model_limit?: number
per_model_limit?: number per_model_limit?: number
}): Promise<ModelStatusMonitorResponse> { }): Promise<ModelStatusMonitorResponse> {
const response = await client.get('/api/public/health/models', { const response = await client.get<ModelStatusMonitorResponse>('/api/public/health/models', {
params params
}) })
return response.data return response.data
@@ -133,7 +149,7 @@ export async function getProviderStatusMonitor(params?: {
per_provider_model_limit?: number per_provider_model_limit?: number
per_model_limit?: number per_model_limit?: number
}): Promise<ProviderStatusMonitorResponse> { }): Promise<ProviderStatusMonitorResponse> {
const response = await client.get('/api/admin/endpoints/health/providers', { const response = await client.get<ProviderStatusMonitorResponse>('/api/admin/endpoints/health/providers', {
params params
}) })
return response.data return response.data
@@ -146,7 +162,7 @@ export async function getHealthRelatedMonitor(params: {
related_limit?: number related_limit?: number
per_item_limit?: number per_item_limit?: number
}): Promise<HealthRelatedMonitorResponse> { }): Promise<HealthRelatedMonitorResponse> {
const response = await client.get('/api/admin/endpoints/health/related', { const response = await client.get<HealthRelatedMonitorResponse>('/api/admin/endpoints/health/related', {
params params
}) })
return response.data return response.data
@@ -159,7 +175,7 @@ export async function getPublicHealthRelatedMonitor(params: {
related_limit?: number related_limit?: number
per_item_limit?: number per_item_limit?: number
}): Promise<HealthRelatedMonitorResponse> { }): Promise<HealthRelatedMonitorResponse> {
const response = await client.get('/api/public/health/related', { const response = await client.get<HealthRelatedMonitorResponse>('/api/public/health/related', {
params params
}) })
return response.data return response.data
+18 -14
View File
@@ -37,7 +37,7 @@ export interface ModelCapabilitiesResponse {
* *
*/ */
export async function getAllCapabilities(): Promise<CapabilityDefinition[]> { export async function getAllCapabilities(): Promise<CapabilityDefinition[]> {
const response = await client.get('/api/capabilities') const response = await client.get<{ capabilities: CapabilityDefinition[] }>('/api/capabilities')
return response.data.capabilities return response.data.capabilities
} }
@@ -45,7 +45,7 @@ export async function getAllCapabilities(): Promise<CapabilityDefinition[]> {
* *
*/ */
export async function getUserConfigurableCapabilities(): Promise<CapabilityDefinition[]> { export async function getUserConfigurableCapabilities(): Promise<CapabilityDefinition[]> {
const response = await client.get('/api/capabilities/user-configurable') const response = await client.get<{ capabilities: CapabilityDefinition[] }>('/api/capabilities/user-configurable')
return response.data.capabilities return response.data.capabilities
} }
@@ -53,7 +53,7 @@ export async function getUserConfigurableCapabilities(): Promise<CapabilityDefin
* *
*/ */
export async function getModelCapabilities(modelName: string): Promise<ModelCapabilitiesResponse> { export async function getModelCapabilities(modelName: string): Promise<ModelCapabilitiesResponse> {
const response = await client.get(`/api/capabilities/model/${encodeURIComponent(modelName)}`) const response = await client.get<ModelCapabilitiesResponse>(`/api/capabilities/model/${encodeURIComponent(modelName)}`)
return response.data return response.data
} }
@@ -68,7 +68,7 @@ export interface RevealKeyResult {
} }
export async function revealEndpointKey(keyId: string): Promise<RevealKeyResult> { export async function revealEndpointKey(keyId: string): Promise<RevealKeyResult> {
const response = await client.get(`/api/admin/endpoints/keys/${keyId}/reveal`) const response = await client.get<RevealKeyResult>(`/api/admin/endpoints/keys/${keyId}/reveal`)
return response.data return response.data
} }
@@ -76,7 +76,7 @@ export async function revealEndpointKey(keyId: string): Promise<RevealKeyResult>
* OAuth Key JSON * OAuth Key JSON
*/ */
export async function exportKey(keyId: string): Promise<Record<string, unknown>> { export async function exportKey(keyId: string): Promise<Record<string, unknown>> {
const response = await client.get(`/api/admin/endpoints/keys/${keyId}/export`) const response = await client.get<Record<string, unknown>>(`/api/admin/endpoints/keys/${keyId}/export`)
return response.data return response.data
} }
@@ -84,7 +84,7 @@ export async function exportKey(keyId: string): Promise<Record<string, unknown>>
* Key * Key
*/ */
export async function deleteEndpointKey(keyId: string): Promise<{ message: string }> { export async function deleteEndpointKey(keyId: string): Promise<{ message: string }> {
const response = await client.delete(`/api/admin/endpoints/keys/${keyId}`) const response = await client.delete<{ message: string }>(`/api/admin/endpoints/keys/${keyId}`)
return response.data return response.data
} }
@@ -98,7 +98,7 @@ export interface BatchDeleteKeysResult {
} }
export async function batchDeleteEndpointKeys(ids: string[]): Promise<BatchDeleteKeysResult> { export async function batchDeleteEndpointKeys(ids: string[]): Promise<BatchDeleteKeysResult> {
const response = await client.post('/api/admin/endpoints/keys/batch-delete', { ids }) const response = await client.post<BatchDeleteKeysResult>('/api/admin/endpoints/keys/batch-delete', { ids })
return response.data return response.data
} }
@@ -215,7 +215,7 @@ export async function addProviderKey(
model_exclude_patterns?: string[] // 模型排除规则 model_exclude_patterns?: string[] // 模型排除规则
} }
): Promise<EndpointAPIKey> { ): Promise<EndpointAPIKey> {
const response = await client.post(`/api/admin/endpoints/providers/${providerId}/keys`, data) const response = await client.post<EndpointAPIKey>(`/api/admin/endpoints/providers/${providerId}/keys`, data)
return response.data return response.data
} }
@@ -251,7 +251,7 @@ export async function updateProviderKey(
}>, }>,
requestOptions?: KeyRequestOptions, requestOptions?: KeyRequestOptions,
): Promise<EndpointAPIKey> { ): Promise<EndpointAPIKey> {
const response = await client.put( const response = await client.put<EndpointAPIKey>(
`/api/admin/endpoints/keys/${keyId}`, `/api/admin/endpoints/keys/${keyId}`,
data, data,
requestOptions, requestOptions,
@@ -263,7 +263,7 @@ export async function updateProviderKey(
* Key OAuth * Key OAuth
*/ */
export async function clearOAuthInvalid(keyId: string): Promise<{ message: string }> { export async function clearOAuthInvalid(keyId: string): Promise<{ message: string }> {
const response = await client.post(`/api/admin/endpoints/keys/${keyId}/clear-oauth-invalid`) const response = await client.post<{ message: string }>(`/api/admin/endpoints/keys/${keyId}/clear-oauth-invalid`)
return response.data return response.data
} }
@@ -275,7 +275,11 @@ export async function resetProviderKeyCycleStats(keyId: string): Promise<{
reset_at: number reset_at: number
windows: number windows: number
}> { }> {
const response = await client.post(`/api/admin/endpoints/keys/${keyId}/reset-cycle-stats`) const response = await client.post<{
message: string
reset_at: number
windows: number
}>(`/api/admin/endpoints/keys/${keyId}/reset-cycle-stats`)
return response.data return response.data
} }
@@ -311,7 +315,7 @@ export async function refreshProviderQuota(
keyIds?: string[], keyIds?: string[],
): Promise<RefreshQuotaResult> { ): Promise<RefreshQuotaResult> {
const body = keyIds && keyIds.length > 0 ? { key_ids: keyIds } : undefined const body = keyIds && keyIds.length > 0 ? { key_ids: keyIds } : undefined
const response = await client.post( const response = await client.post<RefreshQuotaResult>(
`/api/admin/endpoints/providers/${providerId}/refresh-quota`, `/api/admin/endpoints/providers/${providerId}/refresh-quota`,
body, body,
{ timeout: 5 * 60 * 1000 }, { timeout: 5 * 60 * 1000 },
@@ -350,7 +354,7 @@ export async function consumeCodexResetCredit(
keyId: string, keyId: string,
payload: ConsumeCodexResetCreditPayload, payload: ConsumeCodexResetCreditPayload,
): Promise<ConsumeCodexResetCreditResult> { ): Promise<ConsumeCodexResetCreditResult> {
const response = await client.post( const response = await client.post<ConsumeCodexResetCreditResult>(
`/api/admin/endpoints/keys/${keyId}/codex-reset-credit/consume`, `/api/admin/endpoints/keys/${keyId}/codex-reset-credit/consume`,
payload, payload,
{ timeout: 5 * 60 * 1000 }, { timeout: 5 * 60 * 1000 },
@@ -383,7 +387,7 @@ export async function batchImportOAuth(
credentials: string, credentials: string,
proxyNodeId?: string proxyNodeId?: string
): Promise<BatchImportResult> { ): Promise<BatchImportResult> {
const response = await client.post(`/api/admin/provider-oauth/providers/${providerId}/batch-import`, { const response = await client.post<BatchImportResult>(`/api/admin/provider-oauth/providers/${providerId}/batch-import`, {
credentials, credentials,
proxy_node_id: proxyNodeId || undefined, proxy_node_id: proxyNodeId || undefined,
}) })
+20 -10
View File
@@ -19,7 +19,7 @@ export async function getProviderModels(
limit?: number limit?: number
} }
): Promise<Model[]> { ): Promise<Model[]> {
const response = await client.get(`/api/admin/providers/${providerId}/models`, { params }) const response = await client.get<Model[]>(`/api/admin/providers/${providerId}/models`, { params })
return response.data return response.data
} }
@@ -30,7 +30,7 @@ export async function createModel(
providerId: string, providerId: string,
data: ModelCreate data: ModelCreate
): Promise<Model> { ): Promise<Model> {
const response = await client.post(`/api/admin/providers/${providerId}/models`, data) const response = await client.post<Model>(`/api/admin/providers/${providerId}/models`, data)
return response.data return response.data
} }
@@ -41,7 +41,7 @@ export async function getModel(
providerId: string, providerId: string,
modelId: string modelId: string
): Promise<Model> { ): Promise<Model> {
const response = await client.get(`/api/admin/providers/${providerId}/models/${modelId}`) const response = await client.get<Model>(`/api/admin/providers/${providerId}/models/${modelId}`)
return response.data return response.data
} }
@@ -53,7 +53,7 @@ export async function updateModel(
modelId: string, modelId: string,
data: ModelUpdate data: ModelUpdate
): Promise<Model> { ): Promise<Model> {
const response = await client.patch(`/api/admin/providers/${providerId}/models/${modelId}`, data) const response = await client.patch<Model>(`/api/admin/providers/${providerId}/models/${modelId}`, data)
return response.data return response.data
} }
@@ -64,7 +64,7 @@ export async function deleteModel(
providerId: string, providerId: string,
modelId: string modelId: string
): Promise<{ message: string }> { ): Promise<{ message: string }> {
const response = await client.delete(`/api/admin/providers/${providerId}/models/${modelId}`) const response = await client.delete<{ message: string }>(`/api/admin/providers/${providerId}/models/${modelId}`)
return response.data return response.data
} }
@@ -75,7 +75,7 @@ export async function batchCreateModels(
providerId: string, providerId: string,
modelsData: ModelCreate[] modelsData: ModelCreate[]
): Promise<Model[]> { ): Promise<Model[]> {
const response = await client.post(`/api/admin/providers/${providerId}/models/batch`, modelsData) const response = await client.post<Model[]>(`/api/admin/providers/${providerId}/models/batch`, modelsData)
return response.data return response.data
} }
@@ -83,7 +83,7 @@ export async function batchCreateModels(
* *
*/ */
export async function getModelCatalog(): Promise<ModelCatalogResponse> { export async function getModelCatalog(): Promise<ModelCatalogResponse> {
const response = await client.get('/api/admin/models/catalog') const response = await client.get<ModelCatalogResponse>('/api/admin/models/catalog')
return response.data return response.data
} }
@@ -93,7 +93,7 @@ export async function getModelCatalog(): Promise<ModelCatalogResponse> {
export async function getProviderAvailableSourceModels( export async function getProviderAvailableSourceModels(
providerId: string providerId: string
): Promise<ProviderAvailableSourceModelsResponse> { ): Promise<ProviderAvailableSourceModelsResponse> {
const response = await client.get(`/api/admin/providers/${providerId}/available-source-models`) const response = await client.get<ProviderAvailableSourceModelsResponse>(`/api/admin/providers/${providerId}/available-source-models`)
return response.data return response.data
} }
@@ -114,7 +114,17 @@ export async function batchAssignModelsToProvider(
error: string error: string
}> }>
}> { }> {
const response = await client.post( const response = await client.post<{
success: Array<{
global_model_id: string
global_model_name: string
model_id: string
}>
errors: Array<{
global_model_id: string
error: string
}>
}>(
`/api/admin/providers/${providerId}/assign-global-models`, `/api/admin/providers/${providerId}/assign-global-models`,
{ global_model_ids: globalModelIds } { global_model_ids: globalModelIds }
) )
@@ -137,7 +147,7 @@ export async function importModelsFromUpstream(
price_per_request?: number price_per_request?: number
} }
): Promise<ImportFromUpstreamResponse> { ): Promise<ImportFromUpstreamResponse> {
const response = await client.post( const response = await client.post<ImportFromUpstreamResponse>(
`/api/admin/providers/${providerId}/import-from-upstream`, `/api/admin/providers/${providerId}/import-from-upstream`,
{ {
model_ids: modelIds, model_ids: modelIds,
+4 -3
View File
@@ -2,6 +2,7 @@ import client from '../client'
import { buildCacheKey, cachedRequest } from '@/utils/cache' import { buildCacheKey, cachedRequest } from '@/utils/cache'
import type { import type {
AllowedModels, AllowedModels,
ProviderType,
OAuthOrganizationInfo, OAuthOrganizationInfo,
ProxyConfig, ProxyConfig,
UpstreamMetadata, UpstreamMetadata,
@@ -77,7 +78,7 @@ export async function resetPoolCost(
export interface PoolOverviewItem { export interface PoolOverviewItem {
provider_id: string provider_id: string
provider_name: string provider_name: string
provider_type: string provider_type: ProviderType
total_keys: number total_keys: number
active_keys: number active_keys: number
cooldown_count: number cooldown_count: number
@@ -508,7 +509,7 @@ export async function batchActionPoolKeys(
providerId: string, providerId: string,
body: PoolBatchAction, body: PoolBatchAction,
): Promise<{ affected: number; message: string; task_id?: string }> { ): Promise<{ affected: number; message: string; task_id?: string }> {
const response = await client.post( const response = await client.post<{ affected: number; message: string; task_id?: string }>(
`/api/admin/pool/${providerId}/keys/batch-action`, `/api/admin/pool/${providerId}/keys/batch-action`,
body, body,
{ timeout: POOL_BATCH_ACTION_TIMEOUT_MS }, { timeout: POOL_BATCH_ACTION_TIMEOUT_MS },
@@ -561,7 +562,7 @@ export async function getPoolBatchDeleteTask(
export async function cleanupBannedPoolKeys( export async function cleanupBannedPoolKeys(
providerId: string, providerId: string,
): Promise<{ affected: number; message: string }> { ): Promise<{ affected: number; message: string }> {
const response = await client.post( const response = await client.post<{ affected: number; message: string }>(
`/api/admin/pool/${providerId}/keys/cleanup-banned`, `/api/admin/pool/${providerId}/keys/cleanup-banned`,
undefined, undefined,
{ timeout: POOL_BATCH_ACTION_TIMEOUT_MS }, { timeout: POOL_BATCH_ACTION_TIMEOUT_MS },
+11 -11
View File
@@ -230,14 +230,14 @@ export function normalizeBatchImportCredentials(text: string): BatchImportCreden
} }
export async function refreshProviderOAuth(keyId: string): Promise<ProviderOAuthCompleteResponse> { export async function refreshProviderOAuth(keyId: string): Promise<ProviderOAuthCompleteResponse> {
const resp = await client.post(`/api/admin/provider-oauth/keys/${keyId}/refresh`) const resp = await client.post<ProviderOAuthCompleteResponse>(`/api/admin/provider-oauth/keys/${keyId}/refresh`)
return resp.data return resp.data
} }
// Provider-level OAuth (不需要预先创建 key) // Provider-level OAuth (不需要预先创建 key)
export async function startProviderLevelOAuth(providerId: string): Promise<ProviderOAuthStartResponse> { export async function startProviderLevelOAuth(providerId: string): Promise<ProviderOAuthStartResponse> {
const resp = await client.post(`/api/admin/provider-oauth/providers/${providerId}/start`) const resp = await client.post<ProviderOAuthStartResponse>(`/api/admin/provider-oauth/providers/${providerId}/start`)
return resp.data return resp.data
} }
@@ -245,7 +245,7 @@ export async function completeProviderLevelOAuth(
providerId: string, providerId: string,
data: ProviderOAuthCompleteRequest data: ProviderOAuthCompleteRequest
): Promise<ProviderOAuthCompleteResponseWithKey> { ): Promise<ProviderOAuthCompleteResponseWithKey> {
const resp = await client.post(`/api/admin/provider-oauth/providers/${providerId}/complete`, data) const resp = await client.post<ProviderOAuthCompleteResponseWithKey>(`/api/admin/provider-oauth/providers/${providerId}/complete`, data)
return resp.data return resp.data
} }
@@ -253,7 +253,7 @@ export async function authorizeProviderWithCookie(
providerId: string, providerId: string,
data: ProviderCookieAuthorizeRequest data: ProviderCookieAuthorizeRequest
): Promise<ProviderOAuthCompleteResponseWithKey> { ): Promise<ProviderOAuthCompleteResponseWithKey> {
const resp = await client.post( const resp = await client.post<ProviderOAuthCompleteResponseWithKey>(
`/api/admin/provider-oauth/providers/${providerId}/cookie-authorize`, `/api/admin/provider-oauth/providers/${providerId}/cookie-authorize`,
data, data,
{ timeout: CLAUDE_COOKIE_AUTHORIZE_TIMEOUT_MS }, { timeout: CLAUDE_COOKIE_AUTHORIZE_TIMEOUT_MS },
@@ -265,7 +265,7 @@ export async function startProviderCookieAuthorizeTask(
providerId: string, providerId: string,
data: ProviderCookieAuthorizeBatchTaskRequest, data: ProviderCookieAuthorizeBatchTaskRequest,
): Promise<OAuthBatchImportTaskStartResponse> { ): Promise<OAuthBatchImportTaskStartResponse> {
const resp = await client.post( const resp = await client.post<OAuthBatchImportTaskStartResponse>(
`/api/admin/provider-oauth/providers/${providerId}/cookie-authorize/tasks`, `/api/admin/provider-oauth/providers/${providerId}/cookie-authorize/tasks`,
data, data,
) )
@@ -276,7 +276,7 @@ export async function getProviderCookieAuthorizeTaskStatus(
providerId: string, providerId: string,
taskId: string, taskId: string,
): Promise<OAuthBatchImportTaskStatusResponse> { ): Promise<OAuthBatchImportTaskStatusResponse> {
const resp = await client.get( const resp = await client.get<OAuthBatchImportTaskStatusResponse>(
`/api/admin/provider-oauth/providers/${providerId}/cookie-authorize/tasks/${taskId}`, `/api/admin/provider-oauth/providers/${providerId}/cookie-authorize/tasks/${taskId}`,
) )
return resp.data return resp.data
@@ -313,7 +313,7 @@ export async function importProviderRefreshToken(
headers?: Record<string, string> headers?: Record<string, string>
} }
): Promise<ProviderOAuthCompleteResponseWithKey> { ): Promise<ProviderOAuthCompleteResponseWithKey> {
const resp = await client.post(`/api/admin/provider-oauth/providers/${providerId}/import-refresh-token`, data) const resp = await client.post<ProviderOAuthCompleteResponseWithKey>(`/api/admin/provider-oauth/providers/${providerId}/import-refresh-token`, data)
return resp.data return resp.data
} }
@@ -325,7 +325,7 @@ export async function startBatchImportOAuthTask(
const route = containsAgentIdentityImport(credentials) const route = containsAgentIdentityImport(credentials)
? 'agent-identity-import/tasks' ? 'agent-identity-import/tasks'
: 'batch-import/tasks' : 'batch-import/tasks'
const resp = await client.post(`/api/admin/provider-oauth/providers/${providerId}/${route}`, { const resp = await client.post<OAuthBatchImportTaskStartResponse>(`/api/admin/provider-oauth/providers/${providerId}/${route}`, {
credentials, credentials,
proxy_node_id: proxyNodeId || undefined, proxy_node_id: proxyNodeId || undefined,
}) })
@@ -339,7 +339,7 @@ export async function getBatchImportOAuthTaskStatus(
const route = taskId.startsWith('agent-identity-') const route = taskId.startsWith('agent-identity-')
? 'agent-identity-import/tasks' ? 'agent-identity-import/tasks'
: 'batch-import/tasks' : 'batch-import/tasks'
const resp = await client.get(`/api/admin/provider-oauth/providers/${providerId}/${route}/${taskId}`) const resp = await client.get<OAuthBatchImportTaskStatusResponse>(`/api/admin/provider-oauth/providers/${providerId}/${route}/${taskId}`)
return resp.data return resp.data
} }
@@ -412,7 +412,7 @@ export async function startDeviceAuthorize(
providerId: string, providerId: string,
data: DeviceAuthorizeRequest data: DeviceAuthorizeRequest
): Promise<DeviceAuthorizeResponse> { ): Promise<DeviceAuthorizeResponse> {
const resp = await client.post(`/api/admin/provider-oauth/providers/${providerId}/device-authorize`, data) const resp = await client.post<DeviceAuthorizeResponse>(`/api/admin/provider-oauth/providers/${providerId}/device-authorize`, data)
return resp.data return resp.data
} }
@@ -420,6 +420,6 @@ export async function pollDeviceAuthorize(
providerId: string, providerId: string,
data: DevicePollRequest data: DevicePollRequest
): Promise<DevicePollResponse> { ): Promise<DevicePollResponse> {
const resp = await client.post(`/api/admin/provider-oauth/providers/${providerId}/device-poll`, data) const resp = await client.post<DevicePollResponse>(`/api/admin/provider-oauth/providers/${providerId}/device-poll`, data)
return resp.data return resp.data
} }
+5 -4
View File
@@ -14,6 +14,7 @@ import {
normalizePoolAdvancedConfig as normalizePoolAdvanced, normalizePoolAdvancedConfig as normalizePoolAdvanced,
} from './types' } from './types'
export type { ProviderWithEndpointsSummary } from './types'
interface ProviderRequestOptions { interface ProviderRequestOptions {
timeout?: number timeout?: number
} }
@@ -140,7 +141,7 @@ export async function updateProvider(
}>, }>,
requestOptions?: ProviderRequestOptions, requestOptions?: ProviderRequestOptions,
): Promise<ProviderWithEndpointsSummary> { ): Promise<ProviderWithEndpointsSummary> {
const response = await client.patch(`/api/admin/providers/${providerId}`, data, requestOptions) const response = await client.patch<ProviderWithEndpointsSummary>(`/api/admin/providers/${providerId}`, data, requestOptions)
return normalizeProviderSummary(response.data) return normalizeProviderSummary(response.data)
} }
@@ -175,7 +176,7 @@ export async function createProvider(
config?: ProviderConfig | null config?: ProviderConfig | null
} }
): Promise<{ id: string; name: string; message?: string }> { ): Promise<{ id: string; name: string; message?: string }> {
const response = await client.post('/api/admin/providers/', data) const response = await client.post<{ id: string; name: string; message?: string }>('/api/admin/providers/', data)
return response.data return response.data
} }
@@ -261,7 +262,7 @@ export async function testModel(
data: TestModelRequest, data: TestModelRequest,
options: { signal?: AbortSignal } = {}, options: { signal?: AbortSignal } = {},
): Promise<TestModelResponse> { ): Promise<TestModelResponse> {
const response = await client.post('/api/admin/provider-query/test-model', data, { const response = await client.post<TestModelResponse>('/api/admin/provider-query/test-model', data, {
timeout: 10 * 60 * 1000, timeout: 10 * 60 * 1000,
signal: options.signal, signal: options.signal,
}) })
@@ -350,7 +351,7 @@ export async function testModelFailover(
const failoverModels = Array.isArray(data.failover_models) && data.failover_models.length > 0 const failoverModels = Array.isArray(data.failover_models) && data.failover_models.length > 0
? data.failover_models ? data.failover_models
: (normalizedModelName ? [normalizedModelName] : undefined) : (normalizedModelName ? [normalizedModelName] : undefined)
const response = await client.post('/api/admin/provider-query/test-model-failover', { const response = await client.post<TestModelFailoverResponse>('/api/admin/provider-query/test-model-failover', {
...data, ...data,
...(failoverModels ? { failover_models: failoverModels } : {}), ...(failoverModels ? { failover_models: failoverModels } : {}),
}, { }, {
+27 -7
View File
@@ -1,5 +1,12 @@
import type { ProviderModelMapping } from './provider' import type { ProviderModelMapping } from './provider'
export interface ModelProviderReference {
id: string
model_id?: string | null
name: string
is_active: boolean
}
// ========== 阶梯计费类型 ========== // ========== 阶梯计费类型 ==========
/** 缓存时长定价配置 */ /** 缓存时长定价配置 */
@@ -67,13 +74,26 @@ export interface ProviderTieredPricingConfig {
[key: string]: unknown [key: string]: unknown
} }
export interface ModelConfig extends Record<string, unknown> {
description?: string
model_mappings?: string[]
api_formats?: string[]
billing?: {
video?: {
price_per_second_by_resolution?: Record<string, number>
[key: string]: unknown
}
[key: string]: unknown
}
}
export interface Model { export interface Model {
id: string id: string
provider_id: string provider_id: string
global_model_id: string // 关联的 GlobalModel ID global_model_id: string // 关联的 GlobalModel ID
provider_model_name: string // Provider 侧的主模型名称 provider_model_name: string // Provider 侧的主模型名称
provider_model_mappings?: ProviderModelMapping[] | null // 模型名称映射列表(带优先级) provider_model_mappings?: ProviderModelMapping[] | null // 模型名称映射列表(带优先级)
config?: Record<string, unknown> | null // 额外配置(如 billing/video 等) config?: ModelConfig | null // 额外配置(如 billing/video 等)
// 原始配置值(可能为空,为空时使用 GlobalModel 默认值) // 原始配置值(可能为空,为空时使用 GlobalModel 默认值)
price_per_request?: number | null // 按次计费价格 price_per_request?: number | null // 按次计费价格
tiered_pricing?: ProviderTieredPricingConfig | null // Provider 原始覆盖,可仅包含 processing_tiers tiered_pricing?: ProviderTieredPricingConfig | null // Provider 原始覆盖,可仅包含 processing_tiers
@@ -102,7 +122,7 @@ export interface Model {
global_model_name?: string global_model_name?: string
global_model_display_name?: string global_model_display_name?: string
// 有效配置(合并 Model 和 GlobalModel 的 config // 有效配置(合并 Model 和 GlobalModel 的 config
effective_config?: Record<string, unknown> | null effective_config?: ModelConfig | null
model_test_capabilities?: ModelTestCapabilities | null model_test_capabilities?: ModelTestCapabilities | null
} }
@@ -120,7 +140,7 @@ export interface ModelCreate {
supports_extended_thinking?: boolean supports_extended_thinking?: boolean
supports_image_generation?: boolean supports_image_generation?: boolean
is_active?: boolean is_active?: boolean
config?: Record<string, unknown> config?: ModelConfig
} }
export interface ModelUpdate { export interface ModelUpdate {
@@ -136,7 +156,7 @@ export interface ModelUpdate {
supports_image_generation?: boolean supports_image_generation?: boolean
is_active?: boolean is_active?: boolean
is_available?: boolean is_available?: boolean
config?: Record<string, unknown> | null config?: ModelConfig | null
} }
export interface ModelCapabilities { export interface ModelCapabilities {
@@ -243,7 +263,7 @@ export interface GlobalModelCreate {
// Key 能力配置 - 模型支持的能力列表 // Key 能力配置 - 模型支持的能力列表
supported_capabilities?: string[] supported_capabilities?: string[]
// 模型配置(JSON格式)- 包含能力、规格、元信息等 // 模型配置(JSON格式)- 包含能力、规格、元信息等
config?: Record<string, unknown> config?: ModelConfig
is_active?: boolean is_active?: boolean
} }
@@ -257,7 +277,7 @@ export interface GlobalModelUpdate {
// Key 能力配置 - 模型支持的能力列表 // Key 能力配置 - 模型支持的能力列表
supported_capabilities?: string[] | null supported_capabilities?: string[] | null
// 模型配置(JSON格式)- 包含能力、规格、元信息等 // 模型配置(JSON格式)- 包含能力、规格、元信息等
config?: Record<string, unknown> | null config?: ModelConfig | null
} }
export interface GlobalModelResponse { export interface GlobalModelResponse {
@@ -273,7 +293,7 @@ export interface GlobalModelResponse {
supported_capabilities?: string[] | null supported_capabilities?: string[] | null
supports_embedding?: boolean | null supports_embedding?: boolean | null
// 模型配置(JSON格式) // 模型配置(JSON格式)
config?: Record<string, unknown> | null config?: ModelConfig | null
// 统计数据 // 统计数据
provider_count?: number provider_count?: number
active_provider_count?: number active_provider_count?: number
+6 -6
View File
@@ -68,7 +68,7 @@ export const geminiFilesApi = {
* *
*/ */
async getStats(): Promise<FileMappingStatsResponse> { async getStats(): Promise<FileMappingStatsResponse> {
const response = await apiClient.get('/api/admin/gemini-files/stats') const response = await apiClient.get<FileMappingStatsResponse>('/api/admin/gemini-files/stats')
return response.data return response.data
}, },
@@ -76,7 +76,7 @@ export const geminiFilesApi = {
* *
*/ */
async listMappings(params?: ListMappingsParams): Promise<FileMappingListResponse> { async listMappings(params?: ListMappingsParams): Promise<FileMappingListResponse> {
const response = await apiClient.get('/api/admin/gemini-files/mappings', { params }) const response = await apiClient.get<FileMappingListResponse>('/api/admin/gemini-files/mappings', { params })
return response.data return response.data
}, },
@@ -84,7 +84,7 @@ export const geminiFilesApi = {
* *
*/ */
async deleteMapping(mappingId: string): Promise<{ message: string; file_name: string }> { async deleteMapping(mappingId: string): Promise<{ message: string; file_name: string }> {
const response = await apiClient.delete(`/api/admin/gemini-files/mappings/${mappingId}`) const response = await apiClient.delete<{ message: string; file_name: string }>(`/api/admin/gemini-files/mappings/${mappingId}`)
return response.data return response.data
}, },
@@ -92,7 +92,7 @@ export const geminiFilesApi = {
* *
*/ */
async cleanupExpired(): Promise<{ message: string; deleted_count: number }> { async cleanupExpired(): Promise<{ message: string; deleted_count: number }> {
const response = await apiClient.delete('/api/admin/gemini-files/mappings') const response = await apiClient.delete<{ message: string; deleted_count: number }>('/api/admin/gemini-files/mappings')
return response.data return response.data
}, },
@@ -100,7 +100,7 @@ export const geminiFilesApi = {
* Key * Key
*/ */
async getCapableKeys(): Promise<CapableKeyResponse[]> { async getCapableKeys(): Promise<CapableKeyResponse[]> {
const response = await apiClient.get('/api/admin/gemini-files/capable-keys') const response = await apiClient.get<CapableKeyResponse[]>('/api/admin/gemini-files/capable-keys')
return response.data return response.data
}, },
@@ -110,7 +110,7 @@ export const geminiFilesApi = {
async uploadFile(file: File, keyIds: string[]): Promise<UploadResponse> { async uploadFile(file: File, keyIds: string[]): Promise<UploadResponse> {
const formData = new FormData() const formData = new FormData()
formData.append('file', file) formData.append('file', file)
const response = await apiClient.post( const response = await apiClient.post<UploadResponse>(
`/api/admin/gemini-files/upload?key_ids=${keyIds.join(',')}`, `/api/admin/gemini-files/upload?key_ids=${keyIds.join(',')}`,
formData, formData,
{ {
+86 -16
View File
@@ -1,4 +1,5 @@
import apiClient from './client' import apiClient from './client'
import type { ImageProgress } from './requestTrace'
import type { ActivityHeatmap } from '@/types/activity' import type { ActivityHeatmap } from '@/types/activity'
import type { TieredPricingConfig } from './endpoints/types' import type { TieredPricingConfig } from './endpoints/types'
import { cachedRequest, buildCacheKey } from '@/utils/cache' import { cachedRequest, buildCacheKey } from '@/utils/cache'
@@ -238,13 +239,13 @@ export const meApi = {
username?: string username?: string
feature_settings?: FeatureSettingsMap | null feature_settings?: FeatureSettingsMap | null
}): Promise<{ message: string }> { }): Promise<{ message: string }> {
const response = await apiClient.put('/api/users/me', data) const response = await apiClient.put<{ message: string }>('/api/users/me', data)
return response.data return response.data
}, },
// 修改密码 // 修改密码
async changePassword(data: ChangePasswordRequest): Promise<{ message: string }> { async changePassword(data: ChangePasswordRequest): Promise<{ message: string }> {
const response = await apiClient.patch('/api/users/me/password', data) const response = await apiClient.patch<{ message: string }>('/api/users/me/password', data)
return response.data return response.data
}, },
@@ -261,12 +262,12 @@ export const meApi = {
}, },
async revokeSession(sessionId: string): Promise<{ message: string }> { async revokeSession(sessionId: string): Promise<{ message: string }> {
const response = await apiClient.delete(`/api/users/me/sessions/${sessionId}`) const response = await apiClient.delete<{ message: string }>(`/api/users/me/sessions/${sessionId}`)
return response.data return response.data
}, },
async revokeOtherSessions(): Promise<{ message: string; revoked_count: number }> { async revokeOtherSessions(): Promise<{ message: string; revoked_count: number }> {
const response = await apiClient.delete('/api/users/me/sessions/others') const response = await apiClient.delete<{ message: string; revoked_count: number }>('/api/users/me/sessions/others')
return response.data return response.data
}, },
@@ -303,7 +304,7 @@ export const meApi = {
}, },
async deleteApiKey(keyId: string): Promise<{ message: string }> { async deleteApiKey(keyId: string): Promise<{ message: string }> {
const response = await apiClient.delete(`/api/users/me/api-keys/${keyId}`) const response = await apiClient.delete<{ message: string }>(`/api/users/me/api-keys/${keyId}`)
return response.data return response.data
}, },
@@ -396,16 +397,61 @@ export const meApi = {
reasoning_effort?: string | null reasoning_effort?: string | null
service_tier?: string | null service_tier?: string | null
actual_service_tier?: string | null actual_service_tier?: string | null
image_progress?: ImageProgress | null
}> }>
}> { }> {
const params = ids ? { ids } : {} const params = ids ? { ids } : {}
const response = await apiClient.get('/api/users/me/usage/active', { params }) const response = await apiClient.get<{
requests: Array<{
id: string
status: 'pending' | 'streaming' | 'completed' | 'failed' | 'cancelled'
input_tokens: number
effective_input_tokens?: number | null
output_tokens: number
cache_creation_input_tokens?: number | null
cache_creation_ephemeral_5m_input_tokens?: number | null
cache_creation_ephemeral_1h_input_tokens?: number | null
cache_read_input_tokens?: number | null
cost: number
actual_cost?: number | null
rate_multiplier?: number | null
response_time_ms: number | null
first_byte_time_ms: number | null
end_to_end_time_ms?: number | null
end_to_end_first_byte_time_ms?: number | null
updated_at?: string | null
response_time_updated_at?: string | null
status_code?: number | null
error_message?: string | null
api_format?: string | null
endpoint_api_format?: string | null
is_stream?: boolean | null
is_websocket?: boolean | null
websocket_transport?: string | null
usage_available?: boolean | null
usage_pricing_available?: boolean | null
input_audio_tokens?: number | null
output_audio_tokens?: number | null
upstream_is_stream?: boolean | null
client_requested_stream?: boolean | null
client_is_stream?: boolean | null
has_format_conversion?: boolean | null
has_fallback?: boolean | null
target_model?: string | null
request_type?: string | null
requested_reasoning_effort?: string | null
reasoning_effort?: string | null
service_tier?: string | null
actual_service_tier?: string | null
image_progress?: ImageProgress | null
}>
}>('/api/users/me/usage/active', { params })
return response.data return response.data
}, },
// 获取可用的提供商 // 获取可用的提供商
async getAvailableProviders(): Promise<Array<Record<string, unknown>>> { async getAvailableProviders(): Promise<Array<Record<string, unknown>>> {
const response = await apiClient.get('/api/users/me/providers') const response = await apiClient.get<Array<Record<string, unknown>>>('/api/users/me/providers')
return response.data return response.data
}, },
@@ -429,24 +475,38 @@ export const meApi = {
}> }>
total: number total: number
}> { }> {
const response = await apiClient.get('/api/users/me/available-models', { params }) const response = await apiClient.get<{
models: Array<{
id: string
name: string
display_name: string | null
is_active: boolean
default_price_per_request: number | null
default_tiered_pricing: TieredPricingConfig | null
supported_capabilities: string[] | null
supports_embedding?: boolean | null
config: Record<string, unknown> | null
usage_count: number
}>
total: number
}>('/api/users/me/available-models', { params })
return response.data return response.data
}, },
// 获取端点状态(不包含敏感信息) // 获取端点状态(不包含敏感信息)
async getEndpointStatus(): Promise<Array<Record<string, unknown>>> { async getEndpointStatus(): Promise<Array<Record<string, unknown>>> {
const response = await apiClient.get('/api/users/me/endpoint-status') const response = await apiClient.get<Array<Record<string, unknown>>>('/api/users/me/endpoint-status')
return response.data return response.data
}, },
// 偏好设置 // 偏好设置
async getPreferences(): Promise<UserPreferences> { async getPreferences(): Promise<UserPreferences> {
const response = await apiClient.get('/api/users/me/preferences') const response = await apiClient.get<UserPreferences>('/api/users/me/preferences')
return response.data return response.data
}, },
async updatePreferences(data: Partial<UserPreferences>): Promise<{ message: string }> { async updatePreferences(data: Partial<UserPreferences>): Promise<{ message: string }> {
const response = await apiClient.put('/api/users/me/preferences', data) const response = await apiClient.put<{ message: string }>('/api/users/me/preferences', data)
return response.data return response.data
}, },
@@ -456,7 +516,7 @@ export const meApi = {
async updateApiKeyProviders(keyId: string, data: { async updateApiKeyProviders(keyId: string, data: {
allowed_providers?: ProviderConfig[] allowed_providers?: ProviderConfig[]
}): Promise<{ message: string }> { }): Promise<{ message: string }> {
const response = await apiClient.put(`/api/users/me/api-keys/${keyId}/providers`, data) const response = await apiClient.put<{ message: string }>(`/api/users/me/api-keys/${keyId}/providers`, data)
return response.data return response.data
}, },
@@ -464,7 +524,7 @@ export const meApi = {
async updateApiKeyCapabilities(keyId: string, data: { async updateApiKeyCapabilities(keyId: string, data: {
force_capabilities?: Record<string, boolean> | null force_capabilities?: Record<string, boolean> | null
}): Promise<{ message: string; force_capabilities?: Record<string, boolean> | null }> { }): Promise<{ message: string; force_capabilities?: Record<string, boolean> | null }> {
const response = await apiClient.put(`/api/users/me/api-keys/${keyId}/capabilities`, data) const response = await apiClient.put<{ message: string; force_capabilities?: Record<string, boolean> | null }>(`/api/users/me/api-keys/${keyId}/capabilities`, data)
return response.data return response.data
}, },
@@ -472,7 +532,9 @@ export const meApi = {
async getModelCapabilitySettings(): Promise<{ async getModelCapabilitySettings(): Promise<{
model_capability_settings: Record<string, Record<string, boolean>> model_capability_settings: Record<string, Record<string, boolean>>
}> { }> {
const response = await apiClient.get('/api/users/me/model-capabilities') const response = await apiClient.get<{
model_capability_settings: Record<string, Record<string, boolean>>
}>('/api/users/me/model-capabilities')
return response.data return response.data
}, },
@@ -482,7 +544,10 @@ export const meApi = {
message: string message: string
model_capability_settings: Record<string, Record<string, boolean>> | null model_capability_settings: Record<string, Record<string, boolean>> | null
}> { }> {
const response = await apiClient.put('/api/users/me/model-capabilities', data) const response = await apiClient.put<{
message: string
model_capability_settings: Record<string, Record<string, boolean>> | null
}>('/api/users/me/model-capabilities', data)
return response.data return response.data
}, },
@@ -500,7 +565,12 @@ export const meApi = {
return cachedRequest( return cachedRequest(
cacheKey, cacheKey,
async () => { async () => {
const response = await apiClient.get('/api/users/me/usage/interval-timeline', { params }) const response = await apiClient.get<{
analysis_period_hours: number
total_points: number
points: Array<{ x: string; y: number; model?: string }>
models?: string[]
}>('/api/users/me/usage/interval-timeline', { params })
return response.data return response.data
}, },
30000 30000
+64
View File
@@ -0,0 +1,64 @@
export interface CredentialsSchema {
type: 'object'
properties: Record<string, SchemaProperty>
required?: string[]
'x-field-groups'?: SchemaFieldGroup[]
'x-auth-type'?: string
'x-auth-method'?: string
'x-validation'?: SchemaValidation[]
'x-quota-divisor'?: number | null
'x-currency'?: string
'x-default-base-url'?: string
'x-balance-extra-format'?: BalanceExtraFormat[]
'x-field-hooks'?: Record<string, { action: string; target: string }>
}
export interface SchemaProperty {
type: string
title?: string
description?: string
'x-sensitive'?: boolean
'x-input-type'?: string
'x-default-value'?: string
'x-help'?: string
}
export interface SchemaFieldGroup {
fields: string[]
layout?: 'inline' | 'vertical'
'x-flex'?: Record<string, number>
'x-help'?: string
}
interface SchemaValidation {
type: 'required' | 'any_required' | 'conditional_required'
fields?: string[]
message: string
/** conditional_required: 当此字段有值时 */
if?: string
/** conditional_required: 除非此字段有值 */
unless?: string
/** conditional_required: 则这些字段必填 */
then?: string[]
}
export interface BalanceExtraFormat {
label: string
type: 'window_limit' | 'daily_quota' | 'weekly_spent' | 'monthly_expiry'
/** window_limit: extra 中的字段名 */
source?: string
/** window_limit: 单位除数 */
unit_divisor?: number
/** daily_quota / weekly_spent: limit 字段名 */
source_limit?: string
/** daily_quota: remaining 字段名 */
source_remaining?: string
/** daily_quota: 每日重置基准时间字段名(计算下次重置时间) */
source_start_date?: string
/** weekly_spent: spent 字段名 */
source_spent?: string
/** weekly_spent: resets_at 字段名 */
source_resets_at?: string
/** monthly_expiry: 到期日期字段名 */
source_end_date?: string
}
+3 -2
View File
@@ -8,6 +8,7 @@
*/ */
import client from './client' import client from './client'
import type { CredentialsSchema } from './providerCredentials'
// ==================== Types ==================== // ==================== Types ====================
@@ -46,11 +47,11 @@ export interface ArchitectureInfo {
architecture_id: string architecture_id: string
display_name: string display_name: string
description: string description: string
credentials_schema: Record<string, unknown> credentials_schema: CredentialsSchema
supported_auth_types: Array<{ supported_auth_types: Array<{
type: string type: string
display_name: string display_name: string
credentials_schema?: Record<string, unknown> credentials_schema?: CredentialsSchema
}> }>
supported_actions: Array<{ supported_actions: Array<{
type: string type: string
+1 -1
View File
@@ -36,6 +36,6 @@ export async function getPublicGlobalModels(params?: {
is_active?: boolean is_active?: boolean
search?: string search?: string
}): Promise<PublicGlobalModelListResponse> { }): Promise<PublicGlobalModelListResponse> {
const response = await client.get('/api/public/global-models', { params }) const response = await client.get<PublicGlobalModelListResponse>('/api/public/global-models', { params })
return response.data return response.data
} }
+4 -4
View File
@@ -87,7 +87,7 @@ export const referralApi = {
async getAdminReferrals( async getAdminReferrals(
params: ReferralRelationshipQuery = {} params: ReferralRelationshipQuery = {}
): Promise<ReferralListResponse<ReferralRelationshipRecord>> { ): Promise<ReferralListResponse<ReferralRelationshipRecord>> {
const response = await apiClient.get('/api/admin/referrals', { const response = await apiClient.get<ReferralListResponse<ReferralRelationshipRecord>>('/api/admin/referrals', {
params: cleanParams(params as Record<string, unknown>) params: cleanParams(params as Record<string, unknown>)
}) })
return response.data return response.data
@@ -96,19 +96,19 @@ export const referralApi = {
async getAdminReferralRewards( async getAdminReferralRewards(
params: ReferralRewardQuery = {} params: ReferralRewardQuery = {}
): Promise<ReferralListResponse<ReferralRewardRecord>> { ): Promise<ReferralListResponse<ReferralRewardRecord>> {
const response = await apiClient.get('/api/admin/referral-rewards', { const response = await apiClient.get<ReferralListResponse<ReferralRewardRecord>>('/api/admin/referral-rewards', {
params: cleanParams(params as Record<string, unknown>) params: cleanParams(params as Record<string, unknown>)
}) })
return response.data return response.data
}, },
async retryReferralReward(id: string, note?: string): Promise<{ reward: ReferralRewardRecord }> { async retryReferralReward(id: string, note?: string): Promise<{ reward: ReferralRewardRecord }> {
const response = await apiClient.post(`/api/admin/referral-rewards/${id}/retry`, { note }) const response = await apiClient.post<{ reward: ReferralRewardRecord }>(`/api/admin/referral-rewards/${id}/retry`, { note })
return response.data return response.data
}, },
async voidReferralReward(id: string, note?: string): Promise<{ reward: ReferralRewardRecord }> { async voidReferralReward(id: string, note?: string): Promise<{ reward: ReferralRewardRecord }> {
const response = await apiClient.post(`/api/admin/referral-rewards/${id}/void`, { note }) const response = await apiClient.post<{ reward: ReferralRewardRecord }>(`/api/admin/referral-rewards/${id}/void`, { note })
return response.data return response.data
} }
} }
+42
View File
@@ -31,6 +31,32 @@ export interface CandidateResponseBoundary {
body_state?: string | null body_state?: string | null
} }
export interface CandidateProxyTiming {
body_read_ms?: number | null
decompress_ms?: number | null
wire_size?: number | null
body_size?: number | null
ttfb_ms?: number | null
upstream_ms?: number | null
response_wait_ms?: number | null
connection_acquire_ms?: number | null
upstream_processing_ms?: number | null
connect_ms?: number | null
tls_ms?: number | null
dns_ms?: number | null
total_ms?: number | null
connection_reused?: boolean | null
}
export interface CandidateProxy {
node_name?: string | null
node_id?: string | null
url?: string | null
source?: string | null
ttfb_ms?: number | null
timing?: CandidateProxyTiming | null
}
export interface CandidateRecord { export interface CandidateRecord {
id: string id: string
request_id: string request_id: string
@@ -70,6 +96,22 @@ export interface CandidateRecord {
ranking?: CandidateRankingMetadata | null ranking?: CandidateRankingMetadata | null
image_progress?: ImageProgress | null image_progress?: ImageProgress | null
extra_data?: Record<string, unknown> & { extra_data?: Record<string, unknown> & {
first_byte_time_ms?: number | null
needs_conversion?: boolean
provider_api_format?: string | null
proxy?: CandidateProxy | null
pool_selection?: {
reason: string
cost_soft_threshold?: number | boolean | null
cost_window_usage?: number | null
cost_limit?: number | null
} | null
pool_skip?: {
type: string
cooldown_reason?: string | null
cooldown_ttl?: number | null
cost_window_usage?: number | null
} | null
upstream_response?: CandidateResponseBoundary upstream_response?: CandidateResponseBoundary
image_progress?: ImageProgress | null image_progress?: ImageProgress | null
} }
+3 -3
View File
@@ -59,7 +59,7 @@ export const blacklistApi = {
* *
*/ */
async getStats(): Promise<BlacklistStats> { async getStats(): Promise<BlacklistStats> {
const response = await apiClient.get('/api/admin/security/ip/blacklist/stats') const response = await apiClient.get<BlacklistStats>('/api/admin/security/ip/blacklist/stats')
return response.data return response.data
}, },
@@ -67,7 +67,7 @@ export const blacklistApi = {
* *
*/ */
async getList(): Promise<BlacklistResponse> { async getList(): Promise<BlacklistResponse> {
const response = await apiClient.get('/api/admin/security/ip/blacklist') const response = await apiClient.get<BlacklistResponse>('/api/admin/security/ip/blacklist')
return response.data return response.data
} }
} }
@@ -96,7 +96,7 @@ export const whitelistApi = {
* *
*/ */
async getList(): Promise<WhitelistResponse> { async getList(): Promise<WhitelistResponse> {
const response = await apiClient.get('/api/admin/security/ip/whitelist') const response = await apiClient.get<WhitelistResponse>('/api/admin/security/ip/whitelist')
return response.data return response.data
} }
} }
+64 -3
View File
@@ -2,6 +2,7 @@ import apiClient from './client'
import { cachedRequest, dedupedRequest, buildCacheKey } from '@/utils/cache' import { cachedRequest, dedupedRequest, buildCacheKey } from '@/utils/cache'
import type { ActivityHeatmap } from '@/types/activity' import type { ActivityHeatmap } from '@/types/activity'
import type { ImageProgress } from './requestTrace' import type { ImageProgress } from './requestTrace'
import type { UsageRecord as UsageListRecord } from './usageRecords'
const ACTIVITY_HEATMAP_CACHE_TTL_MS = 30 * 60 * 1000 const ACTIVITY_HEATMAP_CACHE_TTL_MS = 30 * 60 * 1000
const USAGE_ANALYTICS_CACHE_TTL_MS = 30 * 1000 const USAGE_ANALYTICS_CACHE_TTL_MS = 30 * 1000
@@ -58,6 +59,12 @@ export interface UsageStats {
avg_response_time: number avg_response_time: number
error_count?: number error_count?: number
error_rate?: number error_rate?: number
cache_stats?: {
cache_creation_tokens: number
cache_read_tokens: number
cache_creation_cost: number
cache_read_cost: number
}
today?: { today?: {
requests: number requests: number
tokens: number tokens: number
@@ -67,6 +74,7 @@ export interface UsageStats {
} }
export interface UsageByModel { export interface UsageByModel {
actual_cost?: number
model: string model: string
request_count: number request_count: number
total_tokens: number total_tokens: number
@@ -511,7 +519,7 @@ export const usageApi = {
limit?: number limit?: number
offset?: number offset?: number
}): Promise<{ }): Promise<{
records: Array<Record<string, unknown>> records: UsageListRecord[]
total: number total: number
limit: number limit: number
offset: number offset: number
@@ -519,7 +527,13 @@ export const usageApi = {
}> { }> {
const key = buildCacheKey('usage:records', params as Record<string, unknown> | undefined) const key = buildCacheKey('usage:records', params as Record<string, unknown> | undefined)
return dedupedRequest(key, async () => { return dedupedRequest(key, async () => {
const response = await apiClient.get('/api/admin/usage/records', { params }) const response = await apiClient.get<{
records: UsageListRecord[]
total: number
limit: number
offset: number
total_is_estimated?: boolean
}>('/api/admin/usage/records', { params })
return response.data return response.data
}) })
}, },
@@ -630,7 +644,54 @@ export const usageApi = {
if (typeof timeRange?.tz_offset_minutes === 'number') { if (typeof timeRange?.tz_offset_minutes === 'number') {
params.tz_offset_minutes = timeRange.tz_offset_minutes params.tz_offset_minutes = timeRange.tz_offset_minutes
} }
const response = await apiClient.get('/api/admin/usage/active', { params }) const response = await apiClient.get<{
requests: Array<{
id: string
status: 'pending' | 'streaming' | 'completed' | 'failed' | 'cancelled'
input_tokens: number
effective_input_tokens?: number | null
output_tokens: number
cache_creation_input_tokens?: number | null
cache_creation_ephemeral_5m_input_tokens?: number | null
cache_creation_ephemeral_1h_input_tokens?: number | null
cache_read_input_tokens?: number | null
cost: number
actual_cost?: number | null
rate_multiplier?: number | null
response_time_ms: number | null
first_byte_time_ms: number | null
end_to_end_time_ms?: number | null
end_to_end_first_byte_time_ms?: number | null
updated_at?: string | null
response_time_updated_at?: string | null
status_code?: number | null
error_message?: string | null
provider?: string | null
api_key_name?: string | null
provider_key_name?: string | null
api_format?: string | null
endpoint_api_format?: string | null
is_stream?: boolean | null
is_websocket?: boolean | null
websocket_transport?: string | null
usage_available?: boolean | null
usage_pricing_available?: boolean | null
input_audio_tokens?: number | null
output_audio_tokens?: number | null
upstream_is_stream?: boolean | null
client_requested_stream?: boolean | null
client_is_stream?: boolean | null
has_format_conversion?: boolean | null
has_fallback?: boolean | null
target_model?: string | null
request_type?: string | null
requested_reasoning_effort?: string | null
reasoning_effort?: string | null
service_tier?: string | null
actual_service_tier?: string | null
image_progress?: ImageProgress | null
}>
}>('/api/admin/usage/active', { params })
return response.data return response.data
}, },
+69
View File
@@ -0,0 +1,69 @@
import type { ImageProgress } from './requestTrace'
export type RequestStatus = 'pending' | 'streaming' | 'completed' | 'failed' | 'cancelled'
export interface UsageRecord {
id: string
user_id?: string
username?: string
user_email?: string
api_key?: {
id: string | null
name: string | null
display: string | null
} | null
provider?: string // 仅管理员可见
api_key_name?: string
provider_key_name?: string | null
rate_multiplier?: number
model: string
target_model?: string | null // 映射后的目标模型名(若无映射则为空)
model_version?: string | null // Provider 返回的实际模型版本(列表轻量字段)
request_type?: string | null // 由请求语义识别出的操作类型
requested_reasoning_effort?: string | null // 用户请求侧 reasoning 级别,用于展示转换关系
reasoning_effort?: string | null // 从发送给 Provider 的请求体提取的 reasoning 级别
service_tier?: string | null // 从发送给 Provider 的请求体提取的服务层级
actual_service_tier?: string | null // 响应侧审计事实,不用于 Fast 展示或计费
api_format?: string
endpoint_api_format?: string // 端点原生格式
has_format_conversion?: boolean // 是否发生了格式转换
input_tokens: number
effective_input_tokens?: number
output_tokens: number
reasoning_tokens?: number
cache_creation_input_tokens?: number
cache_creation_ephemeral_5m_input_tokens?: number
cache_creation_ephemeral_1h_input_tokens?: number
cache_read_input_tokens?: number
total_tokens: number
cost: number
actual_cost?: number
response_time_ms?: number | null
first_byte_time_ms?: number | null // 首字时间 (TTFB)
end_to_end_time_ms?: number | null // 客户端从请求进入网关到完成的总耗时
end_to_end_first_byte_time_ms?: number | null // 客户端从请求进入网关到首字节的耗时
is_stream: boolean
is_websocket?: boolean
websocket_transport?: string | null
usage_available?: boolean
usage_pricing_available?: boolean
input_audio_tokens?: number | null
output_audio_tokens?: number | null
upstream_is_stream?: boolean
client_requested_stream?: boolean
client_is_stream?: boolean
client_family?: string | null
client_ip?: string | null
user_agent?: string | null
request_path?: string | null
request_path_and_query?: string | null
status_code?: number
error_message?: string
status?: RequestStatus // 请求状态: pending, streaming, completed, failed
created_at: string
updated_at?: string | null
response_time_updated_at?: string | null
has_fallback?: boolean
has_retry?: boolean
image_progress?: ImageProgress | null
}
+1 -1
View File
@@ -60,7 +60,7 @@ export interface User {
export interface CreateUserRequest { export interface CreateUserRequest {
username: string username: string
password: string password: string
email: string email?: string
role?: UserRole role?: UserRole
initial_gift_usd?: number | null initial_gift_usd?: number | null
unlimited?: boolean unlimited?: boolean
+17 -4
View File
@@ -223,7 +223,10 @@ export const walletApi = {
order: PaymentOrder order: PaymentOrder
payment_instructions: Record<string, unknown> payment_instructions: Record<string, unknown>
}> { }> {
const response = await apiClient.post('/api/wallet/recharge', payload) const response = await apiClient.post<{
order: PaymentOrder
payment_instructions: Record<string, unknown>
}>('/api/wallet/recharge', payload)
return response.data return response.data
}, },
@@ -238,12 +241,17 @@ export const walletApi = {
limit: number limit: number
offset: number offset: number
}> { }> {
const response = await apiClient.get('/api/wallet/recharge', { params }) const response = await apiClient.get<{
items: PaymentOrder[]
total: number
limit: number
offset: number
}>('/api/wallet/recharge', { params })
return response.data return response.data
}, },
async getRechargeOrder(orderId: string): Promise<{ order: PaymentOrder }> { async getRechargeOrder(orderId: string): Promise<{ order: PaymentOrder }> {
const response = await apiClient.get(`/api/wallet/recharge/${orderId}`) const response = await apiClient.get<{ order: PaymentOrder }>(`/api/wallet/recharge/${orderId}`)
return response.data return response.data
}, },
@@ -253,7 +261,12 @@ export const walletApi = {
limit: number limit: number
offset: number offset: number
}> { }> {
const response = await apiClient.get('/api/wallet/refunds', { params }) const response = await apiClient.get<{
items: RefundRequest[]
total: number
limit: number
offset: number
}>('/api/wallet/refunds', { params })
return response.data return response.data
}, },
+13 -13
View File
@@ -46,6 +46,7 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import type { TimeScatterChartData, TimeScatterPoint } from './types'
import { getI18nLocale, useI18n } from '@/i18n' import { getI18nLocale, useI18n } from '@/i18n'
import { ref, onMounted, onUnmounted, watch, nextTick, computed } from 'vue' import { ref, onMounted, onUnmounted, watch, nextTick, computed } from 'vue'
import { import {
@@ -57,7 +58,6 @@ import {
Title, Title,
Tooltip, Tooltip,
Legend, Legend,
type ChartData,
type ChartOptions, type ChartOptions,
type Plugin, type Plugin,
type Scale type Scale
@@ -84,7 +84,7 @@ ChartJS.register(
) )
interface Props { interface Props {
data: ChartData<'scatter'> data: TimeScatterChartData
options?: ChartOptions<'scatter'> options?: ChartOptions<'scatter'>
height?: number height?: number
compressGaps?: boolean compressGaps?: boolean
@@ -117,13 +117,13 @@ interface GapInfo {
const chartRef = ref<HTMLCanvasElement>() const chartRef = ref<HTMLCanvasElement>()
const { locale, t } = useI18n() const { locale, t } = useI18n()
let chart: ChartJS<'scatter'> | null = null let chart: ChartJS<'scatter', TimeScatterPoint[]> | null = null
const crosshairY = ref<number | null>(null) const crosshairY = ref<number | null>(null)
const gapInfoList = ref<GapInfo[]>([]) const gapInfoList = ref<GapInfo[]>([])
interface PreparedRenderData { interface PreparedRenderData {
chartData: ChartData<'scatter'> chartData: TimeScatterChartData
gaps: GapInfo[] gaps: GapInfo[]
} }
@@ -141,7 +141,7 @@ const crosshairStats = computed<CrosshairStats | null>(() => {
let dsTotal = 0 let dsTotal = 0
for (const point of dataset.data) { for (const point of dataset.data) {
const p = point as { x: string; y: number } const p = point
if (typeof p.y === 'number') { if (typeof p.y === 'number') {
dsTotal++ dsTotal++
totalCount++ totalCount++
@@ -205,8 +205,8 @@ function toRealValue(displayValue: number): number {
} }
// //
function compressTimeGaps(data: ChartData<'scatter'>): { function compressTimeGaps(data: TimeScatterChartData): {
data: ChartData<'scatter'> data: TimeScatterChartData
gaps: GapInfo[] gaps: GapInfo[]
timeMapping: Map<number, number> // -> timeMapping: Map<number, number> // ->
} { } {
@@ -216,7 +216,7 @@ function compressTimeGaps(data: ChartData<'scatter'>): {
// //
const allTimestamps: number[] = [] const allTimestamps: number[] = []
for (const dataset of data.datasets) { for (const dataset of data.datasets) {
for (const point of dataset.data as Array<{ x: string; y: number }>) { for (const point of dataset.data) {
allTimestamps.push(new Date(point.x).getTime()) allTimestamps.push(new Date(point.x).getTime())
} }
} }
@@ -268,11 +268,11 @@ function compressTimeGaps(data: ChartData<'scatter'>): {
} }
// //
const compressedData: ChartData<'scatter'> = { const compressedData: TimeScatterChartData = {
...data, ...data,
datasets: data.datasets.map(dataset => ({ datasets: data.datasets.map(dataset => ({
...dataset, ...dataset,
data: (dataset.data as Array<{ x: string; y: number }>).map(point => { data: (dataset.data).map(point => {
const originalTs = new Date(point.x).getTime() const originalTs = new Date(point.x).getTime()
const compressedTs = timeMapping.get(originalTs) ?? originalTs const compressedTs = timeMapping.get(originalTs) ?? originalTs
return { return {
@@ -288,12 +288,12 @@ function compressTimeGaps(data: ChartData<'scatter'>): {
} }
// Y // Y
function transformData(data: ChartData<'scatter'>): ChartData<'scatter'> { function transformData(data: TimeScatterChartData): TimeScatterChartData {
return { return {
...data, ...data,
datasets: data.datasets.map(dataset => ({ datasets: data.datasets.map(dataset => ({
...dataset, ...dataset,
data: (dataset.data as Array<{ x: string; y: number; _originalX?: string; _originalY?: number }>).map(point => ({ data: (dataset.data).map(point => ({
...point, ...point,
y: toDisplayValue(Math.min(point.y, 120)), y: toDisplayValue(Math.min(point.y, 120)),
_originalY: point._originalY ?? point.y // tooltip _originalY: point._originalY ?? point.y // tooltip
@@ -547,7 +547,7 @@ function createChart() {
const { chartData, gaps } = prepareRenderData() const { chartData, gaps } = prepareRenderData()
gapInfoList.value = gaps gapInfoList.value = gaps
chart = new ChartJS(chartRef.value, { chart = new ChartJS<'scatter', TimeScatterPoint[]>(chartRef.value, {
type: 'scatter', type: 'scatter',
data: chartData, data: chartData,
options: { options: {
+10
View File
@@ -0,0 +1,10 @@
import type { ChartData } from 'chart.js'
export interface TimeScatterPoint {
x: string | number
y: number
_originalX?: string | number
_originalY?: number
}
export type TimeScatterChartData = ChartData<'scatter', TimeScatterPoint[]>
@@ -21,7 +21,7 @@
: 'border-border hover:border-muted-foreground/40', : 'border-border hover:border-muted-foreground/40',
showManualInput ? 'opacity-0 pointer-events-none' : 'opacity-100', showManualInput ? 'opacity-0 pointer-events-none' : 'opacity-100',
]" ]"
:inert="showManualInput ? '' : undefined" :inert="showManualInput ? true : undefined"
:aria-hidden="showManualInput" :aria-hidden="showManualInput"
data-testid="json-import-file-panel" data-testid="json-import-file-panel"
@click="fileInputRef?.click()" @click="fileInputRef?.click()"
@@ -47,7 +47,7 @@
<div <div
class="space-y-1.5 transition-opacity duration-150" class="space-y-1.5 transition-opacity duration-150"
:class="showManualInput ? 'opacity-100' : 'opacity-0 pointer-events-none'" :class="showManualInput ? 'opacity-100' : 'opacity-0 pointer-events-none'"
:inert="showManualInput ? undefined : ''" :inert="showManualInput ? undefined : true"
:aria-hidden="!showManualInput" :aria-hidden="!showManualInput"
data-testid="json-import-manual-panel" data-testid="json-import-manual-panel"
> >
@@ -30,7 +30,7 @@ describe('JsonImportInput', () => {
expect(filePanel?.getAttribute('aria-hidden')).toBe('false') expect(filePanel?.getAttribute('aria-hidden')).toBe('false')
expect(filePanel?.hasAttribute('inert')).toBe(false) expect(filePanel?.hasAttribute('inert')).toBe(false)
expect(manualPanel?.getAttribute('aria-hidden')).toBe('true') expect(manualPanel?.getAttribute('aria-hidden')).toBe('true')
expect(manualPanel?.getAttribute('inert')).toBe('') expect(manualPanel?.hasAttribute('inert')).toBe(true)
root.querySelector<HTMLButtonElement>('[data-testid="json-import-mode-toggle"]')?.click() root.querySelector<HTMLButtonElement>('[data-testid="json-import-mode-toggle"]')?.click()
await nextTick() await nextTick()
@@ -38,7 +38,7 @@ describe('JsonImportInput', () => {
expect(root.querySelector('[data-testid="json-import-file-panel"]')).toBe(filePanel) expect(root.querySelector('[data-testid="json-import-file-panel"]')).toBe(filePanel)
expect(root.querySelector('[data-testid="json-import-manual-panel"]')).toBe(manualPanel) expect(root.querySelector('[data-testid="json-import-manual-panel"]')).toBe(manualPanel)
expect(filePanel?.getAttribute('aria-hidden')).toBe('true') expect(filePanel?.getAttribute('aria-hidden')).toBe('true')
expect(filePanel?.getAttribute('inert')).toBe('') expect(filePanel?.hasAttribute('inert')).toBe(true)
expect(manualPanel?.getAttribute('aria-hidden')).toBe('false') expect(manualPanel?.getAttribute('aria-hidden')).toBe('false')
expect(manualPanel?.hasAttribute('inert')).toBe(false) expect(manualPanel?.hasAttribute('inert')).toBe(false)
@@ -53,7 +53,7 @@ describe('JsonImportInput', () => {
await nextTick() await nextTick()
expect(value.value).toBe('') expect(value.value).toBe('')
expect(filePanel?.hasAttribute('inert')).toBe(false) expect(filePanel?.hasAttribute('inert')).toBe(false)
expect(manualPanel?.getAttribute('inert')).toBe('') expect(manualPanel?.hasAttribute('inert')).toBe(true)
app.unmount() app.unmount()
root.remove() root.remove()
@@ -91,7 +91,7 @@ const chartOptions = computed(() => ({
scales: { scales: {
y: { y: {
ticks: { ticks: {
callback: (value: number) => `${value}s` callback: (value: string | number) => `${value}s`
} }
} }
} }
+2 -1
View File
@@ -10,6 +10,7 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { computed } from 'vue' import { computed } from 'vue'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
@@ -17,7 +18,7 @@ interface Props {
variant?: 'default' | 'destructive' | 'outline' | 'secondary' | 'ghost' | 'link' variant?: 'default' | 'destructive' | 'outline' | 'secondary' | 'ghost' | 'link'
size?: 'default' | 'sm' | 'lg' | 'icon' size?: 'default' | 'sm' | 'lg' | 'icon'
disabled?: boolean disabled?: boolean
class?: string class?: ClassValue
type?: 'button' | 'submit' | 'reset' type?: 'button' | 'submit' | 'reset'
} }
+3 -2
View File
@@ -59,6 +59,7 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { computed, useAttrs, ref } from 'vue' import { computed, useAttrs, ref } from 'vue'
import { Eye, EyeOff } from 'lucide-vue-next' import { Eye, EyeOff } from 'lucide-vue-next'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
@@ -70,8 +71,8 @@ const emit = defineEmits<{
}>() }>()
interface Props { interface Props {
modelValue?: string | number modelValue?: string | number | null
class?: string class?: ClassValue
autocomplete?: string autocomplete?: string
/** /**
* 输入框尺寸 * 输入框尺寸
+2 -1
View File
@@ -5,11 +5,12 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { computed } from 'vue' import { computed } from 'vue'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
interface Props { interface Props {
class?: string class?: ClassValue
} }
const props = defineProps<Props>() const props = defineProps<Props>()
@@ -1,11 +1,12 @@
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { SelectTrigger as SelectTriggerPrimitive } from 'radix-vue' import { SelectTrigger as SelectTriggerPrimitive } from 'radix-vue'
import { ChevronDown } from 'lucide-vue-next' import { ChevronDown } from 'lucide-vue-next'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
import { computed } from 'vue' import { computed } from 'vue'
interface Props { interface Props {
class?: string class?: ClassValue
disabled?: boolean disabled?: boolean
} }
+2 -2
View File
@@ -4,7 +4,7 @@ import { SelectRoot as SelectRootPrimitive } from 'radix-vue'
interface Props { interface Props {
defaultValue?: string defaultValue?: string
modelValue?: string modelValue?: string | null
open?: boolean open?: boolean
defaultOpen?: boolean defaultOpen?: boolean
dir?: 'ltr' | 'rtl' dir?: 'ltr' | 'rtl'
@@ -35,7 +35,7 @@ const openProp = computed(() =>
// modelValue radix-vue uncontrolled // modelValue radix-vue uncontrolled
const modelValueProp = computed(() => const modelValueProp = computed(() =>
props.modelValue !== undefined ? { modelValue: props.modelValue } : {} props.modelValue !== undefined ? { modelValue: props.modelValue ?? '' } : {}
) )
</script> </script>
@@ -1,4 +1,5 @@
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { computed, nextTick, onBeforeUnmount, onMounted, ref, useAttrs, useSlots } from 'vue' import { computed, nextTick, onBeforeUnmount, onMounted, ref, useAttrs, useSlots } from 'vue'
import { ArrowDown, ArrowUp, ArrowUpDown, ListFilter } from 'lucide-vue-next' import { ArrowDown, ArrowUp, ArrowUpDown, ListFilter } from 'lucide-vue-next'
@@ -9,7 +10,7 @@ import TableHead from './table-head.vue'
type SortDirection = 'asc' | 'desc' type SortDirection = 'asc' | 'desc'
const props = withDefaults(defineProps<{ const props = withDefaults(defineProps<{
class?: string class?: ClassValue
columnKey?: string columnKey?: string
sortable?: boolean sortable?: boolean
activeKey?: string | null activeKey?: string | null
+2 -1
View File
@@ -1,9 +1,10 @@
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
import { computed } from 'vue' import { computed } from 'vue'
interface Props { interface Props {
class?: string class?: ClassValue
} }
const props = defineProps<Props>() const props = defineProps<Props>()
+2 -1
View File
@@ -1,9 +1,10 @@
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
import { computed } from 'vue' import { computed } from 'vue'
interface Props { interface Props {
class?: string class?: ClassValue
} }
const props = defineProps<Props>() const props = defineProps<Props>()
+2 -1
View File
@@ -1,9 +1,10 @@
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
import { computed } from 'vue' import { computed } from 'vue'
interface Props { interface Props {
class?: string class?: ClassValue
} }
const props = defineProps<Props>() const props = defineProps<Props>()
+3 -2
View File
@@ -13,12 +13,13 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import type { ClassValue } from 'clsx'
import { computed, ref, watch, onMounted, onUnmounted, nextTick, inject, type Ref } from 'vue' import { computed, ref, watch, onMounted, onUnmounted, nextTick, inject, type Ref } from 'vue'
import { cn } from '@/lib/utils' import { cn } from '@/lib/utils'
import { useI18n } from '@/i18n' import { useI18n } from '@/i18n'
interface Props { interface Props {
class?: string class?: ClassValue
} }
const props = defineProps<Props>() const props = defineProps<Props>()
@@ -41,7 +42,7 @@ const activeTab = inject<Ref<string>>('activeTab')
// grid // grid
const hasGridClass = computed(() => { const hasGridClass = computed(() => {
return props.class?.includes('grid') return cn(props.class).includes('grid')
}) })
const listClass = computed(() => { const listClass = computed(() => {
+1 -1
View File
@@ -64,7 +64,7 @@ export function useEscapeKey(
activeElement.tagName === 'INPUT' || activeElement.tagName === 'INPUT' ||
activeElement.tagName === 'TEXTAREA' || activeElement.tagName === 'TEXTAREA' ||
activeElement.tagName === 'SELECT' || activeElement.tagName === 'SELECT' ||
activeElement.contentEditable === 'true' || (activeElement instanceof HTMLElement && activeElement.isContentEditable) ||
activeElement.getAttribute('role') === 'textbox' || activeElement.getAttribute('role') === 'textbox' ||
activeElement.getAttribute('role') === 'combobox' activeElement.getAttribute('role') === 'combobox'
) )
+2 -2
View File
@@ -1,3 +1,4 @@
import type { TimeScatterChartData } from '@/components/charts/types'
/** /**
* TTL composable * TTL composable
* TTL * TTL
@@ -10,7 +11,6 @@ import {
type CacheHitAnalysisResponse, type CacheHitAnalysisResponse,
type IntervalTimelineResponse type IntervalTimelineResponse
} from '@/api/cache' } from '@/api/cache'
import type { ChartData } from 'chart.js'
import { log } from '@/utils/logger' import { log } from '@/utils/logger'
// 时间范围选项 // 时间范围选项
@@ -162,7 +162,7 @@ export function useTTLAnalysis() {
} }
// 用户时间线散点图数据 // 用户时间线散点图数据
const userTimelineChartData = computed<ChartData<'scatter'>>(() => { const userTimelineChartData = computed<TimeScatterChartData>(() => {
if (!userTimelineData.value || userTimelineData.value.points.length === 0) { if (!userTimelineData.value || userTimelineData.value.points.length === 0) {
return { datasets: [] } return { datasets: [] }
} }
@@ -163,6 +163,7 @@ async function execute(action: string): Promise<string> {
if (!turnstile || !container || !turnstile.execute) { if (!turnstile || !container || !turnstile.execute) {
throw new Error('Turnstile unavailable') throw new Error('Turnstile unavailable')
} }
const execute = turnstile.execute.bind(turnstile)
clearWidget() clearWidget()
@@ -191,7 +192,7 @@ async function execute(action: string): Promise<string> {
}, },
}) })
widgetId.value = id widgetId.value = id
turnstile.execute(id) execute(id)
}) })
} }
@@ -1189,7 +1189,7 @@ function setEmbeddingEnabled(enabled: boolean) {
setConfigField('embedding', undefined) setConfigField('embedding', undefined)
if (form.value.config?.model_type === 'embedding') setConfigField('model_type', undefined) if (form.value.config?.model_type === 'embedding') setConfigField('model_type', undefined)
if (Array.isArray(form.value.config?.api_formats) if (Array.isArray(form.value.config?.api_formats)
&& form.value.config.api_formats.every((format) => embeddingApiFormats.includes(String(format)))) { && form.value.config.api_formats.every((format) => embeddingApiFormats.some((supportedFormat) => supportedFormat === String(format)))) {
setConfigField('api_formats', undefined) setConfigField('api_formats', undefined)
} }
} }

Some files were not shown because too many files have changed in this diff Show More