mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-04 00:17:45 +08:00
fix: restore security hardening compatibility and validation
Restore authorized rule reveal, explicit full HTTP capture and retention, video task business fields, and valid payment URLs. Add opt-in credential preservation for trusted recovery, fix frontend type contracts and async races, and eliminate PostgreSQL test fixture resource leaks. Document audit coverage and successful fmt and CI-scoped Clippy checks.
This commit is contained in:
@@ -496,6 +496,7 @@ fn access_for_route(method: &http::Method, decision: &GatewayControlDecision) ->
|
||||
Some("admin:endpoints_manage"),
|
||||
Some(
|
||||
"reveal_key"
|
||||
| "reveal_endpoint_rules"
|
||||
| "export_key"
|
||||
| "create_provider_key"
|
||||
| "update_key"
|
||||
@@ -1490,6 +1491,12 @@ mod tests {
|
||||
fn plaintext_credential_reads_require_admin_permission() {
|
||||
let read_only_permissions = read_only_management_token_permissions();
|
||||
let cases = [
|
||||
(
|
||||
"admin:endpoints_manage",
|
||||
"reveal_endpoint_rules",
|
||||
None,
|
||||
"admin:endpoints_manage:admin",
|
||||
),
|
||||
(
|
||||
"admin:endpoints_manage",
|
||||
"reveal_key",
|
||||
|
||||
@@ -302,6 +302,19 @@ pub(super) fn classify_admin_endpoints_family_route(
|
||||
"admin:endpoints_manage",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& normalized_path
|
||||
.strip_prefix("/api/admin/endpoints/")
|
||||
.and_then(|path| path.strip_suffix("/rules/reveal"))
|
||||
.is_some_and(|endpoint_id| !endpoint_id.is_empty() && !endpoint_id.contains('/'))
|
||||
{
|
||||
Some(classified(
|
||||
"admin_proxy",
|
||||
"endpoints_manage",
|
||||
"reveal_endpoint_rules",
|
||||
"admin:endpoints_manage",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& normalized_path.starts_with("/api/admin/endpoints/")
|
||||
&& !normalized_path.starts_with("/api/admin/endpoints/health/")
|
||||
|
||||
@@ -381,6 +381,28 @@ fn classifies_admin_get_endpoint_as_admin_proxy_route() {
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_admin_reveal_endpoint_rules_as_admin_proxy_route() {
|
||||
let headers = headers(&[]);
|
||||
let uri: Uri = "/api/admin/endpoints/endpoint-1/rules/reveal"
|
||||
.parse()
|
||||
.expect("uri should parse");
|
||||
let decision =
|
||||
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
|
||||
|
||||
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
|
||||
assert_eq!(decision.route_family.as_deref(), Some("endpoints_manage"));
|
||||
assert_eq!(
|
||||
decision.route_kind.as_deref(),
|
||||
Some("reveal_endpoint_rules")
|
||||
);
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("admin:endpoints_manage")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_admin_create_endpoint_as_admin_proxy_route() {
|
||||
let headers = http::HeaderMap::new();
|
||||
|
||||
@@ -41,16 +41,9 @@ fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestR
|
||||
return UsageRequestRecordLevel::Basic;
|
||||
};
|
||||
|
||||
if value.eq_ignore_ascii_case("basic")
|
||||
|| value.eq_ignore_ascii_case("base")
|
||||
|| value.eq_ignore_ascii_case("headers")
|
||||
|| value.eq_ignore_ascii_case("minimal")
|
||||
|| value.eq_ignore_ascii_case("none")
|
||||
{
|
||||
UsageRequestRecordLevel::Basic
|
||||
if value.eq_ignore_ascii_case("full") {
|
||||
UsageRequestRecordLevel::Full
|
||||
} else {
|
||||
// Raw HTTP payload capture is disabled at the runtime boundary. The setting remains
|
||||
// accepted for compatibility, but no longer authorizes collecting request/response data.
|
||||
UsageRequestRecordLevel::Basic
|
||||
}
|
||||
}
|
||||
@@ -501,7 +494,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_disables_full_http_capture() {
|
||||
async fn usage_runtime_access_honors_explicit_full_http_capture() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"request_record_level".to_string(),
|
||||
json!("full"),
|
||||
@@ -511,7 +504,32 @@ mod tests {
|
||||
.await
|
||||
.expect("request record level should read");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
assert_eq!(level, UsageRequestRecordLevel::Full);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_honors_legacy_full_without_overriding_current_config() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"request_log_level".to_string(),
|
||||
json!(" FULL "),
|
||||
)]);
|
||||
assert_eq!(
|
||||
UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.unwrap(),
|
||||
UsageRequestRecordLevel::Full
|
||||
);
|
||||
|
||||
let state = state.with_system_config_values_for_tests([
|
||||
("request_log_level".to_string(), json!("full")),
|
||||
("request_record_level".to_string(), json!("basic")),
|
||||
]);
|
||||
assert_eq!(
|
||||
UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.unwrap(),
|
||||
UsageRequestRecordLevel::Basic
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -426,11 +426,8 @@ mod tests {
|
||||
assert!(candidate.finished_at_unix_ms.is_some());
|
||||
}
|
||||
|
||||
/// The guard holds no request body, and the persistence boundary intentionally
|
||||
/// rejects request/response capture material. A dropped-attempt settlement
|
||||
/// must not re-introduce an inline body or a caller-controlled body reference.
|
||||
#[tokio::test]
|
||||
async fn settling_a_dropped_attempt_does_not_reintroduce_request_body_capture() {
|
||||
async fn settling_a_dropped_attempt_respects_disabled_request_body_capture() {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = test_state(&usage_repository, &request_candidate_repository);
|
||||
@@ -444,8 +441,6 @@ mod tests {
|
||||
candidate_started_unix_ms,
|
||||
)
|
||||
.await;
|
||||
// This deliberately supplies capture material to prove that the usage
|
||||
// persistence boundary strips it before either lifecycle write stores it.
|
||||
let captured_body = json!({"stream": true, "service_tier": "priority"});
|
||||
let mut capture = build_pending_usage_record(
|
||||
&plan,
|
||||
@@ -481,7 +476,10 @@ mod tests {
|
||||
.expect("cancelled usage should be recorded");
|
||||
assert_eq!(usage.provider_request_body, None);
|
||||
assert_eq!(usage.provider_request_body_ref, None);
|
||||
assert_eq!(usage.provider_request_body_state, None);
|
||||
assert_eq!(
|
||||
usage.provider_request_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -14661,15 +14661,19 @@ mod tests {
|
||||
assert_eq!(usage.status_code, Some(302));
|
||||
assert_eq!(usage.error_category.as_deref(), Some("redirect"));
|
||||
assert!(usage.error_message.is_none());
|
||||
// HTTP capture is intentionally disabled at the persistence boundary. Keep the
|
||||
// protocol facts above, but do not turn provider/client headers into an audit store.
|
||||
assert!(usage.client_response_headers.is_none());
|
||||
assert!(usage.response_headers.is_none());
|
||||
assert_eq!(
|
||||
usage.client_response_headers.as_ref().unwrap()["content-type"],
|
||||
json!("application/json")
|
||||
);
|
||||
assert_eq!(
|
||||
usage.response_headers.as_ref().unwrap()["content-type"],
|
||||
json!("text/html")
|
||||
);
|
||||
assert!(
|
||||
usage.response_body.is_none(),
|
||||
"upstream redirect did not include a body"
|
||||
);
|
||||
assert!(usage.client_response_body.is_none());
|
||||
assert_eq!(usage.client_response_body.as_ref(), Some(&body_json));
|
||||
let candidates = request_candidate_repository
|
||||
.list_by_request_id("req-remote-runtime-stream-redirect")
|
||||
.await
|
||||
|
||||
@@ -6,6 +6,7 @@ mod extractors;
|
||||
mod list;
|
||||
pub(crate) mod payloads;
|
||||
mod reads;
|
||||
mod reveal;
|
||||
mod support;
|
||||
mod update;
|
||||
|
||||
@@ -41,6 +42,10 @@ pub(crate) async fn maybe_build_local_admin_endpoints_routes_response(
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) = reveal::maybe_handle(state, request_context).await? {
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) = defaults::maybe_handle(state, request_context, request_body).await? {
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
use super::extractors::admin_endpoint_id;
|
||||
use super::support::build_admin_endpoints_data_unavailable_response;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::handlers::admin::shared::{
|
||||
attach_admin_audit_response, mark_sensitive_admin_response_no_store,
|
||||
};
|
||||
use crate::GatewayError;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn maybe_handle(
|
||||
state: &AdminAppState<'_>,
|
||||
request_context: &AdminRequestContext<'_>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let Some(decision) = request_context.decision() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if decision.route_family.as_deref() != Some("endpoints_manage")
|
||||
|| decision.route_kind.as_deref() != Some("reveal_endpoint_rules")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return Ok(Some(build_admin_endpoints_data_unavailable_response()));
|
||||
}
|
||||
let Some(endpoint_id) = request_context
|
||||
.path()
|
||||
.strip_suffix("/rules/reveal")
|
||||
.and_then(admin_endpoint_id)
|
||||
else {
|
||||
return Ok(Some(
|
||||
(
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": "Endpoint 不存在" })),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
};
|
||||
let Some(endpoint) = state
|
||||
.read_provider_catalog_endpoints_by_ids(std::slice::from_ref(&endpoint_id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
else {
|
||||
return Ok(Some(
|
||||
(
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": "Endpoint 不存在" })),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
};
|
||||
let payload = json!({
|
||||
"header_rules": endpoint.header_rules.as_ref().and_then(|value| value.as_array()).cloned().unwrap_or_default(),
|
||||
"body_rules": endpoint.body_rules.as_ref().and_then(|value| value.as_array()).cloned().unwrap_or_default(),
|
||||
"response_header_rules": endpoint.config.as_ref().and_then(|config| config.get("response_header_rules")).and_then(|value| value.as_array()).cloned().unwrap_or_default(),
|
||||
});
|
||||
Ok(Some(mark_sensitive_admin_response_no_store(
|
||||
attach_admin_audit_response(
|
||||
Json(payload).into_response(),
|
||||
"admin_endpoint_rules_revealed",
|
||||
"reveal_endpoint_rules",
|
||||
"provider_endpoint",
|
||||
&endpoint_id,
|
||||
),
|
||||
)))
|
||||
}
|
||||
@@ -117,8 +117,8 @@ where
|
||||
|
||||
use aether_crypto::warm_python_fernet_secret;
|
||||
use aether_data::lifecycle::export::{
|
||||
copy_database_records, export_database_jsonl, import_database_jsonl, DataCopyOptions,
|
||||
ExportDomain, MAX_JSONL_INPUT_BYTES,
|
||||
copy_database_records, export_database_jsonl, import_database_jsonl_with_options,
|
||||
DataCopyOptions, DataImportOptions, ExportDomain, MAX_JSONL_INPUT_BYTES,
|
||||
};
|
||||
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
|
||||
use aether_gateway::{
|
||||
@@ -1351,6 +1351,11 @@ struct DataExportArgs {
|
||||
struct DataImportArgs {
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(
|
||||
long,
|
||||
help = "Preserve passwords and API/management credentials from a trusted import; imported sessions remain revoked. Without this flag identity credentials are revoked."
|
||||
)]
|
||||
preserve_credentials: bool,
|
||||
}
|
||||
|
||||
#[derive(ClapArgs, Debug, Clone)]
|
||||
@@ -1382,6 +1387,11 @@ struct DataCopyArgs {
|
||||
|
||||
#[arg(long)]
|
||||
omit_request_body_details: bool,
|
||||
#[arg(
|
||||
long,
|
||||
help = "Preserve passwords and API/management credentials from the trusted source; imported sessions remain revoked. The target must use the source encryption key."
|
||||
)]
|
||||
preserve_credentials: bool,
|
||||
}
|
||||
|
||||
impl GatewayLoggingArgs {
|
||||
@@ -2905,12 +2915,23 @@ async fn run_data_import(
|
||||
let driver = database.driver;
|
||||
let input_path = args.input.clone();
|
||||
let input = tokio::task::spawn_blocking(move || read_data_import_input(&input_path)).await??;
|
||||
let imported = import_database_jsonl(database, &input).await?;
|
||||
if !args.preserve_credentials {
|
||||
warn!("identity credentials will be revoked; use --preserve-credentials only for trusted recovery or migration");
|
||||
}
|
||||
let imported = import_database_jsonl_with_options(
|
||||
database,
|
||||
&input,
|
||||
DataImportOptions {
|
||||
preserve_credentials: args.preserve_credentials,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
driver = %driver,
|
||||
input = %args.input.display(),
|
||||
imported,
|
||||
preserve_credentials = args.preserve_credentials,
|
||||
"database import complete"
|
||||
);
|
||||
println!(
|
||||
@@ -3171,6 +3192,9 @@ async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Er
|
||||
let target_driver = target.driver;
|
||||
let domains = requested_domains(&args.domains);
|
||||
let created_at_unix_secs = current_unix_secs()?;
|
||||
if !args.preserve_credentials {
|
||||
warn!("identity credentials will be revoked; use --preserve-credentials only for trusted recovery or migration");
|
||||
}
|
||||
let imported = copy_database_records(
|
||||
source,
|
||||
target,
|
||||
@@ -3178,6 +3202,7 @@ async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Er
|
||||
created_at_unix_secs,
|
||||
DataCopyOptions {
|
||||
omit_request_body_details: args.omit_request_body_details,
|
||||
preserve_credentials: args.preserve_credentials,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
@@ -3186,6 +3211,7 @@ async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Er
|
||||
source_driver = %source_driver,
|
||||
target_driver = %target_driver,
|
||||
imported,
|
||||
preserve_credentials = args.preserve_credentials,
|
||||
"database copy complete"
|
||||
);
|
||||
println!(
|
||||
@@ -4357,6 +4383,41 @@ mod tests {
|
||||
};
|
||||
assert!(copy.source_allow_insecure);
|
||||
assert!(!copy.target_allow_insecure);
|
||||
assert!(!copy.preserve_credentials);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_import_and_copy_require_explicit_credential_preservation() {
|
||||
for preserve in [false, true] {
|
||||
let mut import_args = vec!["aether-gateway", "import", "--input", "trusted.jsonl"];
|
||||
let mut copy_args = vec![
|
||||
"aether-gateway",
|
||||
"copy",
|
||||
"--source-driver",
|
||||
"postgres",
|
||||
"--source-url",
|
||||
"postgres://localhost/source",
|
||||
"--target-driver",
|
||||
"postgres",
|
||||
"--target-url",
|
||||
"postgres://localhost/target",
|
||||
];
|
||||
if preserve {
|
||||
import_args.push("--preserve-credentials");
|
||||
copy_args.push("--preserve-credentials");
|
||||
}
|
||||
let Some(DataCommand::Import(import)) =
|
||||
Args::try_parse_from(import_args).unwrap().command
|
||||
else {
|
||||
panic!("expected import command");
|
||||
};
|
||||
let Some(DataCommand::Copy(copy)) = Args::try_parse_from(copy_args).unwrap().command
|
||||
else {
|
||||
panic!("expected copy command");
|
||||
};
|
||||
assert_eq!(import.preserve_credentials, preserve);
|
||||
assert_eq!(copy.preserve_credentials, preserve);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
|
||||
@@ -115,9 +115,6 @@ pub(super) fn usage_cleanup_window(
|
||||
usage_cleanup_window_with_override(now_utc, settings, None)
|
||||
}
|
||||
|
||||
/// Clamp is non-aggressive: each tier's cutoff becomes `max(policy_cutoff, now - override)`.
|
||||
/// A later cutoff = fewer records deleted, so the override can only make cleanup more
|
||||
/// conservative than the configured retention, never more destructive.
|
||||
pub(super) fn usage_cleanup_window_with_override(
|
||||
now_utc: DateTime<Utc>,
|
||||
settings: UsageCleanupSettings,
|
||||
@@ -135,9 +132,9 @@ pub(super) fn usage_cleanup_window_with_override(
|
||||
};
|
||||
let manual_cutoff = now_utc - override_duration;
|
||||
UsageCleanupWindow {
|
||||
detail_cutoff: policy.detail_cutoff.max(manual_cutoff),
|
||||
compressed_cutoff: policy.compressed_cutoff.max(manual_cutoff),
|
||||
header_cutoff: policy.header_cutoff.max(manual_cutoff),
|
||||
log_cutoff: policy.log_cutoff.max(manual_cutoff),
|
||||
detail_cutoff: policy.detail_cutoff.min(manual_cutoff),
|
||||
compressed_cutoff: policy.compressed_cutoff.min(manual_cutoff),
|
||||
header_cutoff: policy.header_cutoff.min(manual_cutoff),
|
||||
log_cutoff: policy.log_cutoff.min(manual_cutoff),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1140,19 +1140,40 @@ fn usage_cleanup_window_with_override_is_always_non_aggressive() {
|
||||
let override_duration = chrono::Duration::days(180);
|
||||
let clamped = usage_cleanup_window_with_override(now_utc, settings, Some(override_duration));
|
||||
|
||||
assert_eq!(clamped.detail_cutoff, policy.detail_cutoff);
|
||||
assert_eq!(clamped.compressed_cutoff, policy.compressed_cutoff);
|
||||
assert_eq!(clamped.header_cutoff, policy.header_cutoff);
|
||||
assert_eq!(clamped.log_cutoff, now_utc - override_duration);
|
||||
assert!(clamped.log_cutoff > policy.log_cutoff);
|
||||
assert_eq!(clamped.detail_cutoff, now_utc - override_duration);
|
||||
assert_eq!(clamped.compressed_cutoff, now_utc - override_duration);
|
||||
assert_eq!(clamped.header_cutoff, now_utc - override_duration);
|
||||
assert_eq!(clamped.log_cutoff, policy.log_cutoff);
|
||||
assert!(clamped.log_cutoff <= policy.log_cutoff);
|
||||
|
||||
let far_override = chrono::Duration::days(5);
|
||||
let far = usage_cleanup_window_with_override(now_utc, settings, Some(far_override));
|
||||
assert_eq!(far.detail_cutoff, now_utc - far_override);
|
||||
assert_eq!(far.compressed_cutoff, now_utc - far_override);
|
||||
assert_eq!(far.header_cutoff, now_utc - far_override);
|
||||
assert_eq!(far.log_cutoff, now_utc - far_override);
|
||||
assert!(far.log_cutoff > policy.log_cutoff);
|
||||
assert_eq!(far, policy);
|
||||
|
||||
for days in [0, 5, 30, 180, 400] {
|
||||
let cutoff = now_utc - chrono::Duration::days(days);
|
||||
let window = usage_cleanup_window_with_override(
|
||||
now_utc,
|
||||
settings,
|
||||
Some(chrono::Duration::days(days)),
|
||||
);
|
||||
for (actual, configured) in [
|
||||
(window.detail_cutoff, policy.detail_cutoff),
|
||||
(window.compressed_cutoff, policy.compressed_cutoff),
|
||||
(window.header_cutoff, policy.header_cutoff),
|
||||
(window.log_cutoff, policy.log_cutoff),
|
||||
] {
|
||||
assert!(actual <= configured);
|
||||
assert!(actual <= cutoff);
|
||||
for age in [1, 7, 15, 30, 90, 180, 365, 401] {
|
||||
let created_at = now_utc - chrono::Duration::days(age);
|
||||
if created_at < actual {
|
||||
assert!(created_at < configured);
|
||||
assert!(created_at < cutoff);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let passthrough = usage_cleanup_window_with_override(now_utc, settings, None);
|
||||
assert_eq!(passthrough, policy);
|
||||
|
||||
@@ -326,7 +326,7 @@ async fn gateway_reads_video_task_detail_via_internal_async_task_endpoint() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endpoint() {
|
||||
async fn gateway_redirects_persisted_openai_video_url_from_authenticated_internal_endpoint() {
|
||||
let repository = Arc::new(InMemoryVideoTaskRepository::default());
|
||||
let mut task = sample_video_task(
|
||||
"task-redirect",
|
||||
@@ -341,7 +341,10 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endp
|
||||
.upsert(task)
|
||||
.await
|
||||
.expect("upsert should succeed");
|
||||
assert_eq!(stored.video_url, None);
|
||||
assert_eq!(
|
||||
stored.video_url.as_deref(),
|
||||
Some("https://8.8.8.8/video-task-redirect.mp4")
|
||||
);
|
||||
|
||||
let state = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
@@ -349,7 +352,10 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endp
|
||||
let (gateway_url, gateway_handle, access_token) =
|
||||
start_authenticated_operational_server(state).await;
|
||||
|
||||
let client = authenticated_operational_client(&access_token);
|
||||
let client = super::authenticated_operational_client_with_builder(
|
||||
reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()),
|
||||
&access_token,
|
||||
);
|
||||
let response = client
|
||||
.get(format!(
|
||||
"{gateway_url}/_gateway/async-tasks/video-tasks/task-redirect/video"
|
||||
@@ -358,7 +364,14 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_from_internal_endp
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("location")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
stored.video_url.as_deref()
|
||||
);
|
||||
|
||||
gateway_handle.abort();
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod keys;
|
||||
mod quota;
|
||||
mod routes;
|
||||
mod rules_reveal;
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use axum::body::Body;
|
||||
use http::{HeaderMap, HeaderValue, Method, Request, StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::super::super::{build_router_with_state, sample_endpoint, sample_provider, AppState};
|
||||
use crate::admin_api::{maybe_build_local_admin_response, AdminRouteRequest};
|
||||
use crate::audit::AdminAuditEvent;
|
||||
use crate::constants::{
|
||||
GATEWAY_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER, TRUSTED_ADMIN_USER_ID_HEADER,
|
||||
TRUSTED_ADMIN_USER_ROLE_HEADER,
|
||||
};
|
||||
use crate::control::resolve_public_request_context;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::tests::send_request;
|
||||
|
||||
fn seeded_state() -> AppState {
|
||||
let mut endpoint = sample_endpoint(
|
||||
"endpoint-rules",
|
||||
"provider-rules",
|
||||
"openai:chat",
|
||||
"https://example.test",
|
||||
);
|
||||
endpoint.header_rules =
|
||||
Some(json!([{"action": "set", "key": "x-auth", "value": "request-secret"}]));
|
||||
endpoint.body_rules =
|
||||
Some(json!([{"action": "set", "path": "auth.token", "value": "body-secret"}]));
|
||||
endpoint.config = Some(json!({
|
||||
"private_token": "unrelated-secret",
|
||||
"response_header_rules": [{"action": "set", "key": "x-auth", "value": "response-secret"}]
|
||||
}));
|
||||
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider("provider-rules", "custom", 10)],
|
||||
vec![endpoint],
|
||||
vec![],
|
||||
));
|
||||
AppState::new().unwrap().with_data_state_for_tests(
|
||||
GatewayDataState::with_provider_catalog_reader_for_tests(repository),
|
||||
)
|
||||
}
|
||||
|
||||
fn admin_headers() -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in [
|
||||
(GATEWAY_HEADER, "rust-phase3b"),
|
||||
(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user"),
|
||||
(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin"),
|
||||
(TRUSTED_ADMIN_SESSION_ID_HEADER, "admin-session"),
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn endpoint_rules_reveal_is_scoped_audited_and_not_cached() {
|
||||
let state = seeded_state();
|
||||
let context = resolve_public_request_context(
|
||||
&state,
|
||||
&Method::GET,
|
||||
&"/api/admin/endpoints/endpoint-rules/rules/reveal"
|
||||
.parse()
|
||||
.unwrap(),
|
||||
&admin_headers(),
|
||||
"reveal-test",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let response = maybe_build_local_admin_response(AdminRouteRequest::new(
|
||||
&state,
|
||||
&context,
|
||||
&"127.0.0.1:12345".parse().unwrap(),
|
||||
&admin_headers(),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()[http::header::CACHE_CONTROL], "no-store");
|
||||
assert_eq!(response.headers()[http::header::PRAGMA], "no-cache");
|
||||
let audit = response.extensions().get::<AdminAuditEvent>().unwrap();
|
||||
assert_eq!(audit.event_name, "admin_endpoint_rules_revealed");
|
||||
assert_eq!(audit.action, "reveal_endpoint_rules");
|
||||
assert_eq!(audit.target_id, "endpoint-rules");
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let payload: Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(payload["header_rules"][0]["value"], "request-secret");
|
||||
assert_eq!(payload["body_rules"][0]["value"], "body-secret");
|
||||
assert_eq!(
|
||||
payload["response_header_rules"][0]["value"],
|
||||
"response-secret"
|
||||
);
|
||||
assert_eq!(payload.as_object().unwrap().len(), 3);
|
||||
assert!(!payload.to_string().contains("unrelated-secret"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn endpoint_rules_reveal_denies_anonymous_and_non_admin_requests() {
|
||||
let router = build_router_with_state(seeded_state());
|
||||
for role in [None, Some("user")] {
|
||||
let mut request =
|
||||
Request::builder().uri("/api/admin/endpoints/endpoint-rules/rules/reveal");
|
||||
if let Some(role) = role {
|
||||
request = request
|
||||
.header(GATEWAY_HEADER, "rust-phase3b")
|
||||
.header(TRUSTED_ADMIN_USER_ID_HEADER, "normal-user")
|
||||
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, role)
|
||||
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "user-session");
|
||||
}
|
||||
let response = send_request(router.clone(), request.body(Body::empty()).unwrap()).await;
|
||||
assert!(matches!(
|
||||
response.status(),
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
|
||||
));
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert!(!String::from_utf8_lossy(&body).contains("request-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn endpoint_rules_reveal_returns_not_found_and_data_unavailable_without_fallback() {
|
||||
for (state, expected) in [
|
||||
(seeded_state(), StatusCode::NOT_FOUND),
|
||||
(AppState::new().unwrap(), StatusCode::SERVICE_UNAVAILABLE),
|
||||
] {
|
||||
let context = resolve_public_request_context(
|
||||
&state,
|
||||
&Method::GET,
|
||||
&"/api/admin/endpoints/missing/rules/reveal".parse().unwrap(),
|
||||
&admin_headers(),
|
||||
"reveal-missing-test",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let response = maybe_build_local_admin_response(AdminRouteRequest::new(
|
||||
&state,
|
||||
&context,
|
||||
&"127.0.0.1:12345".parse().unwrap(),
|
||||
&admin_headers(),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), expected);
|
||||
}
|
||||
}
|
||||
@@ -221,13 +221,17 @@ async fn gateway_handles_admin_video_tasks_list_locally_with_trusted_admin_princ
|
||||
assert_eq!(payload["pages"], json!(1));
|
||||
assert_eq!(payload["items"].as_array().map(Vec::len), Some(1));
|
||||
assert_eq!(payload["items"][0]["id"], "task-completed");
|
||||
// Video-task persistence intentionally drops user-facing PII. The admin
|
||||
// projection must therefore use the privacy-safe fallback when no separate
|
||||
// user snapshot is joined.
|
||||
assert_eq!(payload["items"][0]["username"], "Unknown");
|
||||
assert_eq!(payload["items"][0]["username"], "alice");
|
||||
assert_eq!(payload["items"][0]["provider_name"], "OpenAI");
|
||||
assert_eq!(payload["items"][0]["status"], "completed");
|
||||
assert!(payload["items"][0]["prompt"].is_null());
|
||||
assert_eq!(
|
||||
payload["items"][0]["prompt"],
|
||||
format!("{}...", "x".repeat(100))
|
||||
);
|
||||
assert_eq!(
|
||||
payload["items"][0]["video_url"],
|
||||
"https://8.8.8.8/task-completed.mp4"
|
||||
);
|
||||
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
|
||||
|
||||
gateway_handle.abort();
|
||||
@@ -393,7 +397,9 @@ async fn gateway_handles_admin_video_task_detail_locally_with_trusted_admin_prin
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await.expect("json body should parse");
|
||||
assert_eq!(payload["id"], "task-detail");
|
||||
assert_eq!(payload["username"], "Unknown");
|
||||
assert_eq!(payload["prompt"], "detail prompt");
|
||||
assert_eq!(payload["video_url"], "https://8.8.8.8/task-detail.mp4");
|
||||
assert_eq!(payload["username"], "charlie");
|
||||
assert_eq!(payload["provider_name"], "OpenAI");
|
||||
assert_eq!(payload["endpoint"]["id"], "endpoint-1");
|
||||
assert_eq!(payload["endpoint"]["api_format"], "openai:video");
|
||||
@@ -734,7 +740,7 @@ async fn local_admin_video_task_cancel_attaches_explicit_audit() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstream() {
|
||||
async fn gateway_redirects_persisted_openai_video_url_without_forwarding_admin_request() {
|
||||
let upstream_hits = Arc::new(Mutex::new(0usize));
|
||||
let upstream_hits_clone = Arc::clone(&upstream_hits);
|
||||
let upstream = Router::new().route(
|
||||
@@ -762,7 +768,10 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstrea
|
||||
))
|
||||
.await
|
||||
.expect("task should upsert");
|
||||
assert_eq!(stored.video_url, None);
|
||||
assert_eq!(
|
||||
stored.video_url.as_deref(),
|
||||
Some("https://8.8.8.8/task-redirect.mp4")
|
||||
);
|
||||
|
||||
let (_upstream_url, upstream_handle) = start_server(upstream).await;
|
||||
let gateway = build_router_with_state(
|
||||
@@ -788,7 +797,14 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstrea
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(http::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
stored.video_url.as_deref()
|
||||
);
|
||||
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
|
||||
|
||||
gateway_handle.abort();
|
||||
@@ -796,22 +812,22 @@ async fn gateway_does_not_redirect_sanitized_openai_video_url_or_forward_upstrea
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_admin_video_task_video_is_unavailable_after_openai_url_sanitization() {
|
||||
async fn local_admin_video_task_download_preserves_signed_url_and_attaches_audit() {
|
||||
let repository = Arc::new(InMemoryVideoTaskRepository::default());
|
||||
let stored = repository
|
||||
.upsert(sample_admin_video_task(
|
||||
"task-video-audit",
|
||||
VideoTaskStatus::Completed,
|
||||
1_710_000_550,
|
||||
"user-5",
|
||||
"frank",
|
||||
"provider-openai",
|
||||
"gpt-video",
|
||||
"video audit prompt",
|
||||
))
|
||||
.await
|
||||
.expect("task should upsert");
|
||||
assert_eq!(stored.video_url, None);
|
||||
let mut task = sample_admin_video_task(
|
||||
"task-video-audit",
|
||||
VideoTaskStatus::Completed,
|
||||
1_710_000_550,
|
||||
"user-5",
|
||||
"frank",
|
||||
"provider-openai",
|
||||
"gpt-video",
|
||||
"video audit prompt",
|
||||
);
|
||||
task.video_url =
|
||||
Some("https://8.8.8.8/video.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1".to_string());
|
||||
let stored = repository.upsert(task).await.expect("task should upsert");
|
||||
assert_eq!(stored.prompt.as_deref(), Some("video audit prompt"));
|
||||
|
||||
let state = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
@@ -825,8 +841,15 @@ async fn local_admin_video_task_video_is_unavailable_after_openai_url_sanitizati
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert!(response.extensions().get::<AdminAuditEvent>().is_none());
|
||||
assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(http::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
stored.video_url.as_deref()
|
||||
);
|
||||
assert!(response.extensions().get::<AdminAuditEvent>().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -12,6 +12,7 @@ use super::{
|
||||
UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
|
||||
};
|
||||
use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER;
|
||||
use aether_data_contracts::repository::usage::UsageBodyCaptureState;
|
||||
|
||||
fn deep_nested_metadata(levels: usize) -> serde_json::Value {
|
||||
let mut current = json!({"leaf": "value"});
|
||||
@@ -84,6 +85,58 @@ where
|
||||
stored.expect("usage should be present once the expected status is observed")
|
||||
}
|
||||
|
||||
async fn load_admin_usage_capture_detail(
|
||||
state: &crate::AppState,
|
||||
usage_id: &str,
|
||||
include_bodies: bool,
|
||||
) -> serde_json::Value {
|
||||
use crate::admin_api::{maybe_build_local_admin_response, AdminRouteRequest};
|
||||
use crate::constants::{
|
||||
GATEWAY_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER, TRUSTED_ADMIN_USER_ID_HEADER,
|
||||
TRUSTED_ADMIN_USER_ROLE_HEADER,
|
||||
};
|
||||
use crate::control::resolve_public_request_context;
|
||||
use http_body_util::BodyExt;
|
||||
|
||||
let mut headers = http::HeaderMap::new();
|
||||
for (name, value) in [
|
||||
(GATEWAY_HEADER, "rust-phase3b"),
|
||||
(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user"),
|
||||
(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin"),
|
||||
(TRUSTED_ADMIN_SESSION_ID_HEADER, "admin-session"),
|
||||
] {
|
||||
headers.insert(name, HeaderValue::from_static(value));
|
||||
}
|
||||
let uri = format!("/api/admin/usage/{usage_id}?include_bodies={include_bodies}")
|
||||
.parse()
|
||||
.unwrap();
|
||||
let context = resolve_public_request_context(
|
||||
state,
|
||||
&http::Method::GET,
|
||||
&uri,
|
||||
&headers,
|
||||
"usage-full-detail",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let response = maybe_build_local_admin_response(AdminRouteRequest::new(
|
||||
state,
|
||||
&context,
|
||||
&"127.0.0.1:12345".parse().unwrap(),
|
||||
&headers,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(response
|
||||
.extensions()
|
||||
.get::<crate::audit::AdminAuditEvent>()
|
||||
.is_some());
|
||||
serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_handles_local_openai_chat_sync_report_with_local_reporting_when_usage_runtime_enabled() {
|
||||
run_async_test_on_large_stack(
|
||||
@@ -348,7 +401,7 @@ async fn gateway_truncates_deep_request_echo_for_local_openai_chat_sync_usage_im
|
||||
Arc::clone(&request_candidate_repository),
|
||||
Arc::clone(&usage_repository),
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
),
|
||||
).with_system_config_values_for_tests([("request_record_level".to_string(), json!("full"))]),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
@@ -402,10 +455,28 @@ async fn gateway_truncates_deep_request_echo_for_local_openai_chat_sync_usage_im
|
||||
let stored_usage = stored_usage.expect("usage should be recorded");
|
||||
assert_eq!(stored_usage.status, "completed");
|
||||
assert_eq!(stored_usage.total_tokens, 5);
|
||||
assert!(stored_usage.request_body.is_none());
|
||||
let request_body = stored_usage.request_body.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
request_body["messages"][0]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.len(),
|
||||
128 * 1024
|
||||
);
|
||||
assert!(
|
||||
request_body["metadata"]["child"]["child"]["child"]["child"]["child"]
|
||||
.get("depth")
|
||||
.is_some()
|
||||
);
|
||||
assert!(stored_usage.request_body_ref.is_none());
|
||||
assert!(stored_usage.request_body_state.is_none());
|
||||
assert!(stored_usage.request_headers.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.request_body_state,
|
||||
Some(UsageBodyCaptureState::Inline)
|
||||
);
|
||||
assert_eq!(
|
||||
stored_usage.request_headers.as_ref().unwrap()["authorization"],
|
||||
"[redacted]"
|
||||
);
|
||||
|
||||
gateway_handle.abort();
|
||||
execution_runtime_handle.abort();
|
||||
@@ -489,10 +560,10 @@ async fn gateway_ignores_legacy_max_request_body_size_for_local_openai_chat_sync
|
||||
Arc::clone(&usage_repository),
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests([(
|
||||
"max_request_body_size".to_string(),
|
||||
json!(128),
|
||||
)]),
|
||||
.with_system_config_values_for_tests([
|
||||
("max_request_body_size".to_string(), json!(128)),
|
||||
("request_record_level".to_string(), json!("full")),
|
||||
]),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
@@ -535,12 +606,30 @@ async fn gateway_ignores_legacy_max_request_body_size_for_local_openai_chat_sync
|
||||
)
|
||||
.await;
|
||||
assert_eq!(stored_usage.total_tokens, 5);
|
||||
assert!(stored_usage.request_body.is_none());
|
||||
assert!(
|
||||
stored_usage.request_body.as_ref().unwrap()["messages"][0]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.len()
|
||||
> 128
|
||||
);
|
||||
assert!(stored_usage.request_body_ref.is_none());
|
||||
assert!(stored_usage.request_body_state.is_none());
|
||||
assert!(stored_usage.provider_request_body.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.request_body_state,
|
||||
Some(UsageBodyCaptureState::Inline)
|
||||
);
|
||||
assert!(
|
||||
stored_usage.provider_request_body.as_ref().unwrap()["messages"][0]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.len()
|
||||
> 128
|
||||
);
|
||||
assert!(stored_usage.provider_request_body_ref.is_none());
|
||||
assert!(stored_usage.provider_request_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.provider_request_body_state,
|
||||
Some(UsageBodyCaptureState::Inline)
|
||||
);
|
||||
|
||||
gateway_handle.abort();
|
||||
execution_runtime_handle.abort();
|
||||
@@ -551,11 +640,19 @@ async fn gateway_ignores_legacy_max_request_body_size_for_local_openai_chat_sync
|
||||
fn gateway_strips_request_and_response_bodies_when_request_record_level_is_base() {
|
||||
run_async_test_on_large_stack(
|
||||
"gateway_strips_request_and_response_bodies_when_request_record_level_is_base",
|
||||
gateway_strips_request_and_response_bodies_when_request_record_level_is_base_impl(),
|
||||
gateway_honors_request_record_level_impl("base"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn gateway_strips_request_and_response_bodies_when_request_record_level_is_base_impl() {
|
||||
#[test]
|
||||
fn gateway_full_request_record_level_preserves_sync_bodies_in_admin_detail() {
|
||||
run_async_test_on_large_stack(
|
||||
"gateway_full_request_record_level_preserves_sync_bodies_in_admin_detail",
|
||||
gateway_honors_request_record_level_impl("full"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn gateway_honors_request_record_level_impl(record_level: &str) {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
|
||||
@@ -630,14 +727,14 @@ async fn gateway_strips_request_and_response_bodies_when_request_record_level_is
|
||||
)
|
||||
.with_system_config_values_for_tests([(
|
||||
"request_record_level".to_string(),
|
||||
json!("base"),
|
||||
json!(record_level),
|
||||
)]),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
..UsageRuntimeConfig::default()
|
||||
});
|
||||
let gateway = build_router_with_state(gateway_state);
|
||||
let gateway = build_router_with_state(gateway_state.clone());
|
||||
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
@@ -678,14 +775,39 @@ async fn gateway_strips_request_and_response_bodies_when_request_record_level_is
|
||||
assert_eq!(stored_usage.status, "completed");
|
||||
assert_eq!(stored_usage.total_tokens, 5);
|
||||
assert_eq!(stored_usage.response_time_ms, Some(25));
|
||||
assert!(stored_usage.request_body.is_none());
|
||||
assert!(stored_usage.request_body_ref.is_none());
|
||||
assert!(stored_usage.provider_request_body.is_none());
|
||||
assert!(stored_usage.provider_request_body_ref.is_none());
|
||||
assert!(stored_usage.response_body.is_none());
|
||||
assert!(stored_usage.response_body_ref.is_none());
|
||||
assert!(stored_usage.client_response_body.is_none());
|
||||
assert!(stored_usage.client_response_body_ref.is_none());
|
||||
let detail = load_admin_usage_capture_detail(&gateway_state, &stored_usage.id, true).await;
|
||||
let shallow = load_admin_usage_capture_detail(&gateway_state, &stored_usage.id, false).await;
|
||||
for field in [
|
||||
"request_body",
|
||||
"provider_request_body",
|
||||
"response_body",
|
||||
"client_response_body",
|
||||
] {
|
||||
assert!(shallow[field].is_null());
|
||||
let expected_captured = record_level == "full" && field != "client_response_body";
|
||||
assert_eq!(
|
||||
shallow[format!("has_{field}")],
|
||||
expected_captured,
|
||||
"availability for {field}"
|
||||
);
|
||||
if expected_captured {
|
||||
assert!(!detail[field].is_null(), "full should expose {field}");
|
||||
} else {
|
||||
assert!(
|
||||
detail[field].is_null(),
|
||||
"uncaptured {field} must remain absent"
|
||||
);
|
||||
}
|
||||
}
|
||||
if record_level == "full" {
|
||||
assert_eq!(
|
||||
detail["request_body"]["messages"][0]["content"],
|
||||
"request body should not be persisted"
|
||||
);
|
||||
assert_eq!(detail["provider_request_body"]["model"], "gpt-5-upstream");
|
||||
assert_eq!(detail["response_body"], body_json);
|
||||
assert!(detail["client_response_body"].is_null());
|
||||
}
|
||||
|
||||
let stored_candidates = request_candidate_repository
|
||||
.list_by_request_id("trace-openai-chat-local-report-sync-base-123")
|
||||
@@ -825,10 +947,16 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
|
||||
);
|
||||
assert!(stored_usage.response_body.is_none());
|
||||
assert!(stored_usage.response_body_ref.is_none());
|
||||
assert!(stored_usage.response_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.response_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
assert!(stored_usage.client_response_body.is_none());
|
||||
assert!(stored_usage.client_response_body_ref.is_none());
|
||||
assert!(stored_usage.client_response_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.client_response_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
|
||||
let stored_candidates = request_candidate_repository
|
||||
.list_by_request_id("trace-openai-chat-local-report-sync-failure-123")
|
||||
@@ -930,7 +1058,10 @@ async fn gateway_records_failed_usage_when_sync_runtime_transport_is_unavailable
|
||||
assert_eq!(stored_usage.status_code, Some(503));
|
||||
assert!(stored_usage.response_body.is_none());
|
||||
assert!(stored_usage.response_body_ref.is_none());
|
||||
assert!(stored_usage.response_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.response_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
|
||||
let stored_candidates = request_candidate_repository
|
||||
.list_by_request_id("trace-openai-chat-local-transport-unavailable-123")
|
||||
@@ -1272,7 +1403,10 @@ async fn gateway_records_failed_usage_for_claude_runtime_miss_without_execution_
|
||||
);
|
||||
assert!(stored_usage.client_response_body.is_none());
|
||||
assert!(stored_usage.client_response_body_ref.is_none());
|
||||
assert!(stored_usage.client_response_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.client_response_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
assert!(stored_usage.error_message.is_none());
|
||||
|
||||
let stored_candidates = request_candidate_repository
|
||||
@@ -1296,11 +1430,20 @@ fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usa
|
||||
{
|
||||
run_async_test_on_large_stack(
|
||||
"gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled",
|
||||
gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl(),
|
||||
gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl("basic"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_full_request_record_level_preserves_stream_bodies_in_admin_detail() {
|
||||
run_async_test_on_large_stack(
|
||||
"gateway_full_request_record_level_preserves_stream_bodies_in_admin_detail",
|
||||
gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl("full"),
|
||||
);
|
||||
}
|
||||
|
||||
async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_when_usage_runtime_enabled_impl(
|
||||
record_level: &str,
|
||||
) {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
@@ -1406,13 +1549,13 @@ async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_wh
|
||||
Arc::clone(&request_candidate_repository),
|
||||
Arc::clone(&usage_repository),
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
),
|
||||
).with_system_config_values_for_tests([("request_record_level".to_string(), json!(record_level))]),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
..UsageRuntimeConfig::default()
|
||||
});
|
||||
let gateway = build_router_with_state(gateway_state);
|
||||
let gateway = build_router_with_state(gateway_state.clone());
|
||||
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
@@ -1448,6 +1591,30 @@ async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_wh
|
||||
assert!(stored_usage.response_time_ms >= stored_usage.first_byte_time_ms);
|
||||
assert!(stored_usage.is_stream);
|
||||
|
||||
let detail = load_admin_usage_capture_detail(&gateway_state, &stored_usage.id, true).await;
|
||||
for field in [
|
||||
"request_body",
|
||||
"provider_request_body",
|
||||
"response_body",
|
||||
"client_response_body",
|
||||
] {
|
||||
if record_level == "full" {
|
||||
assert!(
|
||||
!detail[field].is_null(),
|
||||
"full stream should expose {field}"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
detail[field].is_null(),
|
||||
"basic stream must not persist {field}"
|
||||
);
|
||||
}
|
||||
}
|
||||
if record_level == "full" {
|
||||
assert!(detail["response_body"].to_string().contains("hello"));
|
||||
assert!(detail["client_response_body"].to_string().contains("hello"));
|
||||
}
|
||||
|
||||
let stored_candidates = request_candidate_repository
|
||||
.list_by_request_id("trace-openai-chat-local-report-stream-123")
|
||||
.await
|
||||
@@ -1585,10 +1752,10 @@ async fn gateway_ignores_legacy_max_response_body_size_for_stream_usage_impl() {
|
||||
Arc::clone(&usage_repository),
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests([(
|
||||
"max_response_body_size".to_string(),
|
||||
json!(128),
|
||||
)]),
|
||||
.with_system_config_values_for_tests([
|
||||
("max_response_body_size".to_string(), json!(128)),
|
||||
("request_record_level".to_string(), json!("full")),
|
||||
]),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
@@ -1624,12 +1791,34 @@ async fn gateway_ignores_legacy_max_response_body_size_for_stream_usage_impl() {
|
||||
)
|
||||
.await;
|
||||
assert_eq!(stored_usage.total_tokens, 6);
|
||||
assert!(stored_usage.response_body.is_none());
|
||||
assert!(
|
||||
stored_usage
|
||||
.response_body
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.to_string()
|
||||
.len()
|
||||
> 128
|
||||
);
|
||||
assert!(stored_usage.response_body_ref.is_none());
|
||||
assert!(stored_usage.response_body_state.is_none());
|
||||
assert!(stored_usage.client_response_body.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.response_body_state,
|
||||
Some(UsageBodyCaptureState::Inline)
|
||||
);
|
||||
assert!(
|
||||
stored_usage
|
||||
.client_response_body
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.to_string()
|
||||
.len()
|
||||
> 128
|
||||
);
|
||||
assert!(stored_usage.client_response_body_ref.is_none());
|
||||
assert!(stored_usage.client_response_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.client_response_body_state,
|
||||
Some(UsageBodyCaptureState::Inline)
|
||||
);
|
||||
|
||||
gateway_handle.abort();
|
||||
execution_runtime_handle.abort();
|
||||
@@ -1903,10 +2092,16 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
|
||||
Some("all_candidates_skipped")
|
||||
);
|
||||
assert!(stored_usage.error_message.is_none());
|
||||
assert!(stored_usage.request_headers.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.request_headers.as_ref().unwrap()["authorization"],
|
||||
"[redacted]"
|
||||
);
|
||||
assert!(stored_usage.request_body.is_none());
|
||||
assert!(stored_usage.request_body_ref.is_none());
|
||||
assert!(stored_usage.request_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.request_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
assert!(stored_usage.provider_request_body.is_none());
|
||||
assert_eq!(
|
||||
stored_usage
|
||||
@@ -2169,7 +2364,10 @@ fn gateway_keeps_failed_usage_request_capture_lightweight_for_large_local_claude
|
||||
)
|
||||
.await;
|
||||
assert_eq!(stored_usage.status, "failed");
|
||||
assert!(stored_usage.request_body_state.is_none());
|
||||
assert_eq!(
|
||||
stored_usage.request_body_state,
|
||||
Some(UsageBodyCaptureState::Disabled)
|
||||
);
|
||||
assert!(stored_usage.request_body.is_none());
|
||||
assert!(stored_usage.request_body_ref.is_none());
|
||||
assert!(stored_usage.provider_request_body.is_none());
|
||||
|
||||
@@ -213,6 +213,13 @@ async fn gateway_background_video_task_poller_refreshes_due_openai_task_from_rep
|
||||
};
|
||||
|
||||
assert_eq!(stored.status, VideoTaskStatus::Processing);
|
||||
assert_eq!(stored.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(stored.username.as_deref(), Some("video-user"));
|
||||
assert_eq!(stored.api_key_name.as_deref(), Some("video-key"));
|
||||
assert_eq!(stored.duration_seconds, Some(4));
|
||||
assert_eq!(stored.resolution.as_deref(), Some("720p"));
|
||||
assert_eq!(stored.aspect_ratio.as_deref(), Some("16:9"));
|
||||
assert_eq!(stored.size.as_deref(), Some("1280x720"));
|
||||
assert_eq!(stored.progress_percent, 37);
|
||||
assert_eq!(stored.poll_count, 1);
|
||||
assert!(
|
||||
|
||||
@@ -32,6 +32,7 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
|
||||
struct SeenExecutionRuntimeStreamRequest {
|
||||
method: String,
|
||||
url: String,
|
||||
headers: serde_json::Value,
|
||||
}
|
||||
|
||||
fn hash_api_key(value: &str) -> String {
|
||||
@@ -159,6 +160,7 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
|
||||
.and_then(|value| value.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
headers: payload.get("headers").cloned().unwrap_or_else(|| json!({})),
|
||||
});
|
||||
|
||||
let frames = [
|
||||
@@ -252,7 +254,10 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
|
||||
updated_at_unix_secs: 456,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
video_url: Some("https://cdn.example.com/video-content.mp4".to_string()),
|
||||
video_url: Some(
|
||||
"https://cdn.example.com/video-content.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1"
|
||||
.to_string(),
|
||||
),
|
||||
request_metadata: None,
|
||||
})
|
||||
.await
|
||||
@@ -358,8 +363,9 @@ async fn gateway_executes_openai_video_content_from_reconstructed_data_task_with
|
||||
assert_eq!(seen_stream_request.method, "GET");
|
||||
assert_eq!(
|
||||
seen_stream_request.url,
|
||||
"https://api.openai.example/v1/videos/ext-video-content-followup-123/content"
|
||||
"https://cdn.example.com/video-content.mp4?signature=a%2Fb%2Bc%3D&part=2&part=1"
|
||||
);
|
||||
assert!(seen_stream_request.headers.get("authorization").is_none());
|
||||
assert_eq!(*decision_stream_hits.lock().expect("mutex should lock"), 0);
|
||||
assert_eq!(*execute_stream_hits.lock().expect("mutex should lock"), 0);
|
||||
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
||||
|
||||
Reference in New Issue
Block a user