use aether_data_contracts::repository::{ candidates::{ sanitize_request_candidate_api_formats, sanitize_request_candidate_error_type, sanitize_request_candidate_extra_data_for_persistence, sanitize_request_candidate_required_capabilities, sanitize_request_candidate_skip_reason, DecisionTrace, DecisionTraceCandidate, RequestCandidateStatus, }, provider_catalog::StoredProviderCatalogKey, usage::StoredRequestUsageAudit, }; use axum::{ body::Body, http, response::{IntoResponse, Response}, Json, }; use serde_json::{json, Value}; use std::collections::BTreeMap; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct AdminMonitoringKeyAccountDisplay { pub label: Option, pub oauth_plan_type: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AdminMonitoringRoute { AuditLogs, SystemStatus, SuspiciousActivities, UserBehavior, ResilienceStatus, ResilienceErrorStats, ResilienceCircuitHistory, TraceRequest, TraceProviderStats, CacheStats, CacheAffinity, CacheAffinities, CacheUsersDelete, CacheAffinityDelete, CacheFlush, CacheProviderDelete, CacheConfig, CacheMetrics, CacheModelMappingStats, CacheModelMappingDelete, CacheModelMappingDeleteModel, CacheModelMappingDeleteProvider, CacheRedisKeys, CacheRedisKeysDelete, } pub fn admin_monitoring_bad_request_response(detail: impl Into) -> Response { ( http::StatusCode::BAD_REQUEST, Json(json!({ "detail": detail.into() })), ) .into_response() } pub fn admin_monitoring_not_found_response(detail: &'static str) -> Response { ( http::StatusCode::NOT_FOUND, Json(json!({ "detail": detail })), ) .into_response() } pub fn build_admin_monitoring_audit_logs_payload_response( items: Vec, total: usize, limit: usize, offset: usize, username: Option, event_type: Option, days: i64, ) -> Response { let count = items.len(); Json(json!({ "items": items, "meta": { "total": total, "limit": limit, "offset": offset, "count": count, }, "filters": { "username": username, "event_type": event_type, "days": days, }, })) .into_response() } pub fn build_admin_monitoring_suspicious_activities_payload_response( activities: Vec, hours: i64, ) -> Response { let count = activities.len(); Json(json!({ "activities": activities, "count": count, "time_range_hours": hours, })) .into_response() } pub fn build_admin_monitoring_user_behavior_payload_response( user_id: String, days: i64, event_counts: BTreeMap, failed_requests: u64, success_requests: u64, suspicious_activities: u64, ) -> Response { let total_requests = success_requests.saturating_add(failed_requests); let success_rate = if total_requests == 0 { 0.0 } else { success_requests as f64 / total_requests as f64 }; Json(json!({ "user_id": user_id, "period_days": days, "event_counts": event_counts, "failed_requests": failed_requests, "success_requests": success_requests, "success_rate": success_rate, "suspicious_activities": suspicious_activities, "analysis_time": chrono::Utc::now().to_rfc3339(), })) .into_response() } pub fn admin_monitoring_user_behavior_user_id_from_path(request_path: &str) -> Option { path_identifier_from_path(request_path, "/api/admin/monitoring/user-behavior/") } pub fn admin_monitoring_trace_request_id_from_path(request_path: &str) -> Option { path_identifier_from_path(request_path, "/api/admin/monitoring/trace/") } pub fn admin_monitoring_trace_provider_id_from_path(request_path: &str) -> Option { path_identifier_from_path(request_path, "/api/admin/monitoring/trace/stats/provider/") } pub fn parse_admin_monitoring_attempted_only(query: Option<&str>) -> Result { match query_param_value(query, "attempted_only") { None => Ok(false), Some(value) => match value.trim().to_ascii_lowercase().as_str() { "true" | "1" | "yes" => Ok(true), "false" | "0" | "no" => Ok(false), _ => Err("attempted_only must be a boolean".to_string()), }, } } pub fn admin_monitoring_trace_not_found_response( request_id: &str, attempted_only: bool, ) -> Response { ( http::StatusCode::NOT_FOUND, Json(json!({ "detail": "Request trace not found", "request_id": request_id, "attempted_only": attempted_only, })), ) .into_response() } pub fn build_admin_monitoring_cache_affinity_delete_success_response( affinity_key: String, endpoint_id: String, model_id: String, message_name: String, ) -> Response { Json(json!({ "status": "ok", "message": format!("已清除缓存亲和性: {message_name}"), "affinity_key": affinity_key, "endpoint_id": endpoint_id, "model_id": model_id, })) .into_response() } pub fn build_admin_monitoring_cache_users_delete_api_key_success_response( user_id: String, username: Option, email: Option, api_key_id: String, api_key_name: Option, ) -> Response { let display_name = api_key_name.clone().unwrap_or_else(|| api_key_id.clone()); Json(json!({ "status": "ok", "message": format!("已清除 API Key {display_name} 的缓存亲和性"), "user_info": { "user_id": Some(user_id), "username": username, "email": email, "api_key_id": api_key_id, "api_key_name": api_key_name, }, })) .into_response() } pub fn build_admin_monitoring_cache_users_delete_user_success_response( user_id: String, username: String, email: Option, ) -> Response { Json(json!({ "status": "ok", "message": format!("已清除用户 {username} 的所有缓存亲和性"), "user_info": { "user_id": user_id, "username": username, "email": email, }, })) .into_response() } pub fn build_admin_monitoring_cache_provider_delete_success_response( provider_id: String, deleted_affinities: usize, ) -> Response { Json(json!({ "status": "ok", "message": format!("已清除 provider {provider_id} 的缓存亲和性"), "provider_id": provider_id, "deleted_affinities": deleted_affinities, })) .into_response() } #[allow(clippy::too_many_arguments)] pub fn build_admin_monitoring_trace_provider_stats_payload_response( provider_id: String, total_attempts: usize, success_count: usize, failed_count: usize, cancelled_count: usize, skipped_count: usize, pending_count: usize, available_count: usize, unused_count: usize, failure_rate: f64, avg_latency_ms: f64, ) -> Response { Json(json!({ "provider_id": provider_id, "total_attempts": total_attempts, "success_count": success_count, "failed_count": failed_count, "cancelled_count": cancelled_count, "skipped_count": skipped_count, "pending_count": pending_count, "available_count": available_count, "unused_count": unused_count, "failure_rate": failure_rate, "avg_latency_ms": avg_latency_ms, })) .into_response() } pub fn build_admin_monitoring_trace_request_payload_response( trace: &DecisionTrace, usage: Option<&StoredRequestUsageAudit>, ) -> Response { build_admin_monitoring_trace_request_payload_response_with_key_accounts( trace, usage, &BTreeMap::new(), ) } pub fn build_admin_monitoring_trace_request_payload_response_with_key_accounts( trace: &DecisionTrace, usage: Option<&StoredRequestUsageAudit>, key_accounts: &BTreeMap, ) -> Response { let usage_candidate_id = usage.and_then(|item| resolve_admin_monitoring_usage_candidate_id(trace, item)); let candidates = trace .candidates .iter() .map(|item| { let matched_usage = usage_candidate_id .as_deref() .filter(|candidate_id| *candidate_id == item.candidate.id.as_str()) .and(usage); build_admin_monitoring_trace_request_candidate_payload_with_key_accounts( item, matched_usage, key_accounts, ) }) .collect::>(); Json(json!({ "request_id": trace.request_id, "request_path": admin_monitoring_trace_request_path(usage), "request_query_string": admin_monitoring_trace_request_query_string(usage), "request_path_and_query": admin_monitoring_trace_request_path_and_query(usage), "total_candidates": trace.total_candidates, "final_status": trace.final_status, "total_latency_ms": trace.total_latency_ms, "candidates": candidates, })) .into_response() } pub fn build_admin_monitoring_trace_request_candidate_payload( item: &DecisionTraceCandidate, usage: Option<&StoredRequestUsageAudit>, ) -> Value { build_admin_monitoring_trace_request_candidate_payload_with_key_accounts( item, usage, &BTreeMap::new(), ) } pub fn build_admin_monitoring_trace_request_candidate_payload_with_key_accounts( item: &DecisionTraceCandidate, usage: Option<&StoredRequestUsageAudit>, key_accounts: &BTreeMap, ) -> Value { let mut item = item.clone(); item.sanitize_for_admin(); let candidate = &item.candidate; let sanitized_extra_data = build_admin_monitoring_trace_candidate_extra_data( candidate.extra_data.as_ref(), candidate.status_code, usage, ); let sanitized_extra_data_ref = (!sanitized_extra_data.is_null()).then_some(&sanitized_extra_data); let sanitized_key_api_formats = sanitize_request_candidate_api_formats(item.provider_key_api_formats.clone()); let sanitized_key_capabilities = sanitize_request_candidate_required_capabilities(item.provider_key_capabilities.clone()); let sanitized_required_capabilities = sanitize_request_candidate_required_capabilities(candidate.required_capabilities.clone()); let key_account = candidate .key_id .as_deref() .and_then(|key_id| key_accounts.get(key_id)); json!({ "id": candidate.id, "request_id": candidate.request_id, "candidate_index": candidate.candidate_index, "retry_index": candidate.retry_index, "provider_id": candidate.provider_id, "provider_name": item.provider_name, "provider_website": item.provider_website, "provider_priority": item.provider_priority, "provider_keep_priority_on_conversion": item.provider_keep_priority_on_conversion, "provider_enable_format_conversion": item.provider_enable_format_conversion, "endpoint_id": candidate.endpoint_id, "endpoint_name": item.endpoint_api_format, "endpoint_api_family": item.endpoint_api_family, "endpoint_kind": item.endpoint_kind, "endpoint_format_acceptance_config": serde_json::Value::Null, "key_id": candidate.key_id, "key_name": item.provider_key_name, "key_account_label": key_account.and_then(|item| item.label.clone()), "key_preview": serde_json::Value::Null, "key_auth_type": item.provider_key_auth_type, "key_api_formats": sanitized_key_api_formats, "key_internal_priority": item.provider_key_internal_priority, "key_global_priority_by_format": serde_json::Value::Null, "key_oauth_plan_type": key_account.and_then(|item| item.oauth_plan_type.clone()), "key_capabilities": sanitized_key_capabilities, "required_capabilities": sanitized_required_capabilities, "status": candidate.status, "skip_reason": sanitize_request_candidate_skip_reason(candidate.skip_reason.clone()), "is_cached": candidate.is_cached, "status_code": candidate.status_code, "error_type": sanitize_request_candidate_error_type(candidate.error_type.clone()), "error_message": candidate.error_message, "latency_ms": candidate.latency_ms, "concurrent_requests": candidate.concurrent_requests, "ranking": build_admin_monitoring_trace_candidate_ranking(sanitized_extra_data_ref), "image_progress": sanitized_extra_data_ref .and_then(|value| value.get("image_progress")) .cloned() .unwrap_or(Value::Null), "extra_data": sanitized_extra_data, "created_at": unix_ms_to_rfc3339(candidate.created_at_unix_ms), "started_at": candidate.started_at_unix_ms.and_then(unix_ms_to_rfc3339), "finished_at": candidate.finished_at_unix_ms.and_then(unix_ms_to_rfc3339), }) } fn resolve_admin_monitoring_usage_candidate_id( trace: &DecisionTrace, usage: &StoredRequestUsageAudit, ) -> Option { if !admin_monitoring_usage_matches_trace(usage, trace.request_id.as_str()) { return None; } if let Some(candidate_id) = usage .routing_candidate_id() .map(str::trim) .filter(|value| !value.is_empty()) { return Some(candidate_id.to_string()); } let candidate_index = usage.routing_candidate_index()?; trace .candidates .iter() .filter(|item| u64::from(item.candidate.candidate_index) == candidate_index) .max_by_key(|item| { ( item.candidate.retry_index, item.candidate.finished_at_unix_ms.unwrap_or_default(), item.candidate.started_at_unix_ms.unwrap_or_default(), admin_monitoring_candidate_status_rank(item.candidate.status), ) }) .map(|item| item.candidate.id.clone()) } fn admin_monitoring_usage_matches_trace( usage: &StoredRequestUsageAudit, trace_request_id: &str, ) -> bool { let trace_request_id = trace_request_id.trim(); if trace_request_id.is_empty() { return false; } usage.request_id.trim() == trace_request_id || usage .trace_id() .is_some_and(|value| value.trim() == trace_request_id) || admin_monitoring_headers_contain_trace_id( usage.request_headers.as_ref(), trace_request_id, ) || admin_monitoring_headers_contain_trace_id( usage.provider_request_headers.as_ref(), trace_request_id, ) } fn admin_monitoring_headers_contain_trace_id( headers: Option<&Value>, trace_request_id: &str, ) -> bool { headers.and_then(Value::as_object).is_some_and(|object| { object.iter().any(|(key, value)| { key.eq_ignore_ascii_case("x-trace-id") && value .as_str() .map(str::trim) .is_some_and(|value| value == trace_request_id) }) }) } fn admin_monitoring_candidate_status_rank(status: RequestCandidateStatus) -> u8 { match status { RequestCandidateStatus::Success => 7, RequestCandidateStatus::Streaming => 6, RequestCandidateStatus::Pending => 5, RequestCandidateStatus::Failed => 4, RequestCandidateStatus::Cancelled => 3, RequestCandidateStatus::Skipped => 2, RequestCandidateStatus::Unused => 1, RequestCandidateStatus::Available => 0, } } fn build_admin_monitoring_trace_candidate_ranking(existing: Option<&Value>) -> Value { let Some(object) = existing.and_then(Value::as_object) else { return Value::Null; }; let mut ranking = serde_json::Map::new(); if let Some(ranking_mode) = json_string_field(object, "ranking_mode") { ranking.insert("mode".to_string(), Value::String(ranking_mode)); } if let Some(priority_mode) = json_string_field(object, "priority_mode") { ranking.insert("priority_mode".to_string(), Value::String(priority_mode)); } if let Some(ranking_index) = json_u64_field(object, "ranking_index") { ranking.insert("index".to_string(), Value::Number(ranking_index.into())); } if let Some(priority_slot) = json_i64_field(object, "priority_slot") { ranking.insert( "priority_slot".to_string(), Value::Number(priority_slot.into()), ); } if let Some(promoted_by) = json_string_field(object, "promoted_by") { ranking.insert("promoted_by".to_string(), Value::String(promoted_by)); } if let Some(demoted_by) = json_string_field(object, "demoted_by") { ranking.insert("demoted_by".to_string(), Value::String(demoted_by)); } if ranking.is_empty() { Value::Null } else { Value::Object(ranking) } } fn build_admin_monitoring_trace_candidate_extra_data( existing: Option<&Value>, candidate_status_code: Option, usage: Option<&StoredRequestUsageAudit>, ) -> Value { let mut extra_data = sanitize_request_candidate_extra_data_for_persistence(existing.cloned()) .and_then(|value| value.as_object().cloned()); if let Some(usage) = usage { let extra_object = extra_data.get_or_insert_with(serde_json::Map::new); if let Some(first_byte_time_ms) = usage.first_byte_time_ms { extra_object .entry("first_byte_time_ms".to_string()) .or_insert_with(|| json!(first_byte_time_ms)); } if let Some(request_path) = admin_monitoring_usage_request_path(usage) { extra_object .entry("request_path".to_string()) .or_insert_with(|| json!(request_path)); } if let Some(request_query_string) = admin_monitoring_usage_request_query_string(usage) { extra_object .entry("request_query_string".to_string()) .or_insert_with(|| json!(request_query_string)); } if let Some(request_path_and_query) = admin_monitoring_usage_request_path_and_query(usage) { extra_object .entry("request_path_and_query".to_string()) .or_insert_with(|| json!(request_path_and_query)); } if admin_monitoring_usage_is_error_node(usage) { if let Some(response) = admin_monitoring_trace_response_data( "upstream_response", candidate_status_code, usage.response_body_state, ) { merge_admin_monitoring_trace_response(extra_object, "upstream_response", response); } } if let Some(proxy_value) = extra_object.get_mut("proxy") { if let Some(proxy_object) = proxy_value.as_object_mut() { let proxy_timing = parse_admin_monitoring_usage_proxy_timing(usage); let proxy_ttfb_ms = proxy_timing .as_ref() .and_then(|timing| timing.get("ttfb_ms")) .and_then(Value::as_u64) .or(usage.first_byte_time_ms); if let Some(ttfb_ms) = proxy_ttfb_ms { proxy_object .entry("ttfb_ms".to_string()) .or_insert_with(|| json!(ttfb_ms)); } if let Some(timing) = proxy_timing { proxy_object.entry("timing".to_string()).or_insert(timing); } } } } sanitize_request_candidate_extra_data_for_persistence(extra_data.map(Value::Object)) .unwrap_or(Value::Null) } fn admin_monitoring_trace_response_data( source: &str, status_code: Option, body_state: Option, ) -> Option { if status_code.is_none() && body_state.is_none() { return None; } Some(json!({ "source": source, "status_code": status_code, "body_state": body_state.map(|state| state.as_str()), })) } fn merge_admin_monitoring_trace_response( extra_object: &mut serde_json::Map, key: &str, response: Value, ) { let Some(response_object) = response.as_object() else { extra_object.insert(key.to_string(), response); return; }; let Some(existing_object) = extra_object.get_mut(key).and_then(Value::as_object_mut) else { extra_object.insert(key.to_string(), Value::Object(response_object.clone())); return; }; for (field, value) in response_object { if field == "status_code" && existing_object.get(field).is_some_and(Value::is_number) { continue; } if admin_monitoring_trace_response_value_empty(value) && existing_object .get(field) .is_some_and(|existing| !admin_monitoring_trace_response_value_empty(existing)) { continue; } existing_object.insert(field.clone(), value.clone()); } } fn admin_monitoring_trace_response_value_empty(value: &Value) -> bool { value.is_null() || value.as_str().is_some_and(str::is_empty) || value.as_array().is_some_and(Vec::is_empty) || value.as_object().is_some_and(serde_json::Map::is_empty) } fn admin_monitoring_usage_is_error_node(usage: &StoredRequestUsageAudit) -> bool { !usage.status.eq_ignore_ascii_case("completed") || usage .status_code .is_some_and(|status| !(200..300).contains(&status)) } fn admin_monitoring_trace_request_path(usage: Option<&StoredRequestUsageAudit>) -> Option { usage.and_then(admin_monitoring_usage_request_path) } fn admin_monitoring_trace_request_query_string( usage: Option<&StoredRequestUsageAudit>, ) -> Option { usage.and_then(admin_monitoring_usage_request_query_string) } fn admin_monitoring_trace_request_path_and_query( usage: Option<&StoredRequestUsageAudit>, ) -> Option { usage.and_then(admin_monitoring_usage_request_path_and_query) } fn admin_monitoring_usage_request_path(usage: &StoredRequestUsageAudit) -> Option { admin_monitoring_usage_metadata_string(usage, "request_path") .and_then(|value| aether_ai_formats::api::sanitize_request_path(&value)) } fn admin_monitoring_usage_request_query_string(usage: &StoredRequestUsageAudit) -> Option { admin_monitoring_usage_metadata_string(usage, "request_query_string") .and_then(|value| aether_ai_formats::api::sanitize_request_query_string(&value)) } fn admin_monitoring_usage_request_path_and_query( usage: &StoredRequestUsageAudit, ) -> Option { admin_monitoring_usage_metadata_string(usage, "request_path_and_query") .and_then(|value| aether_ai_formats::api::sanitize_request_path_and_query(&value, None)) .or_else(|| { let path = admin_monitoring_usage_metadata_string(usage, "request_path")?; let query = admin_monitoring_usage_metadata_string(usage, "request_query_string") .and_then(|value| aether_ai_formats::api::sanitize_request_query_string(&value)); aether_ai_formats::api::sanitize_request_path_and_query(&path, query.as_deref()) }) } fn admin_monitoring_usage_metadata_string( usage: &StoredRequestUsageAudit, key: &str, ) -> Option { usage .request_metadata .as_ref() .and_then(Value::as_object) .and_then(|metadata| metadata.get(key)) .and_then(Value::as_str) .map(str::trim) .filter(|value| !value.is_empty()) .map(ToOwned::to_owned) } fn json_string_field(object: &serde_json::Map, key: &str) -> Option { object .get(key) .and_then(Value::as_str) .map(str::trim) .filter(|value| !value.is_empty()) .map(ToOwned::to_owned) } fn json_u64_field(object: &serde_json::Map, key: &str) -> Option { match object.get(key)? { Value::Number(value) => value.as_u64(), Value::String(value) => value.trim().parse::().ok(), _ => None, } } fn json_i64_field(object: &serde_json::Map, key: &str) -> Option { match object.get(key)? { Value::Number(value) => value.as_i64().or_else(|| value.as_u64()?.try_into().ok()), Value::String(value) => value.trim().parse::().ok(), _ => None, } } fn parse_admin_monitoring_usage_proxy_timing(usage: &StoredRequestUsageAudit) -> Option { admin_monitoring_header_value(usage.response_headers.as_ref(), "x-proxy-timing") .or_else(|| { admin_monitoring_header_value(usage.client_response_headers.as_ref(), "x-proxy-timing") }) .and_then(|raw| parse_admin_monitoring_proxy_timing_value(&raw)) } fn admin_monitoring_header_value(headers: Option<&Value>, name: &str) -> Option { headers .and_then(Value::as_object) .and_then(|object| { object .iter() .find(|(key, _)| key.eq_ignore_ascii_case(name)) .map(|(_, value)| value) }) .and_then(|value| match value { Value::String(text) => Some(text.trim().to_string()), Value::Object(object) => Some(Value::Object(object.clone()).to_string()), _ => None, }) .filter(|value| !value.is_empty()) } fn parse_admin_monitoring_proxy_timing_value(raw: &str) -> Option { serde_json::from_str::(raw) .ok() .filter(Value::is_object) } #[allow(clippy::too_many_arguments)] pub fn build_admin_monitoring_system_status_payload_response( timestamp: chrono::DateTime, total_users: u64, active_users: u64, total_providers: usize, active_providers: usize, total_api_keys: u64, active_api_keys: u64, today_requests: usize, today_tokens: u64, today_cost: f64, proxy_connections: usize, nodes: usize, active_streams: usize, path_prefixes: &[&str], recent_errors: usize, usage_counter: Value, ) -> Response { Json(json!({ "timestamp": timestamp.to_rfc3339(), "users": { "total": total_users, "active": active_users, }, "providers": { "total": total_providers, "active": active_providers, }, "api_keys": { "total": total_api_keys, "active": active_api_keys, }, "today_stats": { "requests": today_requests, "tokens": today_tokens, "cost_usd": format!("${today_cost:.4}"), }, "tunnel": { "proxy_connections": proxy_connections, "nodes": nodes, "active_streams": active_streams, }, "internal_gateway": { "status": "rust_native_control_plane", "path_prefixes": path_prefixes, }, "recent_errors": recent_errors, "usage_counter": usage_counter, })) .into_response() } pub fn build_admin_monitoring_resilience_status_payload_response( timestamp: chrono::DateTime, health_score: i64, status: &'static str, error_statistics: Value, recent_errors: Vec, recommendations: Vec, ) -> Response { Json(json!({ "timestamp": timestamp.to_rfc3339(), "health_score": health_score, "status": status, "error_statistics": error_statistics, "recent_errors": recent_errors, "recommendations": recommendations, })) .into_response() } pub fn build_admin_monitoring_reset_error_stats_payload_response( previous_stats: Value, reset_by: Option, reset_at: chrono::DateTime, ) -> Response { Json(json!({ "message": "错误统计已重置", "previous_stats": previous_stats, "reset_by": reset_by, "reset_at": reset_at.to_rfc3339(), })) .into_response() } pub fn parse_admin_monitoring_circuit_history_limit(query: Option<&str>) -> Result { match query_param_value(query, "limit") { None => Ok(50), Some(value) => { let parsed = value .parse::() .map_err(|_| "limit must be an integer between 1 and 200".to_string())?; if (1..=200).contains(&parsed) { Ok(parsed) } else { Err("limit must be an integer between 1 and 200".to_string()) } } } } pub fn build_admin_monitoring_circuit_history_items( keys: &[StoredProviderCatalogKey], provider_name_by_id: &BTreeMap, limit: usize, ) -> Vec { let mut items = Vec::new(); for key in keys { let health_by_format = key .health_by_format .as_ref() .and_then(serde_json::Value::as_object) .cloned() .unwrap_or_default(); let circuit_by_format = key .circuit_breaker_by_format .as_ref() .and_then(serde_json::Value::as_object) .cloned() .unwrap_or_default(); for (api_format, circuit_value) in circuit_by_format { let Some(circuit) = circuit_value.as_object() else { continue; }; let is_open = circuit .get("open") .and_then(serde_json::Value::as_bool) .unwrap_or(false); let is_half_open = circuit .get("half_open_until") .and_then(serde_json::Value::as_str) .is_some(); if !is_open && !is_half_open { continue; } let health = health_by_format .get(&api_format) .and_then(serde_json::Value::as_object); let timestamp = circuit .get("open_at") .and_then(serde_json::Value::as_str) .or_else(|| { circuit .get("half_open_until") .and_then(serde_json::Value::as_str) }) .or_else(|| { health.and_then(|value| { value .get("last_failure_at") .and_then(serde_json::Value::as_str) }) }) .map(ToOwned::to_owned); let event = if is_half_open { "half_open" } else { "opened" }; let reason = circuit .get("reason") .and_then(serde_json::Value::as_str) .map(ToOwned::to_owned) .or_else(|| { health .and_then(|value| { value .get("consecutive_failures") .and_then(serde_json::Value::as_i64) }) .filter(|value| *value > 0) .map(|value| format!("连续失败 {value} 次")) }) .or_else(|| { Some(if is_half_open { "熔断器处于半开状态".to_string() } else { "熔断器处于打开状态".to_string() }) }); let recovery_seconds = circuit .get("recovery_seconds") .and_then(serde_json::Value::as_i64) .or_else(|| { let open_at = circuit .get("open_at") .and_then(serde_json::Value::as_str) .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok()); let next_probe_at = circuit .get("next_probe_at") .and_then(serde_json::Value::as_str) .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok()); match (open_at, next_probe_at) { (Some(open_at), Some(next_probe_at)) => { Some((next_probe_at - open_at).num_seconds().max(0)) } _ => None, } }); items.push(json!({ "event": event, "key_id": key.id, "provider_id": key.provider_id, "provider_name": provider_name_by_id.get(&key.provider_id).cloned(), "key_name": key.name, "api_format": api_format, "reason": reason, "recovery_seconds": recovery_seconds, "timestamp": timestamp, })); } } items.sort_by(|left, right| { let left_ts = left .get("timestamp") .and_then(serde_json::Value::as_str) .unwrap_or_default(); let right_ts = right .get("timestamp") .and_then(serde_json::Value::as_str) .unwrap_or_default(); right_ts.cmp(left_ts) }); items.truncate(limit); items } pub fn build_admin_monitoring_circuit_history_payload_response( items: Vec, ) -> Response { let count = items.len(); Json(json!({ "items": items, "count": count, })) .into_response() } pub fn admin_monitoring_unknown_cache_category_response(category: &str) -> Response { ( http::StatusCode::NOT_FOUND, Json(json!({ "detail": format!("未知的缓存分类: {category}") })), ) .into_response() } pub fn build_admin_monitoring_redis_keys_delete_success_response( category: &str, name: &str, deleted_count: usize, ) -> Response { Json(json!({ "status": "ok", "message": format!("已清除 {name} 缓存"), "category": category, "deleted_count": deleted_count, })) .into_response() } pub fn build_admin_monitoring_cache_flush_success_response( deleted_affinities: usize, ) -> Response { Json(json!({ "status": "ok", "message": "已清除全部缓存亲和性", "deleted_affinities": deleted_affinities, })) .into_response() } pub fn admin_monitoring_cache_provider_not_found_response(provider_id: &str) -> Response { ( http::StatusCode::NOT_FOUND, Json(json!({ "detail": format!("未找到 provider {provider_id} 的缓存亲和性记录") })), ) .into_response() } pub fn build_admin_monitoring_model_mapping_delete_success_response( deleted_count: usize, ) -> Response { Json(json!({ "status": "ok", "message": "已清除所有模型映射缓存", "deleted_count": deleted_count, })) .into_response() } pub fn build_admin_monitoring_model_mapping_delete_model_success_response( model_name: String, deleted_keys: Vec, ) -> Response { Json(json!({ "status": "ok", "message": format!("已清除模型 {model_name} 的映射缓存"), "model_name": model_name, "deleted_keys": deleted_keys, })) .into_response() } pub fn build_admin_monitoring_model_mapping_delete_provider_success_response( provider_id: String, global_model_id: String, deleted_keys: Vec, ) -> Response { Json(json!({ "status": "ok", "message": "已清除 Provider 模型映射缓存", "provider_id": provider_id, "global_model_id": global_model_id, "deleted_keys": deleted_keys, })) .into_response() } pub fn match_admin_monitoring_route( method: &http::Method, path: &str, ) -> Option { let path = normalize_admin_monitoring_path(path); match *method { http::Method::GET => match path { "/api/admin/monitoring/audit-logs" => Some(AdminMonitoringRoute::AuditLogs), "/api/admin/monitoring/system-status" => Some(AdminMonitoringRoute::SystemStatus), "/api/admin/monitoring/suspicious-activities" => { Some(AdminMonitoringRoute::SuspiciousActivities) } "/api/admin/monitoring/resilience-status" => { Some(AdminMonitoringRoute::ResilienceStatus) } "/api/admin/monitoring/resilience/circuit-history" => { Some(AdminMonitoringRoute::ResilienceCircuitHistory) } "/api/admin/monitoring/cache/stats" => Some(AdminMonitoringRoute::CacheStats), "/api/admin/monitoring/cache/affinities" => Some(AdminMonitoringRoute::CacheAffinities), "/api/admin/monitoring/cache/config" => Some(AdminMonitoringRoute::CacheConfig), "/api/admin/monitoring/cache/metrics" => Some(AdminMonitoringRoute::CacheMetrics), "/api/admin/monitoring/cache/model-mapping/stats" => { Some(AdminMonitoringRoute::CacheModelMappingStats) } "/api/admin/monitoring/cache/redis-keys" => Some(AdminMonitoringRoute::CacheRedisKeys), _ if matches_dynamic_segments(path, "/api/admin/monitoring/user-behavior/", 1) => { Some(AdminMonitoringRoute::UserBehavior) } _ if matches_dynamic_segments( path, "/api/admin/monitoring/trace/stats/provider/", 1, ) => { Some(AdminMonitoringRoute::TraceProviderStats) } _ if matches_dynamic_segments(path, "/api/admin/monitoring/trace/", 1) => { Some(AdminMonitoringRoute::TraceRequest) } _ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/affinity/", 1) => { Some(AdminMonitoringRoute::CacheAffinity) } _ => None, }, http::Method::DELETE => match path { "/api/admin/monitoring/resilience/error-stats" => { Some(AdminMonitoringRoute::ResilienceErrorStats) } "/api/admin/monitoring/cache" => Some(AdminMonitoringRoute::CacheFlush), "/api/admin/monitoring/cache/model-mapping" => { Some(AdminMonitoringRoute::CacheModelMappingDelete) } _ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/users/", 1) => { Some(AdminMonitoringRoute::CacheUsersDelete) } _ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/providers/", 1) => { Some(AdminMonitoringRoute::CacheProviderDelete) } _ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/redis-keys/", 1) => { Some(AdminMonitoringRoute::CacheRedisKeysDelete) } _ if matches_dynamic_segments( path, "/api/admin/monitoring/cache/model-mapping/provider/", 2, ) => { Some(AdminMonitoringRoute::CacheModelMappingDeleteProvider) } _ if matches_dynamic_segments( path, "/api/admin/monitoring/cache/model-mapping/", 1, ) => { Some(AdminMonitoringRoute::CacheModelMappingDeleteModel) } _ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/affinity/", 4) => { Some(AdminMonitoringRoute::CacheAffinityDelete) } _ => None, }, _ => None, } } fn normalize_admin_monitoring_path(path: &str) -> &str { let normalized = path.trim_end_matches('/'); if normalized.is_empty() { "/" } else { normalized } } fn path_identifier_from_path(request_path: &str, prefix: &str) -> Option { let value = request_path .strip_prefix(prefix)? .trim() .trim_matches('/') .to_string(); if value.is_empty() || value.contains('/') { None } else { Some(value) } } fn unix_ms_to_rfc3339(unix_ms: u64) -> Option { let secs = (unix_ms / 1000) as i64; let nanos = ((unix_ms % 1000) * 1_000_000) as u32; chrono::DateTime::::from_timestamp(secs, nanos) .map(|value| value.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)) } fn query_param_value(query: Option<&str>, key: &str) -> Option { let query = query?; for (entry_key, value) in url::form_urlencoded::parse(query.as_bytes()) { if entry_key == key { let value = value.trim(); if !value.is_empty() { return Some(value.to_string()); } } } None } fn matches_dynamic_segments(path: &str, prefix: &str, dynamic_segments: usize) -> bool { let Some(suffix) = path.strip_prefix(prefix) else { return false; }; let segments = suffix .split('/') .filter(|segment| !segment.is_empty()) .collect::>(); segments.len() == dynamic_segments }