mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
Merge branch 'fawney19:main' into main
This commit is contained in:
@@ -1059,6 +1059,71 @@ fn admin_usage_upstream_is_stream(item: &StoredRequestUsageAudit) -> bool {
|
||||
.unwrap_or(item.is_stream)
|
||||
}
|
||||
|
||||
fn admin_usage_metadata_string<'a>(
|
||||
item: &'a StoredRequestUsageAudit,
|
||||
key: &str,
|
||||
) -> Option<&'a str> {
|
||||
item.request_metadata
|
||||
.as_ref()
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get(key))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn infer_client_family_from_user_agent(user_agent: &str) -> Option<&'static str> {
|
||||
let normalized = user_agent.trim().to_ascii_lowercase();
|
||||
if normalized.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if normalized.starts_with("codex_vscode") {
|
||||
return Some("codex_vscode");
|
||||
}
|
||||
if normalized.starts_with("codex") {
|
||||
return Some("codex");
|
||||
}
|
||||
if normalized.contains("claude-code") || normalized.contains("claude_code") {
|
||||
return Some("claude_code");
|
||||
}
|
||||
if normalized.contains("opencode") {
|
||||
return Some("opencode");
|
||||
}
|
||||
if normalized.contains("geminicli") || normalized.contains("gemini-cli") {
|
||||
return Some("gemini_cli");
|
||||
}
|
||||
if normalized.starts_with("openai/js") {
|
||||
return Some("openai_js_sdk");
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn admin_usage_client_family(item: &StoredRequestUsageAudit) -> Option<&str> {
|
||||
item.client_family
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| {
|
||||
item.request_metadata
|
||||
.as_ref()
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| {
|
||||
metadata
|
||||
.get("client_session_affinity")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|affinity| affinity.get("client_family"))
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| metadata.get("client_family").and_then(Value::as_str))
|
||||
})
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
})
|
||||
.or_else(|| {
|
||||
admin_usage_metadata_string(item, "user_agent")
|
||||
.and_then(infer_client_family_from_user_agent)
|
||||
})
|
||||
}
|
||||
|
||||
fn admin_usage_active_request_json(
|
||||
item: &StoredRequestUsageAudit,
|
||||
api_key_name: Option<String>,
|
||||
@@ -1091,6 +1156,11 @@ fn admin_usage_active_request_json(
|
||||
"upstream_is_stream": upstream_is_stream,
|
||||
"client_requested_stream": client_is_stream,
|
||||
"client_is_stream": client_is_stream,
|
||||
"client_family": admin_usage_client_family(item),
|
||||
"client_ip": admin_usage_metadata_string(item, "client_ip"),
|
||||
"user_agent": admin_usage_metadata_string(item, "user_agent"),
|
||||
"request_path": admin_usage_metadata_string(item, "request_path"),
|
||||
"request_path_and_query": admin_usage_metadata_string(item, "request_path_and_query"),
|
||||
"has_fallback": admin_usage_has_fallback(item),
|
||||
});
|
||||
if let Some(api_format) = item.api_format.as_ref() {
|
||||
@@ -1192,6 +1262,27 @@ pub fn admin_usage_record_json(
|
||||
json!(client_is_stream),
|
||||
);
|
||||
object.insert("client_is_stream".to_string(), json!(client_is_stream));
|
||||
maybe_insert_string_field(object, "client_family", admin_usage_client_family(item));
|
||||
maybe_insert_string_field(
|
||||
object,
|
||||
"client_ip",
|
||||
admin_usage_metadata_string(item, "client_ip"),
|
||||
);
|
||||
maybe_insert_string_field(
|
||||
object,
|
||||
"user_agent",
|
||||
admin_usage_metadata_string(item, "user_agent"),
|
||||
);
|
||||
maybe_insert_string_field(
|
||||
object,
|
||||
"request_path",
|
||||
admin_usage_metadata_string(item, "request_path"),
|
||||
);
|
||||
maybe_insert_string_field(
|
||||
object,
|
||||
"request_path_and_query",
|
||||
admin_usage_metadata_string(item, "request_path_and_query"),
|
||||
);
|
||||
payload
|
||||
}
|
||||
|
||||
@@ -2443,6 +2534,74 @@ mod tests {
|
||||
assert_eq!(record["client_is_stream"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_usage_record_infers_client_family_from_user_agent() {
|
||||
let item = StoredRequestUsageAudit {
|
||||
request_metadata: Some(json!({
|
||||
"client_ip": "192.168.0.28",
|
||||
"user_agent": "codex_vscode/0.131.0-alpha.9 (Windows 10.0.26200; x86_64)"
|
||||
})),
|
||||
..sample_usage("completed", Some(200), None)
|
||||
};
|
||||
|
||||
let record = admin_usage_record_json(
|
||||
&item,
|
||||
&BTreeMap::new(),
|
||||
&BTreeMap::new(),
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
);
|
||||
let active = admin_usage_active_request_json(&item, None, None, None);
|
||||
|
||||
assert_eq!(record["client_family"], "codex_vscode");
|
||||
assert_eq!(record["client_ip"], "192.168.0.28");
|
||||
assert_eq!(active["client_family"], "codex_vscode");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_usage_record_labels_openai_js_user_agent_as_sdk() {
|
||||
let item = StoredRequestUsageAudit {
|
||||
request_metadata: Some(json!({
|
||||
"user_agent": "OpenAI/JS 6.34.0"
|
||||
})),
|
||||
..sample_usage("completed", Some(200), None)
|
||||
};
|
||||
|
||||
let record = admin_usage_record_json(
|
||||
&item,
|
||||
&BTreeMap::new(),
|
||||
&BTreeMap::new(),
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(record["client_family"], "openai_js_sdk");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_usage_record_prefers_typed_client_family() {
|
||||
let item = StoredRequestUsageAudit {
|
||||
client_family: Some("codex".to_string()),
|
||||
request_metadata: Some(json!({
|
||||
"user_agent": "OpenAI/JS 6.34.0"
|
||||
})),
|
||||
..sample_usage("completed", Some(200), None)
|
||||
};
|
||||
|
||||
let record = admin_usage_record_json(
|
||||
&item,
|
||||
&BTreeMap::new(),
|
||||
&BTreeMap::new(),
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(record["client_family"], "codex");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_requested_stream_uses_stream_generate_content_path_over_stale_metadata_flag() {
|
||||
let item = StoredRequestUsageAudit {
|
||||
|
||||
@@ -6,6 +6,10 @@ use chrono::{TimeZone, Utc};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub fn normalize_endpoint_api_format(api_format: &str) -> String {
|
||||
aether_ai_formats::normalize_api_format_alias(api_format)
|
||||
}
|
||||
|
||||
fn unix_secs_to_rfc3339(unix_secs: u64) -> Option<String> {
|
||||
Utc.timestamp_opt(unix_secs as i64, 0)
|
||||
.single()
|
||||
@@ -45,7 +49,7 @@ pub fn key_api_formats_without_entry(
|
||||
fn active_endpoint_api_formats(endpoints: &[StoredProviderCatalogEndpoint]) -> Vec<String> {
|
||||
let mut formats = Vec::new();
|
||||
for endpoint in endpoints.iter().filter(|endpoint| endpoint.is_active) {
|
||||
let api_format = aether_ai_formats::normalize_api_format_alias(&endpoint.api_format);
|
||||
let api_format = normalize_endpoint_api_format(&endpoint.api_format);
|
||||
if !formats.iter().any(|existing| existing == &api_format) {
|
||||
formats.push(api_format);
|
||||
}
|
||||
@@ -63,7 +67,7 @@ fn configured_key_api_formats(key: &StoredProviderCatalogKey) -> Vec<String> {
|
||||
};
|
||||
let mut normalized = Vec::new();
|
||||
for api_format in formats.iter().filter_map(serde_json::Value::as_str) {
|
||||
let api_format = aether_ai_formats::normalize_api_format_alias(api_format);
|
||||
let api_format = normalize_endpoint_api_format(api_format);
|
||||
if !normalized.iter().any(|existing| existing == &api_format) {
|
||||
normalized.push(api_format);
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
mod types;
|
||||
|
||||
pub use types::{
|
||||
parse_usage_body_ref, usage_body_ref, PendingUsageCleanupSummary,
|
||||
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
parse_usage_body_ref, usage_body_ref, usage_request_metadata_client_family,
|
||||
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
StoredProviderApiKeyWindowUsageSummary, StoredProviderUsageSummary, StoredProviderUsageWindow,
|
||||
StoredRequestUsageAudit, StoredUsageAuditAggregation, StoredUsageAuditSummary,
|
||||
StoredUsageBreakdownSummaryRow, StoredUsageCacheAffinityHitSummary,
|
||||
|
||||
@@ -34,6 +34,8 @@ pub struct StoredRequestUsageAudit {
|
||||
pub provider_endpoint_kind: Option<String>,
|
||||
pub has_format_conversion: bool,
|
||||
pub is_stream: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub client_family: Option<String>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
@@ -211,6 +213,7 @@ impl StoredRequestUsageAudit {
|
||||
provider_endpoint_kind,
|
||||
has_format_conversion,
|
||||
is_stream,
|
||||
client_family: None,
|
||||
input_tokens: parse_u64(input_tokens, "usage.input_tokens")?,
|
||||
output_tokens: parse_u64(output_tokens, "usage.output_tokens")?,
|
||||
total_tokens: parse_u64(total_tokens, "usage.total_tokens")?,
|
||||
@@ -311,6 +314,10 @@ impl StoredRequestUsageAudit {
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
pub fn request_metadata_client_family(&self) -> Option<&str> {
|
||||
usage_request_metadata_client_family(self.request_metadata.as_ref())
|
||||
}
|
||||
|
||||
fn billing_snapshot_resolved_number(&self, key: &str) -> Option<f64> {
|
||||
self.request_metadata_object()
|
||||
.and_then(|metadata| metadata.get("billing_snapshot"))
|
||||
@@ -1794,6 +1801,18 @@ pub struct UsageCleanupPreviewCounts {
|
||||
pub log: u64,
|
||||
}
|
||||
|
||||
pub fn usage_request_metadata_client_family(value: Option<&Value>) -> Option<&str> {
|
||||
let metadata = value.and_then(Value::as_object)?;
|
||||
metadata
|
||||
.get("client_session_affinity")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|affinity| affinity.get("client_family"))
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| metadata.get("client_family").and_then(Value::as_str))
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn parse_u64(value: i32, field_name: &str) -> Result<u64, crate::DataLayerError> {
|
||||
u64::try_from(value).map_err(|_| {
|
||||
crate::DataLayerError::UnexpectedValue(format!("invalid {field_name}: {value}"))
|
||||
|
||||
@@ -29,11 +29,12 @@ use serde_json::Value;
|
||||
use super::{
|
||||
api_key_usage_contribution, provider_api_key_usage_contribution,
|
||||
strip_deprecated_usage_display_fields, usage_can_recover_terminal_failure,
|
||||
ApiKeyUsageContribution, ApiKeyUsageDelta, ProviderApiKeyUsageContribution,
|
||||
ProviderApiKeyUsageDelta, ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
StoredProviderApiKeyWindowUsageSummary, StoredProviderUsageSummary, StoredProviderUsageWindow,
|
||||
StoredRequestUsageAudit, StoredUsageDailySummary, UpsertUsageRecord, UsageAuditListQuery,
|
||||
UsageDailyHeatmapQuery, UsageReadRepository, UsageWriteRepository,
|
||||
usage_request_metadata_client_family, ApiKeyUsageContribution, ApiKeyUsageDelta,
|
||||
ProviderApiKeyUsageContribution, ProviderApiKeyUsageDelta, ProviderApiKeyWindowUsageRequest,
|
||||
StoredProviderApiKeyUsageSummary, StoredProviderApiKeyWindowUsageSummary,
|
||||
StoredProviderUsageSummary, StoredProviderUsageWindow, StoredRequestUsageAudit,
|
||||
StoredUsageDailySummary, UpsertUsageRecord, UsageAuditListQuery, UsageDailyHeatmapQuery,
|
||||
UsageReadRepository, UsageWriteRepository,
|
||||
};
|
||||
use crate::repository::auth::InMemoryAuthApiKeySnapshotRepository;
|
||||
use crate::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
@@ -56,6 +57,7 @@ impl InMemoryUsageReadRepository {
|
||||
let mut by_request_id = BTreeMap::new();
|
||||
for mut item in items {
|
||||
hydrate_legacy_body_refs(&mut item);
|
||||
hydrate_client_family(&mut item);
|
||||
by_request_id.insert(item.request_id.clone(), item);
|
||||
}
|
||||
Self {
|
||||
@@ -75,6 +77,7 @@ impl InMemoryUsageReadRepository {
|
||||
let mut detached_bodies = BTreeMap::new();
|
||||
for mut item in items {
|
||||
hydrate_legacy_body_refs(&mut item);
|
||||
hydrate_client_family(&mut item);
|
||||
let request_id = item.request_id.clone();
|
||||
if let Some(body_ref) = detach_usage_body(
|
||||
&request_id,
|
||||
@@ -2546,6 +2549,13 @@ fn hydrate_legacy_body_refs(item: &mut StoredRequestUsageAudit) {
|
||||
}
|
||||
}
|
||||
|
||||
fn hydrate_client_family(item: &mut StoredRequestUsageAudit) {
|
||||
if item.client_family.is_none() {
|
||||
item.client_family = usage_request_metadata_client_family(item.request_metadata.as_ref())
|
||||
.map(ToOwned::to_owned);
|
||||
}
|
||||
}
|
||||
|
||||
fn persisted_usage_body_ref(
|
||||
incoming_ref: Option<&str>,
|
||||
incoming_body: Option<&Value>,
|
||||
@@ -2851,6 +2861,13 @@ impl UsageWriteRepository for InMemoryUsageReadRepository {
|
||||
})
|
||||
},
|
||||
),
|
||||
client_family: usage_request_metadata_client_family(request_metadata.as_ref())
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| {
|
||||
existing
|
||||
.as_ref()
|
||||
.and_then(|existing| existing.client_family.clone())
|
||||
}),
|
||||
request_metadata,
|
||||
created_at_unix_ms,
|
||||
updated_at_unix_secs: usage.updated_at_unix_secs,
|
||||
@@ -4399,6 +4416,7 @@ mod tests {
|
||||
provider_endpoint_kind: Some("chat".to_string()),
|
||||
has_format_conversion: false,
|
||||
is_stream: false,
|
||||
client_family: None,
|
||||
input_tokens: 10,
|
||||
output_tokens: 20,
|
||||
total_tokens: 30,
|
||||
|
||||
@@ -363,7 +363,8 @@ mod sqlite;
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) use aether_data_contracts::repository::usage::{
|
||||
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
usage_request_metadata_client_family, PendingUsageCleanupSummary,
|
||||
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
StoredProviderApiKeyWindowUsageSummary, StoredProviderUsageSummary, StoredProviderUsageWindow,
|
||||
StoredRequestUsageAudit, StoredUsageAuditAggregation, StoredUsageAuditSummary,
|
||||
StoredUsageBreakdownSummaryRow, StoredUsageCacheAffinityHitSummary,
|
||||
|
||||
@@ -6,8 +6,8 @@ use sqlx::{mysql::MySqlRow, Row};
|
||||
use super::{
|
||||
provider_api_key_usage_is_error, provider_api_key_usage_is_success,
|
||||
strip_deprecated_usage_display_fields, usage_can_recover_terminal_failure,
|
||||
InMemoryUsageReadRepository, PendingUsageCleanupSummary, StoredRequestUsageAudit,
|
||||
UpsertUsageRecord, UsageWriteRepository,
|
||||
usage_request_metadata_client_family, InMemoryUsageReadRepository, PendingUsageCleanupSummary,
|
||||
StoredRequestUsageAudit, UpsertUsageRecord, UsageWriteRepository,
|
||||
};
|
||||
use crate::driver::mysql::MysqlPool;
|
||||
use crate::error::SqlResultExt;
|
||||
@@ -850,6 +850,8 @@ fn map_usage_row(row: &MySqlRow) -> Result<StoredRequestUsageAudit, DataLayerErr
|
||||
.map(|raw| serde_json::from_str(&raw))
|
||||
.transpose()
|
||||
.map_err(|err| DataLayerError::UnexpectedValue(err.to_string()))?;
|
||||
audit.client_family = usage_request_metadata_client_family(audit.request_metadata.as_ref())
|
||||
.map(ToOwned::to_owned);
|
||||
let upstream_is_stream = row
|
||||
.try_get::<Option<bool>, _>("upstream_is_stream")
|
||||
.map_sql_err()?;
|
||||
|
||||
@@ -8591,6 +8591,9 @@ fn map_usage_row(
|
||||
.try_get::<f64, _>("cache_read_cost_usd")
|
||||
.map_postgres_err()?;
|
||||
usage.output_price_per_1m = row.try_get("output_price_per_1m").map_postgres_err()?;
|
||||
usage.client_family = row
|
||||
.try_get::<Option<String>, _>("client_family")
|
||||
.map_postgres_err()?;
|
||||
usage.request_headers = row.try_get("request_headers").map_postgres_err()?;
|
||||
let request_body = usage_json_column(
|
||||
row,
|
||||
|
||||
@@ -93,6 +93,10 @@ SELECT
|
||||
"usage".first_byte_time_ms,
|
||||
"usage".status,
|
||||
COALESCE(usage_settlement_snapshots.billing_status, "usage".billing_status) AS billing_status,
|
||||
COALESCE(
|
||||
NULLIF(BTRIM("usage".request_metadata->'client_session_affinity'->>'client_family'), ''),
|
||||
NULLIF(BTRIM("usage".request_metadata->>'client_family'), '')
|
||||
) AS client_family,
|
||||
COALESCE(usage_http_audits.request_headers, "usage".request_headers) AS request_headers,
|
||||
"usage".request_body,
|
||||
"usage".request_body_compressed,
|
||||
|
||||
@@ -93,6 +93,10 @@ SELECT
|
||||
"usage".first_byte_time_ms,
|
||||
"usage".status,
|
||||
COALESCE(usage_settlement_snapshots.billing_status, "usage".billing_status) AS billing_status,
|
||||
COALESCE(
|
||||
NULLIF(BTRIM("usage".request_metadata->'client_session_affinity'->>'client_family'), ''),
|
||||
NULLIF(BTRIM("usage".request_metadata->>'client_family'), '')
|
||||
) AS client_family,
|
||||
COALESCE(usage_http_audits.request_headers, "usage".request_headers) AS request_headers,
|
||||
"usage".request_body,
|
||||
"usage".request_body_compressed,
|
||||
|
||||
@@ -93,6 +93,10 @@ SELECT
|
||||
"usage".first_byte_time_ms,
|
||||
"usage".status,
|
||||
COALESCE(usage_settlement_snapshots.billing_status, "usage".billing_status) AS billing_status,
|
||||
COALESCE(
|
||||
NULLIF(BTRIM("usage".request_metadata->'client_session_affinity'->>'client_family'), ''),
|
||||
NULLIF(BTRIM("usage".request_metadata->>'client_family'), '')
|
||||
) AS client_family,
|
||||
NULL::json AS request_headers,
|
||||
NULL::json AS request_body,
|
||||
NULL::bytea AS request_body_compressed,
|
||||
@@ -106,9 +110,21 @@ SELECT
|
||||
NULL::json AS client_response_body,
|
||||
NULL::bytea AS client_response_body_compressed,
|
||||
CASE
|
||||
WHEN ("usage".request_metadata->>'client_requested_stream') IN ('true', 'false')
|
||||
WHEN NULLIF(BTRIM("usage".request_metadata->>'client_ip'), '') IS NOT NULL
|
||||
OR NULLIF(BTRIM("usage".request_metadata->>'user_agent'), '') IS NOT NULL
|
||||
OR NULLIF(BTRIM("usage".request_metadata->>'request_path'), '') IS NOT NULL
|
||||
OR NULLIF(BTRIM("usage".request_metadata->>'request_path_and_query'), '') IS NOT NULL
|
||||
OR ("usage".request_metadata->>'client_requested_stream') IN ('true', 'false')
|
||||
OR ("usage".request_metadata->>'upstream_is_stream') IN ('true', 'false')
|
||||
THEN jsonb_build_object(
|
||||
THEN jsonb_strip_nulls(jsonb_build_object(
|
||||
'client_ip',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'client_ip'), ''),
|
||||
'user_agent',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'user_agent'), ''),
|
||||
'request_path',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'request_path'), ''),
|
||||
'request_path_and_query',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'request_path_and_query'), ''),
|
||||
'client_requested_stream',
|
||||
CASE
|
||||
WHEN ("usage".request_metadata->>'client_requested_stream') IN ('true', 'false')
|
||||
@@ -121,7 +137,7 @@ SELECT
|
||||
THEN ("usage".request_metadata->>'upstream_is_stream')::boolean
|
||||
ELSE NULL
|
||||
END
|
||||
)::json
|
||||
))::json
|
||||
ELSE NULL::json
|
||||
END AS request_metadata,
|
||||
NULL::varchar AS http_request_body_ref,
|
||||
|
||||
@@ -93,6 +93,10 @@ SELECT
|
||||
"usage".first_byte_time_ms,
|
||||
"usage".status,
|
||||
COALESCE(usage_settlement_snapshots.billing_status, "usage".billing_status) AS billing_status,
|
||||
COALESCE(
|
||||
NULLIF(BTRIM("usage".request_metadata->'client_session_affinity'->>'client_family'), ''),
|
||||
NULLIF(BTRIM("usage".request_metadata->>'client_family'), '')
|
||||
) AS client_family,
|
||||
NULL::json AS request_headers,
|
||||
NULL::json AS request_body,
|
||||
NULL::bytea AS request_body_compressed,
|
||||
@@ -106,9 +110,21 @@ SELECT
|
||||
NULL::json AS client_response_body,
|
||||
NULL::bytea AS client_response_body_compressed,
|
||||
CASE
|
||||
WHEN ("usage".request_metadata->>'client_requested_stream') IN ('true', 'false')
|
||||
WHEN NULLIF(BTRIM("usage".request_metadata->>'client_ip'), '') IS NOT NULL
|
||||
OR NULLIF(BTRIM("usage".request_metadata->>'user_agent'), '') IS NOT NULL
|
||||
OR NULLIF(BTRIM("usage".request_metadata->>'request_path'), '') IS NOT NULL
|
||||
OR NULLIF(BTRIM("usage".request_metadata->>'request_path_and_query'), '') IS NOT NULL
|
||||
OR ("usage".request_metadata->>'client_requested_stream') IN ('true', 'false')
|
||||
OR ("usage".request_metadata->>'upstream_is_stream') IN ('true', 'false')
|
||||
THEN jsonb_build_object(
|
||||
THEN jsonb_strip_nulls(jsonb_build_object(
|
||||
'client_ip',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'client_ip'), ''),
|
||||
'user_agent',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'user_agent'), ''),
|
||||
'request_path',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'request_path'), ''),
|
||||
'request_path_and_query',
|
||||
NULLIF(BTRIM("usage".request_metadata->>'request_path_and_query'), ''),
|
||||
'client_requested_stream',
|
||||
CASE
|
||||
WHEN ("usage".request_metadata->>'client_requested_stream') IN ('true', 'false')
|
||||
@@ -121,7 +137,7 @@ SELECT
|
||||
THEN ("usage".request_metadata->>'upstream_is_stream')::boolean
|
||||
ELSE NULL
|
||||
END
|
||||
)::json
|
||||
))::json
|
||||
ELSE NULL::json
|
||||
END AS request_metadata,
|
||||
NULL::varchar AS http_request_body_ref,
|
||||
|
||||
@@ -633,6 +633,14 @@ fn usage_sql_uses_json_null_placeholders_for_usage_payload_columns() {
|
||||
super::LIST_USAGE_AUDITS_PREFIX,
|
||||
super::LIST_RECENT_USAGE_AUDITS_PREFIX,
|
||||
] {
|
||||
assert!(sql.contains("jsonb_strip_nulls(jsonb_build_object("));
|
||||
assert!(sql.contains("'client_ip'"));
|
||||
assert!(sql.contains("request_metadata->>'client_ip'"));
|
||||
assert!(sql.contains("'user_agent'"));
|
||||
assert!(sql.contains("request_metadata->>'user_agent'"));
|
||||
assert!(sql.contains("AS client_family"));
|
||||
assert!(sql.contains("request_metadata->'client_session_affinity'->>'client_family'"));
|
||||
assert!(sql.contains("request_metadata->>'client_family'"));
|
||||
assert!(sql.contains("CAST(\"usage\".input_tokens AS INTEGER) AS input_tokens"));
|
||||
assert!(sql.contains(
|
||||
"usage_settlement_snapshots.billing_input_tokens AS settlement_billing_input_tokens"
|
||||
|
||||
@@ -8,7 +8,8 @@ use sqlx::{sqlite::SqliteRow, QueryBuilder, Row, Sqlite};
|
||||
|
||||
use super::{
|
||||
strip_deprecated_usage_display_fields, usage_can_recover_terminal_failure,
|
||||
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
usage_request_metadata_client_family, PendingUsageCleanupSummary,
|
||||
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
|
||||
StoredProviderApiKeyWindowUsageSummary, StoredProviderUsageSummary, StoredRequestUsageAudit,
|
||||
StoredUsageAuditAggregation, StoredUsageAuditSummary, StoredUsageBreakdownSummaryRow,
|
||||
StoredUsageCacheAffinityHitSummary, StoredUsageCacheAffinityIntervalRow,
|
||||
@@ -3812,6 +3813,8 @@ fn map_usage_row(row: &SqliteRow) -> Result<StoredRequestUsageAudit, DataLayerEr
|
||||
.map(|raw| serde_json::from_str(&raw))
|
||||
.transpose()
|
||||
.map_err(|err| DataLayerError::UnexpectedValue(err.to_string()))?;
|
||||
audit.client_family = usage_request_metadata_client_family(audit.request_metadata.as_ref())
|
||||
.map(ToOwned::to_owned);
|
||||
let upstream_is_stream = row
|
||||
.try_get::<Option<i64>, _>("upstream_is_stream")
|
||||
.map_sql_err()?
|
||||
|
||||
@@ -72,8 +72,10 @@ fn copy_allowed_metadata_fields(source: &Map<String, Value>, target: &mut Map<St
|
||||
copy_non_empty_string(source, target, "trace_id");
|
||||
copy_non_empty_string(source, target, "client_ip");
|
||||
copy_non_empty_string(source, target, "user_agent");
|
||||
copy_non_empty_string(source, target, "client_family");
|
||||
copy_bool(source, target, "client_requested_stream");
|
||||
copy_bool(source, target, "upstream_is_stream");
|
||||
copy_non_null_value(source, target, "client_session_affinity");
|
||||
copy_bool(source, target, "api_key_is_standalone");
|
||||
copy_non_empty_string(source, target, "request_path");
|
||||
copy_non_empty_string(source, target, "request_query_string");
|
||||
@@ -110,8 +112,10 @@ fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map
|
||||
remove_non_empty_string(&mut source, target, "trace_id");
|
||||
remove_non_empty_string(&mut source, target, "client_ip");
|
||||
remove_non_empty_string(&mut source, target, "user_agent");
|
||||
remove_non_empty_string(&mut source, target, "client_family");
|
||||
remove_bool(&mut source, target, "client_requested_stream");
|
||||
remove_bool(&mut source, target, "upstream_is_stream");
|
||||
remove_non_null_value(&mut source, target, "client_session_affinity");
|
||||
remove_bool(&mut source, target, "api_key_is_standalone");
|
||||
remove_non_empty_string(&mut source, target, "request_path");
|
||||
remove_non_empty_string(&mut source, target, "request_query_string");
|
||||
|
||||
Reference in New Issue
Block a user