Normalize canonical API formats

This commit is contained in:
fawney19
2026-04-29 09:25:19 +08:00
parent 02ad67fe33
commit 07a319259b
171 changed files with 2460 additions and 1962 deletions

View File

@@ -0,0 +1,361 @@
CREATE OR REPLACE FUNCTION public.aether_canonical_api_format_alias(value text)
RETURNS text
LANGUAGE sql
IMMUTABLE
AS $$
SELECT CASE LOWER(BTRIM(COALESCE(value, '')))
WHEN 'openai:cli' THEN 'openai:responses'
WHEN 'openai:compact' THEN 'openai:responses:compact'
WHEN 'claude:chat' THEN 'claude:messages'
WHEN 'claude:cli' THEN 'claude:messages'
WHEN 'gemini:chat' THEN 'gemini:generate_content'
WHEN 'gemini:cli' THEN 'gemini:generate_content'
ELSE LOWER(BTRIM(COALESCE(value, '')))
END
$$;
DO $$
DECLARE
conflict_summary text;
BEGIN
SELECT string_agg(
DISTINCT provider_id::text || ':' || canonical_api_format,
', ' ORDER BY provider_id::text || ':' || canonical_api_format
)
INTO conflict_summary
FROM (
SELECT
left_endpoint.provider_id,
public.aether_canonical_api_format_alias(left_endpoint.api_format) AS canonical_api_format
FROM public.provider_endpoints AS left_endpoint
INNER JOIN public.provider_endpoints AS right_endpoint
ON right_endpoint.provider_id = left_endpoint.provider_id
AND right_endpoint.id > left_endpoint.id
AND public.aether_canonical_api_format_alias(right_endpoint.api_format)
= public.aether_canonical_api_format_alias(left_endpoint.api_format)
WHERE left_endpoint.api_format IN ('openai:responses', 'openai:cli', 'openai:responses:compact', 'openai:compact', 'claude:messages', 'claude:chat', 'claude:cli', 'gemini:generate_content', 'gemini:chat', 'gemini:cli')
AND right_endpoint.api_format IN ('openai:responses', 'openai:cli', 'openai:responses:compact', 'openai:compact', 'claude:messages', 'claude:chat', 'claude:cli', 'gemini:generate_content', 'gemini:chat', 'gemini:cli')
AND (
left_endpoint.base_url IS DISTINCT FROM right_endpoint.base_url
OR left_endpoint.custom_path IS DISTINCT FROM right_endpoint.custom_path
OR left_endpoint.max_retries IS DISTINCT FROM right_endpoint.max_retries
OR left_endpoint.header_rules::jsonb IS DISTINCT FROM right_endpoint.header_rules::jsonb
OR left_endpoint.body_rules::jsonb IS DISTINCT FROM right_endpoint.body_rules::jsonb
OR left_endpoint.config::jsonb IS DISTINCT FROM right_endpoint.config::jsonb
OR left_endpoint.proxy IS DISTINCT FROM right_endpoint.proxy
OR left_endpoint.format_acceptance_config::jsonb IS DISTINCT FROM right_endpoint.format_acceptance_config::jsonb
)
) AS conflicts;
IF conflict_summary IS NOT NULL THEN
RAISE EXCEPTION
'Cannot normalize OpenAI/Claude/Gemini provider_endpoints because transport fields differ for: %',
conflict_summary;
END IF;
END $$;
WITH grouped AS (
SELECT
id,
provider_id,
api_format,
public.aether_canonical_api_format_alias(api_format) AS canonical_api_format,
ROW_NUMBER() OVER (
PARTITION BY provider_id, public.aether_canonical_api_format_alias(api_format)
ORDER BY
CASE
WHEN api_format = public.aether_canonical_api_format_alias(api_format) THEN 0
ELSE 1
END,
created_at ASC,
id ASC
) AS rank
FROM public.provider_endpoints
WHERE api_format IN ('openai:responses', 'openai:cli', 'openai:responses:compact', 'openai:compact', 'claude:messages', 'claude:chat', 'claude:cli', 'gemini:generate_content', 'gemini:chat', 'gemini:cli')
),
survivors AS (
SELECT *
FROM grouped
WHERE rank = 1
),
retired AS (
UPDATE public.provider_endpoints AS endpoint
SET
is_active = FALSE,
updated_at = NOW()
FROM grouped
WHERE endpoint.id = grouped.id
AND grouped.rank > 1
RETURNING endpoint.id
)
UPDATE public.provider_endpoints AS endpoint
SET
api_format = survivors.canonical_api_format,
api_family = SPLIT_PART(survivors.canonical_api_format, ':', 1),
endpoint_kind = SUBSTRING(survivors.canonical_api_format FROM POSITION(':' IN survivors.canonical_api_format) + 1),
updated_at = NOW()
FROM survivors
WHERE endpoint.id = survivors.id
AND endpoint.api_format IS DISTINCT FROM survivors.canonical_api_format;
WITH expanded AS (
SELECT
pak.id,
formats.ordinality,
public.aether_canonical_api_format_alias(formats.value) AS api_format
FROM public.provider_api_keys AS pak
CROSS JOIN LATERAL json_array_elements_text(
CASE
WHEN json_typeof(pak.api_formats) = 'array' THEN pak.api_formats
ELSE '[]'::json
END
) WITH ORDINALITY AS formats(value, ordinality)
WHERE pak.api_formats IS NOT NULL
),
deduped AS (
SELECT id, api_format, MIN(ordinality) AS first_ordinality
FROM expanded
WHERE api_format <> ''
GROUP BY id, api_format
),
rebuilt AS (
SELECT id, json_agg(api_format ORDER BY first_ordinality) AS api_formats
FROM deduped
GROUP BY id
)
UPDATE public.provider_api_keys AS pak
SET
api_formats = rebuilt.api_formats,
updated_at = NOW()
FROM rebuilt
WHERE pak.id = rebuilt.id
AND pak.api_formats::jsonb IS DISTINCT FROM rebuilt.api_formats::jsonb;
WITH expanded AS (
SELECT
key.id,
formats.ordinality,
public.aether_canonical_api_format_alias(formats.value) AS api_format
FROM public.api_keys AS key
CROSS JOIN LATERAL json_array_elements_text(
CASE
WHEN json_typeof(key.allowed_api_formats) = 'array' THEN key.allowed_api_formats
ELSE '[]'::json
END
) WITH ORDINALITY AS formats(value, ordinality)
WHERE key.allowed_api_formats IS NOT NULL
),
deduped AS (
SELECT id, api_format, MIN(ordinality) AS first_ordinality
FROM expanded
WHERE api_format <> ''
GROUP BY id, api_format
),
rebuilt AS (
SELECT id, json_agg(api_format ORDER BY first_ordinality) AS allowed_api_formats
FROM deduped
GROUP BY id
)
UPDATE public.api_keys AS key
SET
allowed_api_formats = rebuilt.allowed_api_formats,
updated_at = NOW()
FROM rebuilt
WHERE key.id = rebuilt.id
AND key.allowed_api_formats::jsonb IS DISTINCT FROM rebuilt.allowed_api_formats::jsonb;
WITH expanded AS (
SELECT
users.id,
formats.ordinality,
public.aether_canonical_api_format_alias(formats.value) AS api_format
FROM public.users AS users
CROSS JOIN LATERAL json_array_elements_text(
CASE
WHEN json_typeof(users.allowed_api_formats) = 'array' THEN users.allowed_api_formats
ELSE '[]'::json
END
) WITH ORDINALITY AS formats(value, ordinality)
WHERE users.allowed_api_formats IS NOT NULL
),
deduped AS (
SELECT id, api_format, MIN(ordinality) AS first_ordinality
FROM expanded
WHERE api_format <> ''
GROUP BY id, api_format
),
rebuilt AS (
SELECT id, json_agg(api_format ORDER BY first_ordinality) AS allowed_api_formats
FROM deduped
GROUP BY id
)
UPDATE public.users AS users
SET
allowed_api_formats = rebuilt.allowed_api_formats,
updated_at = NOW()
FROM rebuilt
WHERE users.id = rebuilt.id
AND users.allowed_api_formats::jsonb IS DISTINCT FROM rebuilt.allowed_api_formats::jsonb;
WITH mapping_items AS (
SELECT
models.id,
item.ordinality AS item_ordinality,
item.value AS item
FROM public.models AS models
CROSS JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(models.provider_model_mappings) = 'array' THEN models.provider_model_mappings
ELSE '[]'::jsonb
END
) WITH ORDINALITY AS item(value, ordinality)
WHERE models.provider_model_mappings IS NOT NULL
),
rebuilt_items AS (
SELECT
id,
item_ordinality,
CASE
WHEN jsonb_typeof(item) = 'object'
AND jsonb_typeof(item->'api_formats') = 'array'
THEN jsonb_set(
item,
'{api_formats}',
COALESCE(
(
SELECT jsonb_agg(api_format ORDER BY first_ordinality)
FROM (
SELECT
public.aether_canonical_api_format_alias(format.value) AS api_format,
MIN(format.ordinality) AS first_ordinality
FROM jsonb_array_elements_text(item->'api_formats') WITH ORDINALITY AS format(value, ordinality)
GROUP BY public.aether_canonical_api_format_alias(format.value)
) AS deduped_formats
WHERE api_format <> ''
),
'[]'::jsonb
),
true
)
ELSE item
END AS item
FROM mapping_items
),
rebuilt AS (
SELECT id, jsonb_agg(item ORDER BY item_ordinality) AS provider_model_mappings
FROM rebuilt_items
GROUP BY id
)
UPDATE public.models AS models
SET
provider_model_mappings = rebuilt.provider_model_mappings,
updated_at = NOW()
FROM rebuilt
WHERE models.id = rebuilt.id
AND models.provider_model_mappings IS DISTINCT FROM rebuilt.provider_model_mappings;
WITH rebuilt AS (
SELECT
pak.id,
jsonb_object_agg(
public.aether_canonical_api_format_alias(entry.key),
entry.value
ORDER BY entry.ordinality
) AS rate_multipliers
FROM public.provider_api_keys AS pak
CROSS JOIN LATERAL json_each(
CASE
WHEN json_typeof(pak.rate_multipliers) = 'object' THEN pak.rate_multipliers
ELSE '{}'::json
END
) WITH ORDINALITY AS entry(key, value, ordinality)
WHERE pak.rate_multipliers IS NOT NULL
GROUP BY pak.id
)
UPDATE public.provider_api_keys AS pak
SET
rate_multipliers = rebuilt.rate_multipliers::json,
updated_at = NOW()
FROM rebuilt
WHERE pak.id = rebuilt.id
AND pak.rate_multipliers::jsonb IS DISTINCT FROM rebuilt.rate_multipliers;
WITH rebuilt AS (
SELECT
pak.id,
jsonb_object_agg(
public.aether_canonical_api_format_alias(entry.key),
entry.value
ORDER BY entry.ordinality
) AS global_priority_by_format
FROM public.provider_api_keys AS pak
CROSS JOIN LATERAL json_each(
CASE
WHEN json_typeof(pak.global_priority_by_format) = 'object' THEN pak.global_priority_by_format
ELSE '{}'::json
END
) WITH ORDINALITY AS entry(key, value, ordinality)
WHERE pak.global_priority_by_format IS NOT NULL
GROUP BY pak.id
)
UPDATE public.provider_api_keys AS pak
SET
global_priority_by_format = rebuilt.global_priority_by_format::json,
updated_at = NOW()
FROM rebuilt
WHERE pak.id = rebuilt.id
AND pak.global_priority_by_format::jsonb IS DISTINCT FROM rebuilt.global_priority_by_format;
WITH rebuilt AS (
SELECT
pak.id,
jsonb_object_agg(
public.aether_canonical_api_format_alias(entry.key),
entry.value
ORDER BY entry.ordinality
) AS health_by_format
FROM public.provider_api_keys AS pak
CROSS JOIN LATERAL jsonb_each(
CASE
WHEN jsonb_typeof(pak.health_by_format) = 'object' THEN pak.health_by_format
ELSE '{}'::jsonb
END
) WITH ORDINALITY AS entry(key, value, ordinality)
WHERE pak.health_by_format IS NOT NULL
GROUP BY pak.id
)
UPDATE public.provider_api_keys AS pak
SET
health_by_format = rebuilt.health_by_format,
updated_at = NOW()
FROM rebuilt
WHERE pak.id = rebuilt.id
AND pak.health_by_format IS DISTINCT FROM rebuilt.health_by_format;
WITH rebuilt AS (
SELECT
pak.id,
jsonb_object_agg(
public.aether_canonical_api_format_alias(entry.key),
entry.value
ORDER BY entry.ordinality
) AS circuit_breaker_by_format
FROM public.provider_api_keys AS pak
CROSS JOIN LATERAL jsonb_each(
CASE
WHEN jsonb_typeof(pak.circuit_breaker_by_format) = 'object' THEN pak.circuit_breaker_by_format
ELSE '{}'::jsonb
END
) WITH ORDINALITY AS entry(key, value, ordinality)
WHERE pak.circuit_breaker_by_format IS NOT NULL
GROUP BY pak.id
)
UPDATE public.provider_api_keys AS pak
SET
circuit_breaker_by_format = rebuilt.circuit_breaker_by_format,
updated_at = NOW()
FROM rebuilt
WHERE pak.id = rebuilt.id
AND pak.circuit_breaker_by_format IS DISTINCT FROM rebuilt.circuit_breaker_by_format;
DROP FUNCTION public.aether_canonical_api_format_alias(text);

View File

@@ -8,7 +8,7 @@ use tracing::{error, info, warn};
static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
static BASELINE_V2_SQL: &str = include_str!("../bootstrap/20260413020000_baseline_v2.sql");
const BASELINE_V2_CUTOFF_VERSION: i64 = 20260424000000;
const BASELINE_V2_CUTOFF_VERSION: i64 = 20260428000000;
const MIGRATIONS_TABLE_EXISTS_SQL: &str =
"SELECT to_regclass('public._sqlx_migrations') IS NOT NULL";
const PUBLIC_BASE_TABLE_COUNT_SQL: &str = r#"
@@ -664,6 +664,7 @@ SELECT EXISTS (
20260422120000,
20260423000000,
20260424000000,
20260428000000,
]
);
}
@@ -816,6 +817,7 @@ SELECT EXISTS (
20260422120000,
20260423000000,
20260424000000,
20260428000000,
]
);
}

View File

@@ -913,7 +913,7 @@ mod tests {
None,
None,
Some(serde_json::json!(["anthropic"])),
Some(serde_json::json!(["claude:chat"])),
Some(serde_json::json!(["claude:messages"])),
Some(serde_json::json!(["claude-sonnet-4-5"])),
)
.expect("snapshot should build");
@@ -926,7 +926,7 @@ mod tests {
);
assert_eq!(
resolved.effective_allowed_api_formats(),
Some(&["claude:chat".to_string()][..])
Some(&["claude:messages".to_string()][..])
);
assert_eq!(
resolved.effective_allowed_models(),

View File

@@ -41,6 +41,7 @@ impl MinimalCandidateSelectionReadRepository for InMemoryMinimalCandidateSelecti
&& row.model_is_available
&& api_format_matches(&row.endpoint_api_format, api_format)
&& row.key_supports_api_format(api_format)
&& key_auth_channel_matches(row, api_format)
})
.cloned()
.collect::<Vec<_>>();
@@ -70,11 +71,42 @@ impl MinimalCandidateSelectionReadRepository for InMemoryMinimalCandidateSelecti
}
fn normalize_api_format(value: &str) -> String {
aether_ai_formats::normalize_legacy_openai_format_alias(value)
aether_ai_formats::normalize_api_format_alias(value)
}
fn api_format_matches(left: &str, right: &str) -> bool {
normalize_api_format(left) == normalize_api_format(right)
aether_ai_formats::api_format_alias_matches(left, right)
}
fn key_auth_channel_matches(row: &StoredMinimalCandidateSelectionRow, api_format: &str) -> bool {
let provider_type = row.provider_type.trim().to_ascii_lowercase();
let auth_type = row.key_auth_type.trim().to_ascii_lowercase();
let api_format = normalize_api_format(api_format);
match provider_type.as_str() {
"codex" => {
auth_type == "oauth"
&& matches!(
api_format.as_str(),
"openai:responses" | "openai:responses:compact" | "openai:image"
)
}
"claude_code" => auth_type == "oauth" && api_format == "claude:messages",
"kiro" => {
matches!(auth_type.as_str(), "oauth" | "bearer") && api_format == "claude:messages"
}
"gemini_cli" | "antigravity" => {
auth_type == "oauth" && api_format == "gemini:generate_content"
}
"vertex_ai" => {
(auth_type == "api_key" && api_format == "gemini:generate_content")
|| (matches!(auth_type.as_str(), "service_account" | "vertex_ai")
&& matches!(
api_format.as_str(),
"claude:messages" | "gemini:generate_content"
))
}
_ => auth_type != "oauth",
}
}
#[cfg(test)]

View File

@@ -65,18 +65,26 @@ WHERE p.is_active = TRUE
AND LOWER(pe.api_format) = LOWER($1)
AND (
pak.api_formats IS NULL
OR (
LOWER(BTRIM(p.provider_type)) IN (
'claude_code',
'codex',
'gemini_cli',
'vertex_ai',
'antigravity'
)
OR EXISTS (
SELECT 1
FROM json_array_elements_text(pak.api_formats) AS fmt(value)
WHERE LOWER(BTRIM(fmt.value)) = ANY($2::text[])
)
)
AND (
(
LOWER(BTRIM(p.provider_type)) = 'codex'
AND LOWER(BTRIM(pak.auth_type)) = 'oauth'
AND LOWER($3) IN ('openai:responses', 'openai:responses:compact', 'openai:image')
)
OR (
LOWER(BTRIM(p.provider_type)) = 'claude_code'
AND LOWER(BTRIM(pak.auth_type)) = 'oauth'
AND LOWER($3) = 'claude:messages'
)
OR (
LOWER(BTRIM(p.provider_type)) = 'kiro'
AND LOWER($3) = 'claude:messages'
AND (
LOWER(BTRIM(pak.auth_type)) = 'oauth'
OR (
@@ -86,10 +94,34 @@ WHERE p.is_active = TRUE
)
)
)
OR EXISTS (
SELECT 1
FROM json_array_elements_text(pak.api_formats) AS fmt(value)
WHERE LOWER(fmt.value) = LOWER($1)
OR (
LOWER(BTRIM(p.provider_type)) IN ('gemini_cli', 'antigravity')
AND LOWER(BTRIM(pak.auth_type)) = 'oauth'
AND LOWER($3) = 'gemini:generate_content'
)
OR (
LOWER(BTRIM(p.provider_type)) = 'vertex_ai'
AND (
(
LOWER(BTRIM(pak.auth_type)) = 'api_key'
AND LOWER($3) = 'gemini:generate_content'
)
OR (
LOWER(BTRIM(pak.auth_type)) IN ('service_account', 'vertex_ai')
AND LOWER($3) IN ('claude:messages', 'gemini:generate_content')
)
)
)
OR (
LOWER(BTRIM(p.provider_type)) NOT IN (
'claude_code',
'codex',
'gemini_cli',
'vertex_ai',
'antigravity',
'kiro'
)
AND LOWER(BTRIM(pak.auth_type)) <> 'oauth'
)
)
ORDER BY
@@ -159,18 +191,26 @@ WHERE p.is_active = TRUE
AND gm.name = $2
AND (
pak.api_formats IS NULL
OR (
LOWER(BTRIM(p.provider_type)) IN (
'claude_code',
'codex',
'gemini_cli',
'vertex_ai',
'antigravity'
)
OR EXISTS (
SELECT 1
FROM json_array_elements_text(pak.api_formats) AS fmt(value)
WHERE LOWER(BTRIM(fmt.value)) = ANY($3::text[])
)
)
AND (
(
LOWER(BTRIM(p.provider_type)) = 'codex'
AND LOWER(BTRIM(pak.auth_type)) = 'oauth'
AND LOWER($4) IN ('openai:responses', 'openai:responses:compact', 'openai:image')
)
OR (
LOWER(BTRIM(p.provider_type)) = 'claude_code'
AND LOWER(BTRIM(pak.auth_type)) = 'oauth'
AND LOWER($4) = 'claude:messages'
)
OR (
LOWER(BTRIM(p.provider_type)) = 'kiro'
AND LOWER($4) = 'claude:messages'
AND (
LOWER(BTRIM(pak.auth_type)) = 'oauth'
OR (
@@ -180,10 +220,34 @@ WHERE p.is_active = TRUE
)
)
)
OR EXISTS (
SELECT 1
FROM json_array_elements_text(pak.api_formats) AS fmt(value)
WHERE LOWER(fmt.value) = LOWER($1)
OR (
LOWER(BTRIM(p.provider_type)) IN ('gemini_cli', 'antigravity')
AND LOWER(BTRIM(pak.auth_type)) = 'oauth'
AND LOWER($4) = 'gemini:generate_content'
)
OR (
LOWER(BTRIM(p.provider_type)) = 'vertex_ai'
AND (
(
LOWER(BTRIM(pak.auth_type)) = 'api_key'
AND LOWER($4) = 'gemini:generate_content'
)
OR (
LOWER(BTRIM(pak.auth_type)) IN ('service_account', 'vertex_ai')
AND LOWER($4) IN ('claude:messages', 'gemini:generate_content')
)
)
)
OR (
LOWER(BTRIM(p.provider_type)) NOT IN (
'claude_code',
'codex',
'gemini_cli',
'vertex_ai',
'antigravity',
'kiro'
)
AND LOWER(BTRIM(pak.auth_type)) <> 'oauth'
)
)
ORDER BY
@@ -228,11 +292,16 @@ impl SqlxMinimalCandidateSelectionReadRepository {
api_format: &str,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
let mut rows = Vec::new();
for api_format in api_format_aliases(api_format) {
let canonical_api_format = normalize_api_format(api_format);
let storage_aliases = api_format_aliases(&canonical_api_format);
let sql_match_aliases = sql_match_aliases(&storage_aliases);
for api_format in storage_aliases {
rows.extend(
Self::collect_query_rows(
sqlx::query(LIST_FOR_EXACT_API_FORMAT_SQL)
.bind(api_format)
.bind(sql_match_aliases.clone())
.bind(canonical_api_format.clone())
.fetch(&self.pool),
map_candidate_selection_row,
)
@@ -248,12 +317,17 @@ impl SqlxMinimalCandidateSelectionReadRepository {
global_model_name: &str,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
let mut rows = Vec::new();
for api_format in api_format_aliases(api_format) {
let canonical_api_format = normalize_api_format(api_format);
let storage_aliases = api_format_aliases(&canonical_api_format);
let sql_match_aliases = sql_match_aliases(&storage_aliases);
for api_format in storage_aliases {
rows.extend(
Self::collect_query_rows(
sqlx::query(LIST_FOR_EXACT_API_FORMAT_AND_GLOBAL_MODEL_SQL)
.bind(api_format)
.bind(global_model_name)
.bind(sql_match_aliases.clone())
.bind(canonical_api_format.clone())
.fetch(&self.pool),
map_candidate_selection_row,
)
@@ -265,7 +339,18 @@ impl SqlxMinimalCandidateSelectionReadRepository {
}
fn api_format_aliases(api_format: &str) -> Vec<String> {
aether_ai_formats::openai_format_storage_aliases(api_format)
aether_ai_formats::api_format_storage_aliases(api_format)
}
fn normalize_api_format(api_format: &str) -> String {
aether_ai_formats::normalize_api_format_alias(api_format)
}
fn sql_match_aliases(api_formats: &[String]) -> Vec<String> {
api_formats
.iter()
.map(|value| value.trim().to_ascii_lowercase())
.collect()
}
fn dedupe_candidate_selection_rows(
@@ -521,7 +606,13 @@ fn parse_provider_model_mapping_object_lenient(
let api_formats = parse_string_list(
object.get("api_formats").cloned(),
"models.provider_model_mappings.api_formats",
)?;
)?
.map(|formats| {
formats
.into_iter()
.map(|value| aether_ai_formats::normalize_api_format_alias(&value))
.collect()
});
Ok(Some(StoredProviderModelMapping {
name: name.to_string(),