i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长

Browse Source 
- 移除 token/latency Prometheus 指标的 model 标签，避免 provider x model 笛卡尔积
- HealthMonitor 滑动窗口从 DB JSON 迁移至进程内存，减少写放大
- ModelCostService 三层缓存增加 500 条上限，超限时清空
- StickyPriority 粘性缓存和健康状态字典增加容量淘汰
- AffinityManager 请求锁字典增加 500 条上限，淘汰空闲锁
- 配额刷新/探测查询使用 defer/load_only 避免加载大 JSON 列
- Alembic 迁移清理 DB 中遗留的 request_results_window 数据
- 同步更新测试适配 batch_get_cooldowns 返回值和批量删除异步化
This commit is contained in:
fawney19
2026-03-10 10:40:50 +08:00
parent c9f0685b40
commit 7c580e843f
20 changed files with 241 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
tests/api/test_admin_pool_batch_action_routes.py
View File

@@ -22,73 +22,67 @@ def _mock_provider_lookup(db: MagicMock, provider_id: str) -> None:
@pytest.mark.asyncio
async def test_batch_delete_uses_single_statement_for_non_sqlite(
async def test_batch_delete_submits_async_task(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Delete action now submits an async batch-delete task instead of
executing SQL synchronously."""
db = MagicMock()
provider_id = "provider-1"
_mock_provider_lookup(db, provider_id)
db.get_bind.return_value = SimpleNamespace(dialect=SimpleNamespace(name="postgresql"))
db.execute.return_value = SimpleNamespace(rowcount=1200)
side_effect = AsyncMock()
mock_submit = AsyncMock(return_value="task-abc-123")
monkeypatch.setattr(
"src.services.provider_keys.key_side_effects.run_delete_key_side_effects",
side_effect,
"src.services.provider_keys.batch_delete_task.submit_batch_delete",
mock_submit,
)
key_ids = [f"key-{idx}" for idx in range(1200)]
adapter = AdminBatchActionKeysAdapter(
provider_id=provider_id,
body=BatchActionRequest(
key_ids=[f"key-{idx}" for idx in range(1200)],
key_ids=key_ids,
action="delete",
),
)
result = await adapter.handle(_build_context(db))
assert result.affected == 1200
assert db.execute.call_count == 1
db.commit.assert_called_once()
side_effect.assert_awaited_once_with(
db=db,
provider_id=provider_id,
deleted_key_allowed_models=None,
)
# Async task returns affected=0 and a task_id
assert result.affected == 0
assert result.task_id == "task-abc-123"
assert "1200" in result.message
mock_submit.assert_awaited_once_with(provider_id, list(dict.fromkeys(key_ids)))
@pytest.mark.asyncio
async def test_batch_delete_chunks_sqlite_and_runs_side_effect_once(
async def test_batch_delete_deduplicates_key_ids(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Duplicate key IDs should be deduplicated before submission."""
db = MagicMock()
provider_id = "provider-2"
_mock_provider_lookup(db, provider_id)
db.get_bind.return_value = SimpleNamespace(dialect=SimpleNamespace(name="sqlite"))
db.execute.side_effect = [
SimpleNamespace(rowcount=900),
SimpleNamespace(rowcount=300),
]
side_effect = AsyncMock()
mock_submit = AsyncMock(return_value="task-xyz-456")
monkeypatch.setattr(
"src.services.provider_keys.key_side_effects.run_delete_key_side_effects",
side_effect,
"src.services.provider_keys.batch_delete_task.submit_batch_delete",
mock_submit,
)
adapter = AdminBatchActionKeysAdapter(
provider_id=provider_id,
body=BatchActionRequest(
key_ids=[f"key-{idx}" for idx in range(1200)],
key_ids=["key-1", "key-2", "key-1", "key-3", "key-2"],
action="delete",
),
)
result = await adapter.handle(_build_context(db))
assert result.affected == 1200
assert db.execute.call_count == 2
db.commit.assert_called_once()
side_effect.assert_awaited_once_with(
db=db,
provider_id=provider_id,
deleted_key_allowed_models=None,
)
assert result.affected == 0
assert result.task_id == "task-xyz-456"
# Deduplicated: 3 unique keys
submitted_ids = mock_submit.call_args[0][1]
assert len(submitted_ids) == 3
assert submitted_ids == ["key-1", "key-2", "key-3"]

19
tests/services/test_claude_code_runtime_controls.py
View File

@@ -75,7 +75,12 @@ def test_get_ssl_context_for_unknown_profile_falls_back_to_default() -> None:
assert get_ssl_context_for_profile("unknown_profile") is get_ssl_context()
def test_wrap_request_masks_session_id_when_enabled() -> None:
def test_wrap_request_masks_session_id_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
# Force local session control mode (avoid interference from Redis initialized by other tests)
monkeypatch.setattr(
"src.services.provider.adapters.claude_code.envelope._is_distributed_session_control_available",
lambda: False,
)
scope_key = f"key:test-mask-{uuid.uuid4()}"
set_claude_code_request_context(
ClaudeCodeRequestContext(
@@ -117,7 +122,12 @@ def test_wrap_request_masks_session_id_when_enabled() -> None:
assert tail1 == tail2
def test_wrap_request_enforces_max_sessions() -> None:
def test_wrap_request_enforces_max_sessions(monkeypatch: pytest.MonkeyPatch) -> None:
# Force local session control mode (avoid interference from Redis initialized by other tests)
monkeypatch.setattr(
"src.services.provider.adapters.claude_code.envelope._is_distributed_session_control_available",
lambda: False,
)
scope_key = f"key:test-limit-{uuid.uuid4()}"
set_claude_code_request_context(
ClaudeCodeRequestContext(
@@ -153,6 +163,11 @@ def test_wrap_request_enforces_max_sessions() -> None:
def test_wrap_request_releases_expired_sessions(monkeypatch: pytest.MonkeyPatch) -> None:
# Force local session control mode
monkeypatch.setattr(
"src.services.provider.adapters.claude_code.envelope._is_distributed_session_control_available",
lambda: False,
)
scope_key = f"key:test-expire-{uuid.uuid4()}"
set_claude_code_request_context(
ClaudeCodeRequestContext(

5
tests/services/test_global_model_auto_disassociate.py
View File

@@ -45,9 +45,10 @@ def test_auto_disassociate_deletes_unmatched_auto_associated_models(
unlimited_query.filter.return_value.limit.return_value.first.return_value = None
allowed_models_query = MagicMock()
# db.query(ProviderAPIKey.allowed_models).all() returns list of tuples
allowed_models_query.filter.return_value.all.return_value = [
SimpleNamespace(allowed_models=["gpt-4o"]),
SimpleNamespace(allowed_models=[]),
(["gpt-4o"],),
([],),
]
model = SimpleNamespace(

13
tests/services/test_pool_config.py
View File

@@ -22,12 +22,12 @@ def test_parse_pool_config_returns_defaults_for_empty_advanced() -> None:
assert cfg is not None
assert cfg.sticky_session_ttl_seconds == 3600
assert cfg.load_threshold_percent == 80
assert cfg.lru_enabled is True
assert cfg.scheduling_mode == "lru"
assert cfg.lru_enabled is False
assert cfg.scheduling_mode == "multi_score"
assert cfg.scoring_weights == ScoringWeights()
# Default: only LRU preset enabled
# Default: only cache_affinity preset enabled
assert len(cfg.scheduling_presets) == 1
assert cfg.scheduling_presets[0].preset == "lru"
assert cfg.scheduling_presets[0].preset == "cache_affinity"
assert cfg.scheduling_presets[0].enabled is True
assert cfg.latency_window_seconds == 3600
assert cfg.latency_sample_limit == 50
@@ -263,10 +263,11 @@ def test_parse_pool_config_handles_invalid_types_gracefully() -> None:
assert cfg.cost_limit_per_key_tokens is None # default for opt_int
def test_parse_pool_config_invalid_scheduling_mode_falls_back_to_lru() -> None:
def test_parse_pool_config_invalid_scheduling_mode_falls_back_to_cache_affinity() -> None:
cfg = parse_pool_config({"pool_advanced": {"scheduling_mode": "unknown"}})
assert cfg is not None
assert cfg.scheduling_mode == "lru"
# Default with no explicit presets: cache_affinity -> multi_score
assert cfg.scheduling_mode == "multi_score"
def test_parse_pool_config_scoring_weights_invalid_values_are_clamped() -> None:

3
tests/services/test_pool_config_backward_compat.py
View File

@@ -34,7 +34,8 @@ def test_empty_pool_advanced_returns_defaults() -> None:
assert cfg is not None
defaults = PoolConfig()
assert cfg.sticky_session_ttl_seconds == defaults.sticky_session_ttl_seconds
assert cfg.lru_enabled is True
# Default with empty pool_advanced: cache_affinity preset, no LRU
assert cfg.lru_enabled is False
def test_shim_imports_resolve() -> None:

21
tests/services/test_pool_manager_trace.py
View File

@@ -40,7 +40,7 @@ async def test_trace_attached_to_first_candidate() -> None:
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None), "key-2": (None, None)},
return_value={"key-1": None, "key-2": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",
@@ -72,7 +72,7 @@ async def test_pool_extra_data_on_selected_candidate() -> None:
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None)},
return_value={"key-1": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",
@@ -103,8 +103,8 @@ async def test_pool_extra_data_on_skipped_candidate() -> None:
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={
"key-1": ("rate_limited_429", 120),
"key-2": (None, None),
"key-1": "rate_limited_429",
"key-2": None,
},
),
patch(
@@ -122,7 +122,8 @@ async def test_pool_extra_data_on_skipped_candidate() -> None:
assert extra is not None
assert extra["pool_skip"]["type"] == "cooldown"
assert extra["pool_skip"]["cooldown_reason"] == "rate_limited_429"
assert extra["pool_skip"]["cooldown_ttl"] == 120
# TTL is not fetched on the scheduling hot path (include_ttl=False)
assert "cooldown_ttl" not in extra["pool_skip"]
@pytest.mark.asyncio
@@ -142,9 +143,9 @@ async def test_trace_build_summary_matches() -> None:
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={
"key-1": ("overloaded_529", 60),
"key-2": (None, None),
"key-3": (None, None),
"key-1": "overloaded_529",
"key-2": None,
"key-3": None,
},
),
patch(
@@ -181,7 +182,7 @@ async def test_trace_build_summary_uses_attempted_key_ids_when_provided() -> Non
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None), "key-2": (None, None)},
return_value={"key-1": None, "key-2": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",
@@ -218,7 +219,7 @@ async def test_sticky_trace_info() -> None:
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None), "key-2": (None, None)},
return_value={"key-1": None, "key-2": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",

2
tests/services/test_pool_multi_score_strategy.py
View File

@@ -34,6 +34,7 @@ def test_multi_score_prefers_low_latency_when_latency_weight_is_high() -> None:
strategy = MultiScoreStrategy()
cfg = PoolConfig(
scheduling_mode="multi_score",
scheduling_presets=(),
scoring_weights=ScoringWeights(lru=0.0, latency=1.0, health=0.0, cost_remaining=0.0),
)
ctx = _context()
@@ -48,6 +49,7 @@ def test_multi_score_combines_health_and_cost() -> None:
strategy = MultiScoreStrategy()
cfg = PoolConfig(
scheduling_mode="multi_score",
scheduling_presets=(),
scoring_weights=ScoringWeights(lru=0.0, latency=0.0, health=0.5, cost_remaining=0.5),
cost_limit_per_key_tokens=1000,
)

3
tests/services/test_provider_keys_quota_service.py
View File

@@ -35,6 +35,9 @@ class _FakeQuery:
self._first_result = first_result
self._all_result = all_result or []
def options(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return self
def filter(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return self

4
tests/unit/test_admin_endpoint_create_defaults.py
View File

@@ -53,7 +53,7 @@ async def test_create_endpoint_injects_default_body_rules_when_missing(
monkeypatch.setattr(
routes,
"get_default_body_rules_for_endpoint",
lambda _fmt: [{"action": "drop", "path": "max_output_tokens"}],
lambda _fmt, **_kw: [{"action": "drop", "path": "max_output_tokens"}],
)
db = _FakeDB(
@@ -81,7 +81,7 @@ async def test_create_endpoint_keeps_user_body_rules_when_provided(
monkeypatch.setattr(
routes,
"get_default_body_rules_for_endpoint",
lambda _fmt: [{"action": "drop", "path": "max_output_tokens"}],
lambda _fmt, **_kw: [{"action": "drop", "path": "max_output_tokens"}],
)
user_rules = [{"action": "set", "path": "metadata.source", "value": "user"}]

2
tests/unit/test_admin_system_users_export_import.py
View File

@@ -12,8 +12,6 @@ def test_export_user_api_key_prefers_plaintext_key() -> None:
key_encrypted=crypto_service.encrypt(plaintext_key),
name="Demo Key",
is_standalone=False,
balance_used_usd=1.5,
current_balance_usd=8.5,
is_active=True,
)

3
tests/unit/test_api_format_metadata.py
View File

@@ -27,8 +27,9 @@ def test_get_default_body_rules_for_endpoint_returns_codex_cli_rules_only() -> N
assert cli_rules[4]["path"] == "instructions"
assert cli_rules[4]["condition"]["op"] == "not_exists"
# openai:compact also has the same default body rules (defined in EndpointDefinition)
compact_rules = get_default_body_rules_for_endpoint("openai:compact", provider_type="codex")
assert compact_rules == []
assert len(compact_rules) == 5
def test_get_default_body_rules_for_endpoint_returns_deep_copy(