i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长

Browse Source 
- 移除 token/latency Prometheus 指标的 model 标签，避免 provider x model 笛卡尔积
- HealthMonitor 滑动窗口从 DB JSON 迁移至进程内存，减少写放大
- ModelCostService 三层缓存增加 500 条上限，超限时清空
- StickyPriority 粘性缓存和健康状态字典增加容量淘汰
- AffinityManager 请求锁字典增加 500 条上限，淘汰空闲锁
- 配额刷新/探测查询使用 defer/load_only 避免加载大 JSON 列
- Alembic 迁移清理 DB 中遗留的 request_results_window 数据
- 同步更新测试适配 batch_get_cooldowns 返回值和批量删除异步化
This commit is contained in:
fawney19
2026-03-10 10:40:50 +08:00
parent c9f0685b40
commit 7c580e843f
20 changed files with 241 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tests/services/test_claude_code_runtime_controls.py
View File

@@ -75,7 +75,12 @@ def test_get_ssl_context_for_unknown_profile_falls_back_to_default() -> None:
assert get_ssl_context_for_profile("unknown_profile") is get_ssl_context()
def test_wrap_request_masks_session_id_when_enabled() -> None:
def test_wrap_request_masks_session_id_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
# Force local session control mode (avoid interference from Redis initialized by other tests)
monkeypatch.setattr(
"src.services.provider.adapters.claude_code.envelope._is_distributed_session_control_available",
lambda: False,
)
scope_key = f"key:test-mask-{uuid.uuid4()}"
set_claude_code_request_context(
ClaudeCodeRequestContext(
@@ -117,7 +122,12 @@ def test_wrap_request_masks_session_id_when_enabled() -> None:
assert tail1 == tail2
def test_wrap_request_enforces_max_sessions() -> None:
def test_wrap_request_enforces_max_sessions(monkeypatch: pytest.MonkeyPatch) -> None:
# Force local session control mode (avoid interference from Redis initialized by other tests)
monkeypatch.setattr(
"src.services.provider.adapters.claude_code.envelope._is_distributed_session_control_available",
lambda: False,
)
scope_key = f"key:test-limit-{uuid.uuid4()}"
set_claude_code_request_context(
ClaudeCodeRequestContext(
@@ -153,6 +163,11 @@ def test_wrap_request_enforces_max_sessions() -> None:
def test_wrap_request_releases_expired_sessions(monkeypatch: pytest.MonkeyPatch) -> None:
# Force local session control mode
monkeypatch.setattr(
"src.services.provider.adapters.claude_code.envelope._is_distributed_session_control_available",
lambda: False,
)
scope_key = f"key:test-expire-{uuid.uuid4()}"
set_claude_code_request_context(
ClaudeCodeRequestContext(

5
tests/services/test_global_model_auto_disassociate.py
View File

@@ -45,9 +45,10 @@ def test_auto_disassociate_deletes_unmatched_auto_associated_models(
unlimited_query.filter.return_value.limit.return_value.first.return_value = None
allowed_models_query = MagicMock()
# db.query(ProviderAPIKey.allowed_models).all() returns list of tuples
allowed_models_query.filter.return_value.all.return_value = [
SimpleNamespace(allowed_models=["gpt-4o"]),
SimpleNamespace(allowed_models=[]),
(["gpt-4o"],),
([],),
]
model = SimpleNamespace(

13
tests/services/test_pool_config.py
View File

@@ -22,12 +22,12 @@ def test_parse_pool_config_returns_defaults_for_empty_advanced() -> None:
assert cfg is not None
assert cfg.sticky_session_ttl_seconds == 3600
assert cfg.load_threshold_percent == 80
assert cfg.lru_enabled is True
assert cfg.scheduling_mode == "lru"
assert cfg.lru_enabled is False
assert cfg.scheduling_mode == "multi_score"
assert cfg.scoring_weights == ScoringWeights()
# Default: only LRU preset enabled
# Default: only cache_affinity preset enabled
assert len(cfg.scheduling_presets) == 1
assert cfg.scheduling_presets[0].preset == "lru"
assert cfg.scheduling_presets[0].preset == "cache_affinity"
assert cfg.scheduling_presets[0].enabled is True
assert cfg.latency_window_seconds == 3600
assert cfg.latency_sample_limit == 50
@@ -263,10 +263,11 @@ def test_parse_pool_config_handles_invalid_types_gracefully() -> None:
assert cfg.cost_limit_per_key_tokens is None # default for opt_int
def test_parse_pool_config_invalid_scheduling_mode_falls_back_to_lru() -> None:
def test_parse_pool_config_invalid_scheduling_mode_falls_back_to_cache_affinity() -> None:
cfg = parse_pool_config({"pool_advanced": {"scheduling_mode": "unknown"}})
assert cfg is not None
assert cfg.scheduling_mode == "lru"
# Default with no explicit presets: cache_affinity -> multi_score
assert cfg.scheduling_mode == "multi_score"
def test_parse_pool_config_scoring_weights_invalid_values_are_clamped() -> None:

3
tests/services/test_pool_config_backward_compat.py
View File

@@ -34,7 +34,8 @@ def test_empty_pool_advanced_returns_defaults() -> None:
assert cfg is not None
defaults = PoolConfig()
assert cfg.sticky_session_ttl_seconds == defaults.sticky_session_ttl_seconds
assert cfg.lru_enabled is True
# Default with empty pool_advanced: cache_affinity preset, no LRU
assert cfg.lru_enabled is False
def test_shim_imports_resolve() -> None:

21
tests/services/test_pool_manager_trace.py
View File

@@ -40,7 +40,7 @@ async def test_trace_attached_to_first_candidate() -> None:
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None), "key-2": (None, None)},
return_value={"key-1": None, "key-2": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",
@@ -72,7 +72,7 @@ async def test_pool_extra_data_on_selected_candidate() -> None:
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None)},
return_value={"key-1": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",
@@ -103,8 +103,8 @@ async def test_pool_extra_data_on_skipped_candidate() -> None:
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={
"key-1": ("rate_limited_429", 120),
"key-2": (None, None),
"key-1": "rate_limited_429",
"key-2": None,
},
),
patch(
@@ -122,7 +122,8 @@ async def test_pool_extra_data_on_skipped_candidate() -> None:
assert extra is not None
assert extra["pool_skip"]["type"] == "cooldown"
assert extra["pool_skip"]["cooldown_reason"] == "rate_limited_429"
assert extra["pool_skip"]["cooldown_ttl"] == 120
# TTL is not fetched on the scheduling hot path (include_ttl=False)
assert "cooldown_ttl" not in extra["pool_skip"]
@pytest.mark.asyncio
@@ -142,9 +143,9 @@ async def test_trace_build_summary_matches() -> None:
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={
"key-1": ("overloaded_529", 60),
"key-2": (None, None),
"key-3": (None, None),
"key-1": "overloaded_529",
"key-2": None,
"key-3": None,
},
),
patch(
@@ -181,7 +182,7 @@ async def test_trace_build_summary_uses_attempted_key_ids_when_provided() -> Non
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None), "key-2": (None, None)},
return_value={"key-1": None, "key-2": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",
@@ -218,7 +219,7 @@ async def test_sticky_trace_info() -> None:
patch(
"src.services.provider.pool.redis_ops.batch_get_cooldowns",
new_callable=AsyncMock,
return_value={"key-1": (None, None), "key-2": (None, None)},
return_value={"key-1": None, "key-2": None},
),
patch(
"src.services.provider.pool.redis_ops.get_lru_scores",

2
tests/services/test_pool_multi_score_strategy.py
View File

@@ -34,6 +34,7 @@ def test_multi_score_prefers_low_latency_when_latency_weight_is_high() -> None:
strategy = MultiScoreStrategy()
cfg = PoolConfig(
scheduling_mode="multi_score",
scheduling_presets=(),
scoring_weights=ScoringWeights(lru=0.0, latency=1.0, health=0.0, cost_remaining=0.0),
)
ctx = _context()
@@ -48,6 +49,7 @@ def test_multi_score_combines_health_and_cost() -> None:
strategy = MultiScoreStrategy()
cfg = PoolConfig(
scheduling_mode="multi_score",
scheduling_presets=(),
scoring_weights=ScoringWeights(lru=0.0, latency=0.0, health=0.5, cost_remaining=0.5),
cost_limit_per_key_tokens=1000,
)

3
tests/services/test_provider_keys_quota_service.py
View File

@@ -35,6 +35,9 @@ class _FakeQuery:
self._first_result = first_result
self._all_result = all_result or []
def options(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return self
def filter(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return self