refactor: 重构异步任务系统和计费服务架构

- 重构任务系统:新增 lifecycle (TaskStatus/BillingStatus)、context、application 模块
- 将 video tasks 泛化为 async tasks,支持更通用的异步任务管理
- 新增 Gemini Files 管理模块和管理界面
- 重构 billing 服务:拆分 schema.py 和 service.py
- 新增 candidate 服务模块用于请求候选管理
- 数据库迁移:添加 billing_status、request_id、gemini_file_mappings 表和索引
- 移除废弃的 video_telemetry、task orchestrator 等模块
This commit is contained in:
fawney19
2026-02-02 03:16:52 +08:00
parent feb7484fda
commit 9e31efe26c
75 changed files with 7511 additions and 2068 deletions

View File

@@ -29,6 +29,8 @@ from src.services.billing.models import (
CostBreakdown,
StandardizedUsage,
)
from src.services.billing.schema import BillingSnapshot, CostResult
from src.services.billing.service import BillingService
from src.services.billing.templates import BILLING_TEMPLATE_REGISTRY, BillingTemplates
from src.services.billing.usage_mapper import UsageMapper, map_usage, map_usage_from_response
@@ -44,6 +46,10 @@ __all__ = [
# 计算器
"BillingCalculator",
"calculate_request_cost",
# 统一入口Phase2
"BillingService",
"BillingSnapshot",
"CostResult",
# 映射器
"UsageMapper",
"map_usage",

View File

@@ -0,0 +1,64 @@
"""
Billing schema (stable contracts)
These dataclasses are meant to be stored in `Usage.request_metadata` / `Task.request_metadata`
for auditability. They are internal-only and MUST NOT be exposed to end users without sanitizing.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
BILLING_SNAPSHOT_SCHEMA_VERSION = "1.0"
BillingSnapshotStatus = Literal["complete", "incomplete", "no_rule", "legacy"]
@dataclass(frozen=True)
class BillingSnapshot:
"""Stable billing snapshot for audit."""
schema_version: str = BILLING_SNAPSHOT_SCHEMA_VERSION
# Rule info (optional for legacy/no_rule)
rule_id: str | None = None
rule_name: str | None = None
scope: str | None = None
# Rule expression (internal, do not expose to clients)
expression: str | None = None
# Dimensions
dimensions_used: dict[str, Any] = field(default_factory=dict)
missing_required: list[str] = field(default_factory=list)
# Result
cost: float = 0.0
status: BillingSnapshotStatus = "no_rule"
# Audit
calculated_at: str = "" # ISO 8601
def to_dict(self) -> dict[str, Any]:
return {
"schema_version": self.schema_version,
"rule_id": self.rule_id,
"rule_name": self.rule_name,
"scope": self.scope,
"expression": self.expression,
"dimensions_used": self.dimensions_used,
"missing_required": self.missing_required,
"cost": self.cost,
"status": self.status,
"calculated_at": self.calculated_at,
}
@dataclass(frozen=True)
class CostResult:
"""Billing calculation output."""
cost: float
status: BillingSnapshotStatus
snapshot: BillingSnapshot

View File

@@ -0,0 +1,145 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy.orm import Session
from src.config.settings import config
from src.core.logger import logger
from src.services.billing.dimension_collector_service import DimensionCollectorService
from src.services.billing.formula_engine import BillingIncompleteError, FormulaEngine
from src.services.billing.rule_service import BillingRuleService
from src.services.model.cost import ModelCostService
from .schema import BILLING_SNAPSHOT_SCHEMA_VERSION, BillingSnapshot, CostResult
class BillingService:
"""
BillingService (pure-ish application helper for billing domain).
Notes:
- This service **does not** write Usage rows.
- It may read billing rules & collectors from DB.
"""
def __init__(self, db: Session):
self.db = db
self._formula_engine = FormulaEngine()
self._dimension_collector = DimensionCollectorService(db)
def collect_dimensions(
self,
*,
api_format: str | None,
task_type: str | None,
request: dict[str, Any] | None = None,
response: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
base_dimensions: dict[str, Any] | None = None,
) -> dict[str, Any]:
return self._dimension_collector.collect_dimensions(
api_format=api_format,
task_type=task_type,
request=request,
response=response,
metadata=metadata,
base_dimensions=base_dimensions,
)
def calculate(
self,
*,
task_type: str,
model: str,
provider_id: str,
dimensions: dict[str, Any],
strict_mode: bool | None = None,
) -> CostResult:
"""
Calculate cost for a task.
Returns:
CostResult (includes BillingSnapshot)
Raises:
BillingIncompleteError: when strict_mode=True and required dims missing.
"""
strict = config.billing_strict_mode if strict_mode is None else bool(strict_mode)
lookup = BillingRuleService.find_rule(
self.db,
provider_id=provider_id,
model_name=model,
task_type=task_type,
)
if lookup and lookup.rule and lookup.rule.expression:
rule = lookup.rule
result = self._formula_engine.evaluate(
expression=rule.expression,
variables=rule.variables or {},
dimensions=dimensions,
dimension_mappings=rule.dimension_mappings or {},
strict_mode=strict,
)
cost = float(result.cost) if result.status == "complete" else 0.0
snapshot = BillingSnapshot(
schema_version=BILLING_SNAPSHOT_SCHEMA_VERSION,
rule_id=str(rule.id),
rule_name=str(rule.name),
scope=str(getattr(lookup, "scope", None) or ""),
expression=str(rule.expression),
dimensions_used=dimensions,
missing_required=result.missing_required,
cost=cost,
status=result.status,
calculated_at=datetime.now(timezone.utc).isoformat(),
)
return CostResult(cost=cost, status=result.status, snapshot=snapshot)
# No rule fallback
if task_type in ("chat", "cli"):
input_tokens = int(dimensions.get("input_tokens") or 0)
output_tokens = int(dimensions.get("output_tokens") or 0)
cost = float(
ModelCostService.calculate_cost(
model=model,
input_tokens=input_tokens,
output_tokens=output_tokens,
)
)
snapshot = BillingSnapshot(
schema_version=BILLING_SNAPSHOT_SCHEMA_VERSION,
rule_id=None,
rule_name=None,
scope=None,
expression=None,
dimensions_used=dimensions,
missing_required=[],
cost=cost,
status="legacy",
calculated_at=datetime.now(timezone.utc).isoformat(),
)
return CostResult(cost=cost, status="legacy", snapshot=snapshot)
logger.warning(
"No billing rule for task (task_type=%s, model=%s, provider_id=%s)",
task_type,
model,
provider_id,
)
snapshot = BillingSnapshot(
schema_version=BILLING_SNAPSHOT_SCHEMA_VERSION,
rule_id=None,
rule_name=None,
scope=None,
expression=None,
dimensions_used=dimensions,
missing_required=[],
cost=0.0,
status="no_rule",
calculated_at=datetime.now(timezone.utc).isoformat(),
)
return CostResult(cost=0.0, status="no_rule", snapshot=snapshot)