refactor: 引入模块钩子系统，解耦认证逻辑，支持模块/normalizer/parser 自动发现

- 新增 HookDispatcher 钩子分发器，支持 FIRST_RESULT 和 COLLECT_ALL 两种策略
- LDAP 认证逻辑从 AuthService 移至 ldap 模块钩子实现
- Management Token 前缀认证从 pipeline 硬编码改为模块钩子注册
- src/modules/ 改为自动扫描子目录发现 ModuleDefinition
- normalizers 和 parsers 注册改为基于类属性自动发现
- OpenAI CLI 增加 /v1/responses/compact 端点和并行 tool_call 支持
- OpenAI CLI normalizer 支持 Chat Completions 格式自动回退
- Codex 适配器增加 compact 模式上下文传递和 header 调整
- HeaderBuilder 改进非 latin-1 字符处理（UTF-8 字节透传）
- Gunicorn 增加 graceful_timeout 防止僵尸进程

src/core/api_format/conversion/normalizers/openai.py

@@ -609,21 +609,33 @@ class OpenAINormalizer(FormatNormalizer):
         # tool_calls delta
         tool_calls = delta.get("tool_calls")
         if isinstance(tool_calls, list):
+            # index -> tool_id 映射（用于后续 delta 缺少 id 时查找）
+            index_to_id: dict[str, str] = ss.setdefault("tool_index_to_id", {})
+
             for tool_call in tool_calls:
                 if not isinstance(tool_call, dict):
                     continue
 
                 tc_id = str(tool_call.get("id") or "")
+                raw_index = tool_call.get("index")
+                tc_index = str(raw_index) if raw_index is not None else ""
                 fn = tool_call.get("function") or {}
                 fn = fn if isinstance(fn, dict) else {}
                 tc_name = str(fn.get("name") or "")
                 tc_args = fn.get("arguments")
 
-                block_index = self._ensure_tool_block_index(
-                    ss, tc_id or str(tool_call.get("index") or "")
-                )
+                # 首次出现的 delta 同时有 id 和 index，记录映射
+                if tc_id and tc_index:
+                    index_to_id[tc_index] = tc_id
+                # 后续 delta 只有 index 没有 id，通过映射恢复 id
+                elif not tc_id and tc_index and tc_index in index_to_id:
+                    tc_id = index_to_id[tc_index]
 
-                # tool start（只在首次见到该 tool_id 时发）
+                # 用 tool_id 作为优先 key（确保同一 tool call 始终同一 block_index）
+                tool_key = tc_id or tc_index
+                block_index = self._ensure_tool_block_index(ss, tool_key)
+
+                # tool start（只在首次见到该 tool_key 时发）
                 started_key = f"tool_started:{block_index}"
                 if not ss.get(started_key):
                     ss[started_key] = True
@@ -649,7 +661,7 @@ class OpenAINormalizer(FormatNormalizer):
         finish_reason = c0.get("finish_reason")
         if finish_reason is not None:
             stop_reason = self._FINISH_REASON_TO_STOP.get(str(finish_reason), StopReason.UNKNOWN)
-            # 先补齐 content_block_stop（thinking + text），再发送 MessageStop
+            # 先补齐 content_block_stop（thinking + text + tool_calls），再发送 MessageStop
             if ss.get("thinking_block_started") and not ss.get("thinking_block_stopped"):
                 ss["thinking_block_stopped"] = True
                 events.append(
@@ -660,6 +672,14 @@ class OpenAINormalizer(FormatNormalizer):
                 events.append(
                     ContentBlockStopEvent(block_index=_reserve_block_index("text_block_index"))
                 )
+            # 补齐所有已开始但未结束的 tool_call block
+            tool_id_map = ss.get("tool_id_to_block_index")
+            if isinstance(tool_id_map, dict):
+                for _tk, bi in tool_id_map.items():
+                    stopped_key = f"tool_stopped:{bi}"
+                    if ss.get(f"tool_started:{bi}") and not ss.get(stopped_key):
+                        ss[stopped_key] = True
+                        events.append(ContentBlockStopEvent(block_index=int(bi)))
             # 解析 usage（需要请求时设置 stream_options.include_usage: true）
             usage_info = self._openai_usage_to_internal(chunk.get("usage"))
             events.append(MessageStopEvent(stop_reason=stop_reason, usage=usage_info))

src/core/api_format/conversion/normalizers/openai_cli.py

@@ -55,6 +55,31 @@ from src.core.api_format.conversion.stream_events import (
     UnknownStreamEvent,
 )
 from src.core.api_format.conversion.stream_state import StreamState
+from src.core.logger import logger
+
+
+def _is_chat_completions_response(data: dict[str, Any]) -> bool:
+    """检测数据是否为 OpenAI Chat Completions 格式（而非 Responses API 格式）。
+
+    Chat Completions 的特征：
+    - 非流式：有 choices 数组且 object == "chat.completion"
+    - 流式：有 choices 数组且 object == "chat.completion.chunk"
+    """
+    if not isinstance(data, dict):
+        return False
+    obj = data.get("object", "")
+    if isinstance(obj, str) and obj.startswith("chat.completion"):
+        return True
+    if isinstance(data.get("choices"), list) and "type" not in data:
+        return True
+    return False
+
+
+def _get_openai_chat_normalizer() -> "FormatNormalizer | None":
+    """获取已注册的 openai:chat normalizer 实例（延迟获取避免循环导入）。"""
+    from src.core.api_format.conversion.registry import format_conversion_registry
+
+    return format_conversion_registry.get_normalizer("openai:chat")
 
 
 class OpenAICliNormalizer(FormatNormalizer):
@@ -126,7 +151,9 @@ class OpenAICliNormalizer(FormatNormalizer):
         *,
         target_variant: str | None = None,
     ) -> dict[str, Any]:
-        is_codex = str(target_variant or "").lower() == "codex"
+        openai_cli_extra = internal.extra.get("openai_cli", {})
+        is_compact = bool(openai_cli_extra.get("_aether_compact"))
+        is_codex = str(target_variant or "").lower() == "codex" and not is_compact
 
         result: dict[str, Any] = {
             "model": internal.model,
@@ -178,7 +205,6 @@ class OpenAICliNormalizer(FormatNormalizer):
             result["tool_choice"] = self._tool_choice_to_openai(internal.tool_choice)
 
         # 还原 OpenAI Responses API 的其他字段（黑名单：已单独处理的字段不还原）
-        openai_cli_extra = internal.extra.get("openai_cli", {})
         handled_keys = {
             "model",
             "input",
@@ -228,16 +254,25 @@ class OpenAICliNormalizer(FormatNormalizer):
     def response_to_internal(self, response: dict[str, Any]) -> InternalResponse:
         payload = self._unwrap_response_object(response)
 
+        # 检测 Chat Completions 格式回退
+        if _is_chat_completions_response(payload):
+            chat_norm = _get_openai_chat_normalizer()
+            if chat_norm is not None:
+                logger.debug(
+                    "[OpenAICliNormalizer] 检测到 Chat Completions 响应格式，委托给 openai:chat normalizer"
+                )
+                return chat_norm.response_to_internal(payload)
+
         rid = str(payload.get("id") or "")
         model = str(payload.get("model") or "")
 
-        blocks, extra = self._extract_output_text_blocks(payload)
+        blocks, extra, has_tool_use = self._extract_output_blocks(payload)
         usage = self._usage_to_internal(payload.get("usage"))
 
         stop_reason = StopReason.UNKNOWN
         status = payload.get("status")
         if isinstance(status, str) and status == "completed":
-            stop_reason = StopReason.END_TURN
+            stop_reason = StopReason.TOOL_USE if has_tool_use else StopReason.END_TURN
 
         return InternalResponse(
             id=rid,
@@ -254,14 +289,49 @@ class OpenAICliNormalizer(FormatNormalizer):
         *,
         requested_model: str | None = None,
     ) -> dict[str, Any]:
-        text = self._collapse_internal_text(internal.content)
+        output_items: list[dict[str, Any]] = []
 
-        output_message = {
-            "type": "message",
-            "id": f"msg_{internal.id or 'stream'}",
-            "role": "assistant",
-            "content": [{"type": "output_text", "text": text}],
-        }
+        # 构建 output items：message（文本）和 function_call（工具调用）
+        text = self._collapse_internal_text(internal.content)
+        if text:
+            output_items.append(
+                {
+                    "type": "message",
+                    "id": f"msg_{internal.id or 'stream'}",
+                    "role": "assistant",
+                    "status": "completed",
+                    "content": [{"type": "output_text", "text": text}],
+                }
+            )
+
+        for block in internal.content:
+            if isinstance(block, ToolUseBlock):
+                output_items.append(
+                    {
+                        "type": "function_call",
+                        "call_id": block.tool_id,
+                        "id": block.tool_id,
+                        "name": block.tool_name,
+                        "arguments": (
+                            json.dumps(block.tool_input, ensure_ascii=False)
+                            if block.tool_input
+                            else "{}"
+                        ),
+                        "status": "completed",
+                    }
+                )
+
+        # 如果没有任何 output item，添加空 message（保持结构完整）
+        if not output_items:
+            output_items.append(
+                {
+                    "type": "message",
+                    "id": f"msg_{internal.id or 'stream'}",
+                    "role": "assistant",
+                    "status": "completed",
+                    "content": [{"type": "output_text", "text": ""}],
+                }
+            )
 
         usage = internal.usage or UsageInfo()
         usage_obj: dict[str, Any] = {
@@ -279,7 +349,7 @@ class OpenAICliNormalizer(FormatNormalizer):
             "created": int(time.time()),
             "model": model_name,
             "status": "completed",
-            "output": [output_message],
+            "output": output_items,
             "usage": usage_obj,
         }
 
@@ -303,6 +373,18 @@ class OpenAICliNormalizer(FormatNormalizer):
                 pass
             return events
 
+        # 检测 Chat Completions 流式格式回退
+        # 某些 Provider 即使配置为 openai:cli 也可能返回 Chat Completions 格式
+        if _is_chat_completions_response(chunk):
+            chat_norm = _get_openai_chat_normalizer()
+            if chat_norm is not None:
+                if not ss.get("_chat_fallback_logged"):
+                    ss["_chat_fallback_logged"] = True
+                    logger.debug(
+                        "[OpenAICliNormalizer] 检测到 Chat Completions 流式格式，委托给 openai:chat"
+                    )
+                return chat_norm.stream_chunk_to_internal(chunk, state)
+
         etype = str(chunk.get("type") or "")
 
         # 尽量在首次事件补齐 message_start
@@ -376,7 +458,17 @@ class OpenAICliNormalizer(FormatNormalizer):
             ss["text_block_stopped"] = True
             events.append(ContentBlockStopEvent(block_index=0))
 
-        events.append(MessageStopEvent(stop_reason=StopReason.END_TURN, usage=usage))
+        # 补齐所有已开始但未结束的 tool_call block
+        active_tools = ss.get("active_tool_blocks")
+        if isinstance(active_tools, dict):
+            for tool_id, bi in list(active_tools.items()):
+                events.append(ContentBlockStopEvent(block_index=bi))
+            active_tools.clear()
+
+        # 根据流中是否出现过工具调用来判断 stop_reason
+        has_tool_calls = bool(ss.get("tool_calls"))
+        stop_reason = StopReason.TOOL_USE if has_tool_calls else StopReason.END_TURN
+        events.append(MessageStopEvent(stop_reason=stop_reason, usage=usage))
         return events
 
     def _handle_response_failed(
@@ -411,21 +503,29 @@ class OpenAICliNormalizer(FormatNormalizer):
         if isinstance(item, dict):
             item_type = item.get("type")
             if item_type == "function_call":
-                if not ss.get("tool_block_started"):
-                    ss["tool_block_started"] = True
-                    ss["current_tool_id"] = item.get("call_id") or item.get("id") or ""
-                    ss["current_tool_name"] = item.get("name") or ""
-                    events.append(
-                        ContentBlockStartEvent(
-                            block_index=ss.get("block_index", 0),
-                            block_type=ContentType.TOOL_USE,
-                            extra={
-                                "tool_id": ss["current_tool_id"],
-                                "tool_name": ss["current_tool_name"],
-                            },
-                        )
+                tool_id = str(item.get("call_id") or item.get("id") or "")
+                tool_name = str(item.get("name") or "")
+                block_index = int(ss.get("block_index", 0))
+
+                # 记录当前活跃的工具调用（支持并行）
+                active_tools = ss.setdefault("active_tool_blocks", {})
+                active_tools[tool_id] = block_index
+                ss["current_tool_id"] = tool_id
+                ss["current_tool_name"] = tool_name
+
+                # 初始化工具调用收集
+                tool_calls = ss.setdefault("tool_calls", {})
+                tool_calls.setdefault(tool_id, {"name": tool_name, "args": ""})
+
+                events.append(
+                    ContentBlockStartEvent(
+                        block_index=block_index,
+                        block_type=ContentType.TOOL_USE,
+                        tool_id=tool_id,
+                        tool_name=tool_name,
                     )
-                    ss["block_index"] = ss.get("block_index", 0) + 1
+                )
+                ss["block_index"] = block_index + 1
         return events
 
     def _handle_output_item_done(
@@ -435,9 +535,11 @@ class OpenAICliNormalizer(FormatNormalizer):
         item = chunk.get("item")
         if isinstance(item, dict):
             item_type = item.get("type")
-            if item_type == "function_call" and ss.get("tool_block_started"):
-                ss["tool_block_started"] = False
-                events.append(ContentBlockStopEvent(block_index=ss.get("block_index", 1) - 1))
+            if item_type == "function_call":
+                tool_id = str(item.get("call_id") or item.get("id") or "")
+                active_tools = ss.get("active_tool_blocks", {})
+                block_index = active_tools.pop(tool_id, ss.get("block_index", 1) - 1)
+                events.append(ContentBlockStopEvent(block_index=block_index))
         return events
 
     def _handle_function_call_delta(
@@ -446,10 +548,20 @@ class OpenAICliNormalizer(FormatNormalizer):
         events: list[InternalStreamEvent] = []
         delta = chunk.get("delta") or ""
         if delta:
+            # 确定当前工具调用的 block_index 和 tool_id
+            tool_id = str(chunk.get("item_id") or ss.get("current_tool_id", ""))
+            active_tools = ss.get("active_tool_blocks", {})
+            block_index = active_tools.get(tool_id, ss.get("block_index", 1) - 1)
+
+            # 累积参数
+            tool_calls = ss.setdefault("tool_calls", {})
+            entry = tool_calls.setdefault(tool_id, {"name": "", "args": ""})
+            entry["args"] = str(entry.get("args") or "") + delta
+
             events.append(
                 ToolCallDeltaEvent(
-                    block_index=ss.get("block_index", 1) - 1,
-                    tool_id=ss.get("current_tool_id", ""),
+                    block_index=block_index,
+                    tool_id=tool_id,
                     input_delta=delta,
                 )
             )
@@ -904,17 +1016,27 @@ class OpenAICliNormalizer(FormatNormalizer):
             return resp_inner
         return response
 
-    def _extract_output_text_blocks(
+    def _extract_output_blocks(
         self, payload: dict[str, Any]
-    ) -> tuple[list[ContentBlock], dict[str, Any]]:
+    ) -> tuple[list[ContentBlock], dict[str, Any], bool]:
+        """从 Responses API 的 output 提取所有内容块。
+
+        Returns:
+            (blocks, extra, has_tool_use): 内容块列表、extra 信息、是否包含工具调用
+        """
         text_parts: list[str] = []
+        blocks: list[ContentBlock] = []
+        has_tool_use = False
 
         output = payload.get("output")
         if isinstance(output, list):
             for item in output:
                 if not isinstance(item, dict):
                     continue
-                if item.get("type") == "message":
+
+                item_type = item.get("type")
+
+                if item_type == "message":
                     content = item.get("content")
                     if isinstance(content, list):
                         for part in content:
@@ -927,22 +1049,44 @@ class OpenAICliNormalizer(FormatNormalizer):
                                 text_parts.append(part.get("text") or "")
                     continue
 
-                if item.get("type") in ("output_text", "text") and isinstance(
-                    item.get("text"), str
-                ):
+                if item_type == "function_call":
+                    has_tool_use = True
+                    tool_id = str(item.get("call_id") or item.get("id") or "")
+                    tool_name = str(item.get("name") or "")
+                    args_raw = item.get("arguments") or "{}"
+                    try:
+                        tool_input = (
+                            json.loads(args_raw)
+                            if isinstance(args_raw, str)
+                            else (args_raw if isinstance(args_raw, dict) else {})
+                        )
+                    except (json.JSONDecodeError, TypeError):
+                        tool_input = {"_raw": args_raw}
+                    blocks.append(
+                        ToolUseBlock(
+                            tool_id=tool_id,
+                            tool_name=tool_name,
+                            tool_input=tool_input,
+                        )
+                    )
+                    continue
+
+                if item_type in ("output_text", "text") and isinstance(item.get("text"), str):
                     text_parts.append(item.get("text") or "")
 
         # 兼容：部分实现可能直接给 output_text
         if not text_parts and isinstance(payload.get("output_text"), str):
             text_parts.append(payload.get("output_text") or "")
 
-        blocks: list[ContentBlock] = []
+        # 文本块放在前面，工具调用块在后面（与 Claude 的 content 顺序一致）
+        result_blocks: list[ContentBlock] = []
         text = "".join(text_parts)
         if text:
-            blocks.append(TextBlock(text=text))
+            result_blocks.append(TextBlock(text=text))
+        result_blocks.extend(blocks)
 
         extra: dict[str, Any] = {"raw": {"openai_cli_output": output}} if output is not None else {}
-        return blocks, extra
+        return result_blocks, extra, has_tool_use
 
     def _usage_to_internal(self, usage: Any) -> UsageInfo:
         if not isinstance(usage, dict):

src/core/api_format/conversion/registry.py

@@ -438,7 +438,7 @@ _REGISTRATION_LOCK = threading.Lock()
 
 
 def register_default_normalizers() -> None:
-    """注册默认 Normalizers（OPENAI/CLAUDE/GEMINI + *_CLI）"""
+    """自动发现并注册 normalizers/ 目录下的所有 FormatNormalizer 实现"""
     global _DEFAULT_NORMALIZERS_REGISTERED  # noqa: PLW0603 - module-level 缓存
 
     # 快速路径：已注册则直接返回（无锁）
@@ -450,23 +450,44 @@ def register_default_normalizers() -> None:
         if _DEFAULT_NORMALIZERS_REGISTERED:
             return
 
-        from src.core.api_format.conversion.normalizers.claude import ClaudeNormalizer
-        from src.core.api_format.conversion.normalizers.claude_cli import ClaudeCliNormalizer
-        from src.core.api_format.conversion.normalizers.gemini import GeminiNormalizer
-        from src.core.api_format.conversion.normalizers.gemini_cli import GeminiCliNormalizer
-        from src.core.api_format.conversion.normalizers.openai import OpenAINormalizer
-        from src.core.api_format.conversion.normalizers.openai_cli import OpenAICliNormalizer
+        import importlib
+        import inspect
+        from pathlib import Path
 
-        format_conversion_registry.register(OpenAINormalizer())
-        format_conversion_registry.register(OpenAICliNormalizer())
-        format_conversion_registry.register(ClaudeNormalizer())
-        format_conversion_registry.register(ClaudeCliNormalizer())
-        format_conversion_registry.register(GeminiNormalizer())
-        format_conversion_registry.register(GeminiCliNormalizer())
+        normalizers_dir = Path(__file__).parent / "normalizers"
+        for py_file in sorted(normalizers_dir.glob("*.py")):
+            if py_file.name.startswith("_"):
+                continue
+            module_name = py_file.stem
+            module_path = f"src.core.api_format.conversion.normalizers.{module_name}"
+            try:
+                mod = importlib.import_module(module_path)
+            except Exception as e:
+                logger.error("[FormatConversionRegistry] 导入 {} 失败: {}", module_path, e)
+                continue
+            for _attr_name, obj in inspect.getmembers(mod, inspect.isclass):
+                if (
+                    issubclass(obj, FormatNormalizer)
+                    and obj is not FormatNormalizer
+                    and hasattr(obj, "FORMAT_ID")
+                    and obj.__module__ == mod.__name__
+                ):
+                    fmt_id = str(obj.FORMAT_ID).upper()
+                    if format_conversion_registry.get_normalizer(fmt_id) is not None:
+                        logger.warning(
+                            "[FormatConversionRegistry] FORMAT_ID '{}' 重复注册，{} 将覆盖已有实现",
+                            fmt_id,
+                            obj.__name__,
+                        )
+                    try:
+                        format_conversion_registry.register(obj())
+                    except Exception as e:
+                        logger.error("[FormatConversionRegistry] 注册 {} 失败: {}", obj.__name__, e)
 
         _DEFAULT_NORMALIZERS_REGISTERED = True
         logger.info(
-            f"[FormatConversionRegistry] 已注册 {len(format_conversion_registry.list_normalizers())} 个 normalizer"
+            "[FormatConversionRegistry] 已注册 {} 个 normalizer",
+            len(format_conversion_registry.list_normalizers()),
         )
 
 

src/core/api_format/headers.py

@@ -334,19 +334,21 @@ class HeaderBuilder:
     def build(self) -> dict[str, str]:
         """构建最终的头部字典
 
-        Safety net: 跳过值中包含非 ASCII 字符的头部并记录警告，
-        防止 httpx 发送时抛出 ``UnicodeEncodeError``。
+        httpx 要求 header 值可被 latin-1 编码。对于包含非 latin-1 字符
+        （如中文）的值，先 UTF-8 编码再按 latin-1 解码，使 httpx 将原始
+        UTF-8 字节逐字节发送到上游 —— 与 Go net/http 的行为一致。
         """
         result: dict[str, str] = {}
         for original_key, value in self._headers.values():
             try:
-                value.encode("ascii")
+                value.encode("latin-1")
             except (UnicodeEncodeError, UnicodeDecodeError):
-                logger.warning(
-                    "Dropping non-ASCII header before upstream request: {}",
+                # 将 UTF-8 字节逐字节映射为 latin-1 字符串，httpx 会原样发送
+                logger.debug(
+                    "Header '{}' contains non-latin-1 chars, encoding as raw UTF-8 bytes",
                     original_key,
                 )
-                continue
+                value = value.encode("utf-8").decode("latin-1")
             result[original_key] = value
         return result