#!/usr/bin/env python3 """飞书执行链路回放评测。 目标： 1) 回放样本消息，驱动 FeishuSignalPipelineService。 2) 断言 DB 结果 + route_status + reply_text 一致。 3) 输出结构化报告，支持 CI 门禁。 """ from __future__ import annotations import argparse import asyncio import contextvars import json import sys from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional from sqlalchemy import and_, delete, func, select from sqlalchemy.ext.asyncio import AsyncSession from app.database import AsyncSessionLocal from app.models import ( AIProcessingLog, FeishuItem, Notification, Project, ProjectSignalLink, ProjectSuggestion, Signal, User, ) from app.services.adaptive_rag_service import AdaptiveRAGService from app.services.feishu_integration import FeishuIntegrationService from app.services.feishu_signal_pipeline import FeishuSignalPipelineService CURRENT_CASE_ID: contextvars.ContextVar[str | None] = contextvars.ContextVar( "feishu_replay_case_id", default=None, ) DEFAULT_REPLAY_TENANT_ID = 9901 @dataclass class ReplayCase: id: str message_content: str sender: str source_chat_id: str feishu_message_id: str mock: dict[str, Any] expected: dict[str, Any] def _load_cases(path: Path) -> list[ReplayCase]: rows: list[ReplayCase] = [] with path.open("r", encoding="utf-8") as f: for idx, raw in enumerate(f, start=1): line = raw.strip() if not line: continue payload = json.loads(line) case_id = str(payload.get("id") or f"CASE-{idx:03d}").strip() rows.append( ReplayCase( id=case_id, message_content=str(payload.get("message_content") or "").strip(), sender=str(payload.get("sender") or "回放用户").strip(), source_chat_id=str(payload.get("source_chat_id") or "oc-replay-chat").strip(), feishu_message_id=str(payload.get("feishu_message_id") or "").strip(), mock=payload.get("mock") if isinstance(payload.get("mock"), dict) else {}, expected=payload.get("expected") if isinstance(payload.get("expected"), dict) else {}, ) ) return rows def _lint_cases(cases: list[ReplayCase]) -> dict[str, Any]: unique_case_ids = len({c.id for c in cases}) non_empty_messages = sum(1 for c in cases if c.message_content) with_duplicates = sum(1 for c in cases if c.expected.get("status") == "ignored" and c.expected.get("reason") == "duplicate_filtered") with_qa = sum(1 for c in cases if c.expected.get("scene") == "qa") with_execute = sum(1 for c in cases if c.expected.get("scene") == "execute") with_multi_write = sum(1 for c in cases if bool(c.expected.get("multi_write"))) valid = ( len(cases) >= 50 and unique_case_ids == len(cases) and non_empty_messages == len(cases) and with_duplicates > 0 and with_qa > 0 and with_execute > 0 and with_multi_write > 0 ) return { "count": len(cases), "unique_case_ids": unique_case_ids, "non_empty_messages": non_empty_messages, "duplicate_cases": with_duplicates, "qa_cases": with_qa, "execute_cases": with_execute, "multi_write_cases": with_multi_write, "valid": valid, } def _default_level_signal() -> dict[str, Any]: return { "build_intent_score": 0.2, "scope_score": 0.2, "timeline_score": 0.1, "owner_score": 0.1, "project_score": 0.2, "level": "signal", "level_confidence": 0.88, "level_reasoning": "回放样本：明确执行动作，按 signal 处理。", "should_split": False, "split_items": [], } def _default_signal(project_id: int = 1001, confidence: float = 0.92) -> dict[str, Any]: return { "title": "新增信号源", "type": "新需求", "priority": "P1", "matched_project_id": int(project_id), "summary": "新增一个可执行的信号源需求。", "reasoning": ["目标明确", "动作清晰"], "confidence": float(confidence), "recommended_action": "enter_only", } def _default_level_project() -> dict[str, Any]: return { "build_intent_score": 0.95, "scope_score": 0.86, "timeline_score": 0.8, "owner_score": 0.75, "project_score": 0.9, "level": "project", "level_confidence": 0.9, "level_reasoning": "回放样本：范围较大，按 project 处理。", "should_split": False, "split_items": [], } def _default_project_prefill(index: int = 1) -> dict[str, Any]: return { "suggested_name": f"回放项目-{index}", "suggested_description": "回放样本自动生成项目建议。", "suggested_owner": "回放负责人", "suggested_timeline": "2周", "confidence": 0.86, "reasoning": ["目标明确", "时间可控"], } def _default_routing_policy() -> dict[str, Any]: return { "cold_start_enabled": False, "signal_auto_enter_threshold": 0.8, "project_auto_create_threshold": 0.85, "project_uncertain_threshold": 0.6, "chat_context_auto_link_enabled": False, } def _case_by_id(cases: list[ReplayCase]) -> dict[str, ReplayCase]: return {row.id: row for row in cases} class ReplayPatcher: """在回放期间替换外部依赖（RAG/LLM）为样本驱动。""" def __init__(self, cases: list[ReplayCase]) -> None: self._case_map = _case_by_id(cases) self._project_id_map: dict[int, int] = {1001: 1001, 1002: 1002, 1003: 1003} self._orig_classify_intent = AdaptiveRAGService.classify_intent self._orig_answer_question = AdaptiveRAGService.answer_question self._orig_classify_level = FeishuSignalPipelineService._classify_level self._orig_extract_signal = FeishuSignalPipelineService._extract_signal self._orig_extract_project_prefill = FeishuSignalPipelineService._extract_project_prefill self._orig_load_routing_policy = FeishuSignalPipelineService._load_routing_policy self._orig_notify_project_created = FeishuSignalPipelineService._notify_project_created self._orig_notify_signal_auto_entered = FeishuSignalPipelineService._notify_signal_auto_entered def _current_case(self) -> ReplayCase: case_id = CURRENT_CASE_ID.get() if not case_id: raise RuntimeError("missing_current_case_id") row = self._case_map.get(case_id) if row is None: raise RuntimeError(f"unknown_case_id:{case_id}") return row def set_project_id_map(self, value: dict[int, int]) -> None: mapped = {int(k): int(v) for k, v in (value or {}).items()} if mapped: self._project_id_map = mapped def _map_project_id(self, raw_value: Any) -> Optional[int]: if raw_value is None: return None try: source = int(raw_value) except Exception: return None return int(self._project_id_map.get(source, source)) def install(self) -> None: async def fake_classify_intent( _: AdaptiveRAGService, message: str, ) -> dict[str, Any]: case = self._current_case() mock = case.mock intent = str(mock.get("intent") or "statement").strip().lower() if intent == "question": return { "intent": "question", "question_part": message, "statement_part": "", } if intent == "mixed": question_part = str(mock.get("question_part") or message).strip() statement_part = str(mock.get("statement_part") or message).strip() return { "intent": "mixed", "question_part": question_part, "statement_part": statement_part, } return { "intent": "statement", "question_part": "", "statement_part": message, } async def fake_answer_question( _: AdaptiveRAGService, *, question: str, asked_by: Optional[int], asked_via: str, project_id: Optional[int], source_chat_id: Optional[str], session_id: Optional[int] = None, ) -> dict[str, Any]: _ = (asked_by, asked_via, project_id, source_chat_id, session_id) case = self._current_case() mock = case.mock answer = str(mock.get("qa_answer") or f"回放答案：{question[:30]}").strip() sources = mock.get("qa_sources") if not isinstance(sources, list): sources = [{"label": "Replay Source"}] return { "answer": answer, "sources": sources, "query_type": "question", "confidence": 0.9, } async def fake_classify_level( _: FeishuSignalPipelineService, *, message: str, projects: list[dict[str, Any]], ) -> dict[str, Any]: _ = (message, projects) case = self._current_case() if bool(case.mock.get("force_schema_error")): raise ValueError("level_classification_schema_invalid: replay_forced") level = case.mock.get("level") if isinstance(level, dict): return level return _default_level_signal() async def fake_extract_signal( _: FeishuSignalPipelineService, message: str, *, projects: list[dict[str, Any]], ) -> dict[str, Any]: _ = (message, projects) case = self._current_case() signal = case.mock.get("signal") if isinstance(signal, dict): payload = dict(signal) mapped_project_id = self._map_project_id(payload.get("matched_project_id")) if mapped_project_id is not None: payload["matched_project_id"] = mapped_project_id return payload payload = _default_signal() mapped_project_id = self._map_project_id(payload.get("matched_project_id")) if mapped_project_id is not None: payload["matched_project_id"] = mapped_project_id return payload async def fake_extract_project_prefill( _: FeishuSignalPipelineService, message: str, ) -> dict[str, Any]: _ = message case = self._current_case() prefill = case.mock.get("project_prefill") if isinstance(prefill, dict): return prefill return _default_project_prefill() async def fake_load_routing_policy(_: FeishuSignalPipelineService) -> dict[str, Any]: case = self._current_case() policy = case.mock.get("routing_policy") merged = _default_routing_policy() if isinstance(policy, dict): merged.update(policy) return merged async def fake_notify_project_created( _: FeishuSignalPipelineService, *, project: Project, ) -> None: _ = project return None async def fake_notify_signal_auto_entered( _: FeishuSignalPipelineService, *, project_id: int, signal: Any, ) -> None: _ = (project_id, signal) return None AdaptiveRAGService.classify_intent = fake_classify_intent # type: ignore[assignment] AdaptiveRAGService.answer_question = fake_answer_question # type: ignore[assignment] FeishuSignalPipelineService._classify_level = fake_classify_level # type: ignore[assignment] FeishuSignalPipelineService._extract_signal = fake_extract_signal # type: ignore[assignment] FeishuSignalPipelineService._extract_project_prefill = fake_extract_project_prefill # type: ignore[assignment] FeishuSignalPipelineService._load_routing_policy = fake_load_routing_policy # type: ignore[assignment] FeishuSignalPipelineService._notify_project_created = fake_notify_project_created # type: ignore[assignment] FeishuSignalPipelineService._notify_signal_auto_entered = fake_notify_signal_auto_entered # type: ignore[assignment] def restore(self) -> None: AdaptiveRAGService.classify_intent = self._orig_classify_intent # type: ignore[assignment] AdaptiveRAGService.answer_question = self._orig_answer_question # type: ignore[assignment] FeishuSignalPipelineService._classify_level = self._orig_classify_level # type: ignore[assignment] FeishuSignalPipelineService._extract_signal = self._orig_extract_signal # type: ignore[assignment] FeishuSignalPipelineService._extract_project_prefill = self._orig_extract_project_prefill # type: ignore[assignment] FeishuSignalPipelineService._load_routing_policy = self._orig_load_routing_policy # type: ignore[assignment] FeishuSignalPipelineService._notify_project_created = self._orig_notify_project_created # type: ignore[assignment] FeishuSignalPipelineService._notify_signal_auto_entered = self._orig_notify_signal_auto_entered # type: ignore[assignment] async def _seed_minimal_data(session: AsyncSession, *, tenant_id: int) -> dict[int, int]: users = ( await session.execute( select(User).where( and_( User.tenant_id == tenant_id, User.username.in_(("replay_admin", "replay_owner")), ) ) ) ).scalars().all() if not users: session.add( User( tenant_id=tenant_id, username="replay_admin", email="replay_admin@example.com", password_hash="x", name="Replay Admin", role="admin", assigned_domains=[], workload=0, status="active", is_active=True, ) ) session.add( User( tenant_id=tenant_id, username="replay_owner", email="replay_owner@example.com", password_hash="x", name="Replay Owner", role="ai_engineer", assigned_domains=[], workload=0, status="active", is_active=True, ) ) await session.flush() alias_to_name = { 1001: "回放项目-健身", 1002: "回放项目-路由", 1003: "回放项目-知识库", } project_map: dict[int, int] = {} for alias, title in alias_to_name.items(): existing = ( await session.execute( select(Project).where( and_( Project.tenant_id == tenant_id, Project.name == title, ) ).limit(1) ) ).scalar_one_or_none() if existing is None: existing = Project( tenant_id=tenant_id, name=title, title=title, description="回放种子项目", owner_id=None, status="active", domain_id=None, assigned_role="ai_engineer", signal_count=0, ) session.add(existing) await session.flush() project_map[int(alias)] = int(existing.id) await session.commit() return project_map async def _count_items_by_message_id( session: AsyncSession, *, tenant_id: int, message_id: str, ) -> int: return int( ( await session.execute( select(func.count(FeishuItem.id)).where( and_( FeishuItem.tenant_id == tenant_id, FeishuItem.feishu_message_id == message_id, ) ) ) ).scalar() or 0 ) async def _latest_item_route_status( session: AsyncSession, *, tenant_id: int, message_id: str, ) -> Optional[str]: row = ( await session.execute( select(FeishuItem.route_status) .where( and_( FeishuItem.tenant_id == tenant_id, FeishuItem.feishu_message_id == message_id, ) ) .order_by(FeishuItem.id.desc()) .limit(1) ) ).scalar_one_or_none() if row is None: return None return str(row) async def _cleanup_tenant_replay_data(session: AsyncSession, *, tenant_id: int) -> None: await session.execute(delete(ProjectSignalLink).where(ProjectSignalLink.tenant_id == tenant_id)) await session.execute(delete(ProjectSuggestion).where(ProjectSuggestion.tenant_id == tenant_id)) await session.execute(delete(Signal).where(Signal.tenant_id == tenant_id)) await session.execute( delete(AIProcessingLog).where( and_( AIProcessingLog.tenant_id == tenant_id, AIProcessingLog.feishu_message_id.like("om_replay_%"), ) ) ) await session.execute( delete(AIProcessingLog).where( and_( AIProcessingLog.tenant_id == tenant_id, AIProcessingLog.feishu_message_id.like("feishu-replay-%"), ) ) ) await session.execute(delete(FeishuItem).where(FeishuItem.tenant_id == tenant_id)) await session.execute(delete(Notification).where(Notification.tenant_id == tenant_id)) await session.execute(delete(Project).where(Project.tenant_id == tenant_id)) await session.execute(delete(User).where(User.tenant_id == tenant_id)) await session.commit() def _build_reply_text( *, service: FeishuIntegrationService, message_content: str, capture_result: dict[str, Any], trace_id: str, ) -> Optional[str]: scene = service._classify_message_scene(capture_result=capture_result) if scene == "execute": text = service._build_execution_receipt_reply( message_content=message_content, capture_result=capture_result, ) return service._guard_execute_reply_text(text, trace_id=trace_id) if scene == "qa": if bool(capture_result.get("direct_reply_required")): return str(capture_result.get("direct_reply_text") or "").strip() return None if scene == "error": return service._build_error_reply_text() return None def _match_reply(expected: dict[str, Any], reply: Optional[str]) -> bool: expected_equals = expected.get("reply_text") if expected_equals is not None: return str(reply or "") == str(expected_equals) expected_contains = expected.get("reply_contains") if expected_contains is None: return True if isinstance(expected_contains, str): expected_contains = [expected_contains] if not isinstance(expected_contains, list): return True text = str(reply or "") return all(str(token) in text for token in expected_contains) def _build_report( *, results: list[dict[str, Any]], started_at: datetime, ) -> dict[str, Any]: total = len(results) passed = sum(1 for row in results if row.get("passed")) failed = total - passed return { "version": "feishu-replay-v1", "generated_at": datetime.now(timezone.utc).isoformat(), "started_at": started_at.isoformat(), "metrics": { "total": total, "passed": passed, "failed": failed, "pass_rate": round((passed / total) if total else 0.0, 4), }, "cases": results, } def _write_report(report: dict[str, Any], output_dir: Path) -> Path: output_dir.mkdir(parents=True, exist_ok=True) ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") report_path = output_dir / f"feishu_replay_eval_{ts}.json" report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") latest = output_dir / "feishu_replay_latest.json" latest.write_text(report_path.read_text(encoding="utf-8"), encoding="utf-8") return report_path async def _run_replay( cases: list[ReplayCase], *, tenant_id: int, allow_tenant_1: bool = False, ) -> dict[str, Any]: if int(tenant_id) == 1 and not allow_tenant_1: raise RuntimeError("tenant_id=1 is blocked for replay cleanup; use dedicated replay tenant") started_at = datetime.now(timezone.utc) patcher = ReplayPatcher(cases) patcher.install() case_results: list[dict[str, Any]] = [] try: async with AsyncSessionLocal() as db: await _cleanup_tenant_replay_data(db, tenant_id=tenant_id) project_id_map = await _seed_minimal_data(db, tenant_id=tenant_id) patcher.set_project_id_map(project_id_map) pipeline = FeishuSignalPipelineService(db, tenant_id=tenant_id) integration = FeishuIntegrationService(db) integration.tenant_id = tenant_id for case in cases: token = CURRENT_CASE_ID.set(case.id) before_count = await _count_items_by_message_id( db, tenant_id=tenant_id, message_id=case.feishu_message_id, ) mismatches: list[str] = [] try: result = await pipeline.process_message( message_content=case.message_content, sender=case.sender, source_chat_id=case.source_chat_id, source_sender_open_id="ou_replay_user", feishu_message_id=case.feishu_message_id, captured_at=datetime.now(timezone.utc), ) except Exception as exc: result = {"status": "error", "classification": "error", "error": str(exc)} if str(case.expected.get("status") or "").strip().lower() != "error": mismatches.append(f"process_exception:{exc}") finally: CURRENT_CASE_ID.reset(token) after_count = await _count_items_by_message_id( db, tenant_id=tenant_id, message_id=case.feishu_message_id, ) item_delta = after_count - before_count route_status = None result_rows = result.get("results") if isinstance(result, dict) else None result_count = len(result_rows) if isinstance(result_rows, list) else 0 result_levels = sorted( { str((row or {}).get("level") or "").strip().lower() for row in (result_rows or []) if isinstance(row, dict) and str((row or {}).get("level") or "").strip() } ) if isinstance(result_rows, list) and result_rows: first = result_rows[0] if isinstance(result_rows[0], dict) else {} route_status = str((first or {}).get("route_status") or "").strip() or None if route_status is None: route_status = await _latest_item_route_status( db, tenant_id=tenant_id, message_id=case.feishu_message_id, ) scene = integration._classify_message_scene(capture_result=result if isinstance(result, dict) else {}) reply_text = _build_reply_text( service=integration, message_content=case.message_content, capture_result=result if isinstance(result, dict) else {}, trace_id=f"replay-{case.id}", ) expected = case.expected expected_status = expected.get("status") if expected_status is not None and str(result.get("status")) != str(expected_status): mismatches.append(f"status expected={expected_status} actual={result.get('status')}") expected_classification = expected.get("classification") if expected_classification is not None and str(result.get("classification")) != str(expected_classification): mismatches.append( f"classification expected={expected_classification} actual={result.get('classification')}" ) expected_reason = expected.get("reason") if expected_reason is not None and str(result.get("reason")) != str(expected_reason): mismatches.append(f"reason expected={expected_reason} actual={result.get('reason')}") expected_scene = expected.get("scene") if expected_scene is not None and str(scene) != str(expected_scene): mismatches.append(f"scene expected={expected_scene} actual={scene}") expected_route_status = expected.get("route_status") if expected_route_status is not None and str(route_status) != str(expected_route_status): mismatches.append(f"route_status expected={expected_route_status} actual={route_status}") expected_item_delta = expected.get("item_delta") if expected_item_delta is not None and int(item_delta) != int(expected_item_delta): mismatches.append(f"item_delta expected={expected_item_delta} actual={item_delta}") expected_result_count = expected.get("result_count") if expected_result_count is not None and int(result_count) != int(expected_result_count): mismatches.append(f"result_count expected={expected_result_count} actual={result_count}") expected_levels = expected.get("result_levels") if isinstance(expected_levels, list): expected_levels_set = sorted({str(x).strip().lower() for x in expected_levels if str(x).strip()}) if expected_levels_set != result_levels: mismatches.append(f"result_levels expected={expected_levels_set} actual={result_levels}") if bool(expected.get("multi_write")): if result_count < 2: mismatches.append(f"multi_write expected=true actual_result_count={result_count}") if len(result_levels) < 2: mismatches.append(f"multi_write expected mixed levels actual_levels={result_levels}") if not _match_reply(expected, reply_text): mismatches.append(f"reply mismatch expected={expected.get('reply_text') or expected.get('reply_contains')} actual={reply_text}") case_results.append( { "id": case.id, "passed": len(mismatches) == 0, "mismatches": mismatches, "expected": expected, "actual": { "status": result.get("status"), "classification": result.get("classification"), "reason": result.get("reason"), "scene": scene, "route_status": route_status, "item_delta": item_delta, "result_count": result_count, "result_levels": result_levels, "reply_text": reply_text, }, } ) await _cleanup_tenant_replay_data(db, tenant_id=tenant_id) finally: patcher.restore() return _build_report(results=case_results, started_at=started_at) async def _main_async(args: argparse.Namespace) -> int: cases = _load_cases(Path(args.input)) if args.case_ids: tokens = [token.strip() for token in str(args.case_ids).split(",") if token.strip()] wanted = set(tokens) filtered = [row for row in cases if row.id in wanted] missing = sorted(wanted - {row.id for row in filtered}) if missing: print(json.dumps({"error": "case_not_found", "missing": missing}, ensure_ascii=False, indent=2)) return 2 cases = filtered lint = _lint_cases(cases) if args.lint_only: print(json.dumps({"lint": lint}, ensure_ascii=False, indent=2)) return 0 if lint["valid"] else 2 if not lint["valid"] and not args.case_ids: print(json.dumps({"lint": lint}, ensure_ascii=False, indent=2)) return 2 tenant_id = int(args.tenant_id) if tenant_id <= 0: print(json.dumps({"error": "invalid_tenant_id", "tenant_id": args.tenant_id}, ensure_ascii=False, indent=2)) return 2 if tenant_id == 1 and not bool(args.allow_tenant_1): print( json.dumps( { "error": "tenant_id_blocked", "message": "tenant_id=1 默认禁用，避免误删主租户数据；请改用回放专用租户（例如 9901）。", }, ensure_ascii=False, indent=2, ) ) return 2 report = await _run_replay( cases, tenant_id=tenant_id, allow_tenant_1=bool(args.allow_tenant_1), ) report_path = _write_report(report, Path(args.output_dir)) metrics = report.get("metrics", {}) print( f"[feishu-replay] total={metrics.get('total')} passed={metrics.get('passed')} " f"failed={metrics.get('failed')} pass_rate={metrics.get('pass_rate')} " f"report={report_path}" ) if int(metrics.get("failed") or 0) > 0 and args.strict: return 2 return 0 def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Feishu replay eval") parser.add_argument( "--input", default="backend/evals/golden_feishu_replay_v1.jsonl", help="Replay dataset path (.jsonl)", ) parser.add_argument( "--output-dir", default="backend/evals/reports", help="Report output directory", ) parser.add_argument( "--tenant-id", default=str(DEFAULT_REPLAY_TENANT_ID), help="Replay tenant id", ) parser.add_argument( "--allow-tenant-1", action="store_true", help="Allow tenant_id=1 (dangerous, only for isolated test db).", ) parser.add_argument( "--lint-only", action="store_true", help="Only validate dataset structure", ) parser.add_argument( "--strict", action="store_true", help="Exit non-zero when replay has failures", ) parser.add_argument( "--case-ids", default="", help="Only run selected case ids, comma separated", ) return parser.parse_args() def main() -> None: args = parse_args() code = asyncio.run(_main_async(args)) raise SystemExit(code) if __name__ == "__main__": main()