834 lines
31 KiB
Python
834 lines
31 KiB
Python
#!/usr/bin/env python3
|
||
"""飞书执行链路回放评测。
|
||
|
||
目标：
|
||
1) 回放样本消息，驱动 FeishuSignalPipelineService。
|
||
2) 断言 DB 结果 + route_status + reply_text 一致。
|
||
3) 输出结构化报告，支持 CI 门禁。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import contextvars
|
||
import json
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Optional
|
||
|
||
from sqlalchemy import and_, delete, func, select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.database import AsyncSessionLocal
|
||
from app.models import (
|
||
AIProcessingLog,
|
||
FeishuItem,
|
||
Notification,
|
||
Project,
|
||
ProjectSignalLink,
|
||
ProjectSuggestion,
|
||
Signal,
|
||
User,
|
||
)
|
||
from app.services.adaptive_rag_service import AdaptiveRAGService
|
||
from app.services.feishu_integration import FeishuIntegrationService
|
||
from app.services.feishu_signal_pipeline import FeishuSignalPipelineService
|
||
|
||
CURRENT_CASE_ID: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||
"feishu_replay_case_id",
|
||
default=None,
|
||
)
|
||
DEFAULT_REPLAY_TENANT_ID = 9901
|
||
|
||
|
||
@dataclass
|
||
class ReplayCase:
|
||
id: str
|
||
message_content: str
|
||
sender: str
|
||
source_chat_id: str
|
||
feishu_message_id: str
|
||
mock: dict[str, Any]
|
||
expected: dict[str, Any]
|
||
|
||
|
||
def _load_cases(path: Path) -> list[ReplayCase]:
|
||
rows: list[ReplayCase] = []
|
||
with path.open("r", encoding="utf-8") as f:
|
||
for idx, raw in enumerate(f, start=1):
|
||
line = raw.strip()
|
||
if not line:
|
||
continue
|
||
payload = json.loads(line)
|
||
case_id = str(payload.get("id") or f"CASE-{idx:03d}").strip()
|
||
rows.append(
|
||
ReplayCase(
|
||
id=case_id,
|
||
message_content=str(payload.get("message_content") or "").strip(),
|
||
sender=str(payload.get("sender") or "回放用户").strip(),
|
||
source_chat_id=str(payload.get("source_chat_id") or "oc-replay-chat").strip(),
|
||
feishu_message_id=str(payload.get("feishu_message_id") or "").strip(),
|
||
mock=payload.get("mock") if isinstance(payload.get("mock"), dict) else {},
|
||
expected=payload.get("expected") if isinstance(payload.get("expected"), dict) else {},
|
||
)
|
||
)
|
||
return rows
|
||
|
||
|
||
def _lint_cases(cases: list[ReplayCase]) -> dict[str, Any]:
|
||
unique_case_ids = len({c.id for c in cases})
|
||
non_empty_messages = sum(1 for c in cases if c.message_content)
|
||
with_duplicates = sum(1 for c in cases if c.expected.get("status") == "ignored" and c.expected.get("reason") == "duplicate_filtered")
|
||
with_qa = sum(1 for c in cases if c.expected.get("scene") == "qa")
|
||
with_execute = sum(1 for c in cases if c.expected.get("scene") == "execute")
|
||
with_multi_write = sum(1 for c in cases if bool(c.expected.get("multi_write")))
|
||
valid = (
|
||
len(cases) >= 50
|
||
and unique_case_ids == len(cases)
|
||
and non_empty_messages == len(cases)
|
||
and with_duplicates > 0
|
||
and with_qa > 0
|
||
and with_execute > 0
|
||
and with_multi_write > 0
|
||
)
|
||
return {
|
||
"count": len(cases),
|
||
"unique_case_ids": unique_case_ids,
|
||
"non_empty_messages": non_empty_messages,
|
||
"duplicate_cases": with_duplicates,
|
||
"qa_cases": with_qa,
|
||
"execute_cases": with_execute,
|
||
"multi_write_cases": with_multi_write,
|
||
"valid": valid,
|
||
}
|
||
|
||
|
||
def _default_level_signal() -> dict[str, Any]:
|
||
return {
|
||
"build_intent_score": 0.2,
|
||
"scope_score": 0.2,
|
||
"timeline_score": 0.1,
|
||
"owner_score": 0.1,
|
||
"project_score": 0.2,
|
||
"level": "signal",
|
||
"level_confidence": 0.88,
|
||
"level_reasoning": "回放样本：明确执行动作，按 signal 处理。",
|
||
"should_split": False,
|
||
"split_items": [],
|
||
}
|
||
|
||
|
||
def _default_signal(project_id: int = 1001, confidence: float = 0.92) -> dict[str, Any]:
|
||
return {
|
||
"title": "新增信号源",
|
||
"type": "新需求",
|
||
"priority": "P1",
|
||
"matched_project_id": int(project_id),
|
||
"summary": "新增一个可执行的信号源需求。",
|
||
"reasoning": ["目标明确", "动作清晰"],
|
||
"confidence": float(confidence),
|
||
"recommended_action": "enter_only",
|
||
}
|
||
|
||
|
||
def _default_level_project() -> dict[str, Any]:
|
||
return {
|
||
"build_intent_score": 0.95,
|
||
"scope_score": 0.86,
|
||
"timeline_score": 0.8,
|
||
"owner_score": 0.75,
|
||
"project_score": 0.9,
|
||
"level": "project",
|
||
"level_confidence": 0.9,
|
||
"level_reasoning": "回放样本：范围较大，按 project 处理。",
|
||
"should_split": False,
|
||
"split_items": [],
|
||
}
|
||
|
||
|
||
def _default_project_prefill(index: int = 1) -> dict[str, Any]:
|
||
return {
|
||
"suggested_name": f"回放项目-{index}",
|
||
"suggested_description": "回放样本自动生成项目建议。",
|
||
"suggested_owner": "回放负责人",
|
||
"suggested_timeline": "2周",
|
||
"confidence": 0.86,
|
||
"reasoning": ["目标明确", "时间可控"],
|
||
}
|
||
|
||
|
||
def _default_routing_policy() -> dict[str, Any]:
|
||
return {
|
||
"cold_start_enabled": False,
|
||
"signal_auto_enter_threshold": 0.8,
|
||
"project_auto_create_threshold": 0.85,
|
||
"project_uncertain_threshold": 0.6,
|
||
"chat_context_auto_link_enabled": False,
|
||
}
|
||
|
||
|
||
def _case_by_id(cases: list[ReplayCase]) -> dict[str, ReplayCase]:
|
||
return {row.id: row for row in cases}
|
||
|
||
|
||
class ReplayPatcher:
|
||
"""在回放期间替换外部依赖（RAG/LLM）为样本驱动。"""
|
||
|
||
def __init__(self, cases: list[ReplayCase]) -> None:
|
||
self._case_map = _case_by_id(cases)
|
||
self._project_id_map: dict[int, int] = {1001: 1001, 1002: 1002, 1003: 1003}
|
||
self._orig_classify_intent = AdaptiveRAGService.classify_intent
|
||
self._orig_answer_question = AdaptiveRAGService.answer_question
|
||
self._orig_classify_level = FeishuSignalPipelineService._classify_level
|
||
self._orig_extract_signal = FeishuSignalPipelineService._extract_signal
|
||
self._orig_extract_project_prefill = FeishuSignalPipelineService._extract_project_prefill
|
||
self._orig_load_routing_policy = FeishuSignalPipelineService._load_routing_policy
|
||
self._orig_notify_project_created = FeishuSignalPipelineService._notify_project_created
|
||
self._orig_notify_signal_auto_entered = FeishuSignalPipelineService._notify_signal_auto_entered
|
||
|
||
def _current_case(self) -> ReplayCase:
|
||
case_id = CURRENT_CASE_ID.get()
|
||
if not case_id:
|
||
raise RuntimeError("missing_current_case_id")
|
||
row = self._case_map.get(case_id)
|
||
if row is None:
|
||
raise RuntimeError(f"unknown_case_id:{case_id}")
|
||
return row
|
||
|
||
def set_project_id_map(self, value: dict[int, int]) -> None:
|
||
mapped = {int(k): int(v) for k, v in (value or {}).items()}
|
||
if mapped:
|
||
self._project_id_map = mapped
|
||
|
||
def _map_project_id(self, raw_value: Any) -> Optional[int]:
|
||
if raw_value is None:
|
||
return None
|
||
try:
|
||
source = int(raw_value)
|
||
except Exception:
|
||
return None
|
||
return int(self._project_id_map.get(source, source))
|
||
|
||
def install(self) -> None:
|
||
async def fake_classify_intent(
|
||
_: AdaptiveRAGService,
|
||
message: str,
|
||
) -> dict[str, Any]:
|
||
case = self._current_case()
|
||
mock = case.mock
|
||
intent = str(mock.get("intent") or "statement").strip().lower()
|
||
if intent == "question":
|
||
return {
|
||
"intent": "question",
|
||
"question_part": message,
|
||
"statement_part": "",
|
||
}
|
||
if intent == "mixed":
|
||
question_part = str(mock.get("question_part") or message).strip()
|
||
statement_part = str(mock.get("statement_part") or message).strip()
|
||
return {
|
||
"intent": "mixed",
|
||
"question_part": question_part,
|
||
"statement_part": statement_part,
|
||
}
|
||
return {
|
||
"intent": "statement",
|
||
"question_part": "",
|
||
"statement_part": message,
|
||
}
|
||
|
||
async def fake_answer_question(
|
||
_: AdaptiveRAGService,
|
||
*,
|
||
question: str,
|
||
asked_by: Optional[int],
|
||
asked_via: str,
|
||
project_id: Optional[int],
|
||
source_chat_id: Optional[str],
|
||
session_id: Optional[int] = None,
|
||
) -> dict[str, Any]:
|
||
_ = (asked_by, asked_via, project_id, source_chat_id, session_id)
|
||
case = self._current_case()
|
||
mock = case.mock
|
||
answer = str(mock.get("qa_answer") or f"回放答案：{question[:30]}").strip()
|
||
sources = mock.get("qa_sources")
|
||
if not isinstance(sources, list):
|
||
sources = [{"label": "Replay Source"}]
|
||
return {
|
||
"answer": answer,
|
||
"sources": sources,
|
||
"query_type": "question",
|
||
"confidence": 0.9,
|
||
}
|
||
|
||
async def fake_classify_level(
|
||
_: FeishuSignalPipelineService,
|
||
*,
|
||
message: str,
|
||
projects: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
_ = (message, projects)
|
||
case = self._current_case()
|
||
if bool(case.mock.get("force_schema_error")):
|
||
raise ValueError("level_classification_schema_invalid: replay_forced")
|
||
level = case.mock.get("level")
|
||
if isinstance(level, dict):
|
||
return level
|
||
return _default_level_signal()
|
||
|
||
async def fake_extract_signal(
|
||
_: FeishuSignalPipelineService,
|
||
message: str,
|
||
*,
|
||
projects: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
_ = (message, projects)
|
||
case = self._current_case()
|
||
signal = case.mock.get("signal")
|
||
if isinstance(signal, dict):
|
||
payload = dict(signal)
|
||
mapped_project_id = self._map_project_id(payload.get("matched_project_id"))
|
||
if mapped_project_id is not None:
|
||
payload["matched_project_id"] = mapped_project_id
|
||
return payload
|
||
payload = _default_signal()
|
||
mapped_project_id = self._map_project_id(payload.get("matched_project_id"))
|
||
if mapped_project_id is not None:
|
||
payload["matched_project_id"] = mapped_project_id
|
||
return payload
|
||
|
||
async def fake_extract_project_prefill(
|
||
_: FeishuSignalPipelineService,
|
||
message: str,
|
||
) -> dict[str, Any]:
|
||
_ = message
|
||
case = self._current_case()
|
||
prefill = case.mock.get("project_prefill")
|
||
if isinstance(prefill, dict):
|
||
return prefill
|
||
return _default_project_prefill()
|
||
|
||
async def fake_load_routing_policy(_: FeishuSignalPipelineService) -> dict[str, Any]:
|
||
case = self._current_case()
|
||
policy = case.mock.get("routing_policy")
|
||
merged = _default_routing_policy()
|
||
if isinstance(policy, dict):
|
||
merged.update(policy)
|
||
return merged
|
||
|
||
async def fake_notify_project_created(
|
||
_: FeishuSignalPipelineService,
|
||
*,
|
||
project: Project,
|
||
) -> None:
|
||
_ = project
|
||
return None
|
||
|
||
async def fake_notify_signal_auto_entered(
|
||
_: FeishuSignalPipelineService,
|
||
*,
|
||
project_id: int,
|
||
signal: Any,
|
||
) -> None:
|
||
_ = (project_id, signal)
|
||
return None
|
||
|
||
AdaptiveRAGService.classify_intent = fake_classify_intent # type: ignore[assignment]
|
||
AdaptiveRAGService.answer_question = fake_answer_question # type: ignore[assignment]
|
||
FeishuSignalPipelineService._classify_level = fake_classify_level # type: ignore[assignment]
|
||
FeishuSignalPipelineService._extract_signal = fake_extract_signal # type: ignore[assignment]
|
||
FeishuSignalPipelineService._extract_project_prefill = fake_extract_project_prefill # type: ignore[assignment]
|
||
FeishuSignalPipelineService._load_routing_policy = fake_load_routing_policy # type: ignore[assignment]
|
||
FeishuSignalPipelineService._notify_project_created = fake_notify_project_created # type: ignore[assignment]
|
||
FeishuSignalPipelineService._notify_signal_auto_entered = fake_notify_signal_auto_entered # type: ignore[assignment]
|
||
|
||
def restore(self) -> None:
|
||
AdaptiveRAGService.classify_intent = self._orig_classify_intent # type: ignore[assignment]
|
||
AdaptiveRAGService.answer_question = self._orig_answer_question # type: ignore[assignment]
|
||
FeishuSignalPipelineService._classify_level = self._orig_classify_level # type: ignore[assignment]
|
||
FeishuSignalPipelineService._extract_signal = self._orig_extract_signal # type: ignore[assignment]
|
||
FeishuSignalPipelineService._extract_project_prefill = self._orig_extract_project_prefill # type: ignore[assignment]
|
||
FeishuSignalPipelineService._load_routing_policy = self._orig_load_routing_policy # type: ignore[assignment]
|
||
FeishuSignalPipelineService._notify_project_created = self._orig_notify_project_created # type: ignore[assignment]
|
||
FeishuSignalPipelineService._notify_signal_auto_entered = self._orig_notify_signal_auto_entered # type: ignore[assignment]
|
||
|
||
|
||
async def _seed_minimal_data(session: AsyncSession, *, tenant_id: int) -> dict[int, int]:
|
||
users = (
|
||
await session.execute(
|
||
select(User).where(
|
||
and_(
|
||
User.tenant_id == tenant_id,
|
||
User.username.in_(("replay_admin", "replay_owner")),
|
||
)
|
||
)
|
||
)
|
||
).scalars().all()
|
||
if not users:
|
||
session.add(
|
||
User(
|
||
tenant_id=tenant_id,
|
||
username="replay_admin",
|
||
email="replay_admin@example.com",
|
||
password_hash="x",
|
||
name="Replay Admin",
|
||
role="admin",
|
||
assigned_domains=[],
|
||
workload=0,
|
||
status="active",
|
||
is_active=True,
|
||
)
|
||
)
|
||
session.add(
|
||
User(
|
||
tenant_id=tenant_id,
|
||
username="replay_owner",
|
||
email="replay_owner@example.com",
|
||
password_hash="x",
|
||
name="Replay Owner",
|
||
role="ai_engineer",
|
||
assigned_domains=[],
|
||
workload=0,
|
||
status="active",
|
||
is_active=True,
|
||
)
|
||
)
|
||
await session.flush()
|
||
|
||
alias_to_name = {
|
||
1001: "回放项目-健身",
|
||
1002: "回放项目-路由",
|
||
1003: "回放项目-知识库",
|
||
}
|
||
project_map: dict[int, int] = {}
|
||
for alias, title in alias_to_name.items():
|
||
existing = (
|
||
await session.execute(
|
||
select(Project).where(
|
||
and_(
|
||
Project.tenant_id == tenant_id,
|
||
Project.name == title,
|
||
)
|
||
).limit(1)
|
||
)
|
||
).scalar_one_or_none()
|
||
if existing is None:
|
||
existing = Project(
|
||
tenant_id=tenant_id,
|
||
name=title,
|
||
title=title,
|
||
description="回放种子项目",
|
||
owner_id=None,
|
||
status="active",
|
||
domain_id=None,
|
||
assigned_role="ai_engineer",
|
||
signal_count=0,
|
||
)
|
||
session.add(existing)
|
||
await session.flush()
|
||
project_map[int(alias)] = int(existing.id)
|
||
await session.commit()
|
||
return project_map
|
||
|
||
|
||
async def _count_items_by_message_id(
|
||
session: AsyncSession,
|
||
*,
|
||
tenant_id: int,
|
||
message_id: str,
|
||
) -> int:
|
||
return int(
|
||
(
|
||
await session.execute(
|
||
select(func.count(FeishuItem.id)).where(
|
||
and_(
|
||
FeishuItem.tenant_id == tenant_id,
|
||
FeishuItem.feishu_message_id == message_id,
|
||
)
|
||
)
|
||
)
|
||
).scalar()
|
||
or 0
|
||
)
|
||
|
||
|
||
async def _latest_item_route_status(
|
||
session: AsyncSession,
|
||
*,
|
||
tenant_id: int,
|
||
message_id: str,
|
||
) -> Optional[str]:
|
||
row = (
|
||
await session.execute(
|
||
select(FeishuItem.route_status)
|
||
.where(
|
||
and_(
|
||
FeishuItem.tenant_id == tenant_id,
|
||
FeishuItem.feishu_message_id == message_id,
|
||
)
|
||
)
|
||
.order_by(FeishuItem.id.desc())
|
||
.limit(1)
|
||
)
|
||
).scalar_one_or_none()
|
||
if row is None:
|
||
return None
|
||
return str(row)
|
||
|
||
|
||
async def _cleanup_tenant_replay_data(session: AsyncSession, *, tenant_id: int) -> None:
|
||
await session.execute(delete(ProjectSignalLink).where(ProjectSignalLink.tenant_id == tenant_id))
|
||
await session.execute(delete(ProjectSuggestion).where(ProjectSuggestion.tenant_id == tenant_id))
|
||
await session.execute(delete(Signal).where(Signal.tenant_id == tenant_id))
|
||
await session.execute(
|
||
delete(AIProcessingLog).where(
|
||
and_(
|
||
AIProcessingLog.tenant_id == tenant_id,
|
||
AIProcessingLog.feishu_message_id.like("om_replay_%"),
|
||
)
|
||
)
|
||
)
|
||
await session.execute(
|
||
delete(AIProcessingLog).where(
|
||
and_(
|
||
AIProcessingLog.tenant_id == tenant_id,
|
||
AIProcessingLog.feishu_message_id.like("feishu-replay-%"),
|
||
)
|
||
)
|
||
)
|
||
await session.execute(delete(FeishuItem).where(FeishuItem.tenant_id == tenant_id))
|
||
await session.execute(delete(Notification).where(Notification.tenant_id == tenant_id))
|
||
await session.execute(delete(Project).where(Project.tenant_id == tenant_id))
|
||
await session.execute(delete(User).where(User.tenant_id == tenant_id))
|
||
await session.commit()
|
||
|
||
|
||
def _build_reply_text(
|
||
*,
|
||
service: FeishuIntegrationService,
|
||
message_content: str,
|
||
capture_result: dict[str, Any],
|
||
trace_id: str,
|
||
) -> Optional[str]:
|
||
scene = service._classify_message_scene(capture_result=capture_result)
|
||
if scene == "execute":
|
||
text = service._build_execution_receipt_reply(
|
||
message_content=message_content,
|
||
capture_result=capture_result,
|
||
)
|
||
return service._guard_execute_reply_text(text, trace_id=trace_id)
|
||
if scene == "qa":
|
||
if bool(capture_result.get("direct_reply_required")):
|
||
return str(capture_result.get("direct_reply_text") or "").strip()
|
||
return None
|
||
if scene == "error":
|
||
return service._build_error_reply_text()
|
||
return None
|
||
|
||
|
||
def _match_reply(expected: dict[str, Any], reply: Optional[str]) -> bool:
|
||
expected_equals = expected.get("reply_text")
|
||
if expected_equals is not None:
|
||
return str(reply or "") == str(expected_equals)
|
||
expected_contains = expected.get("reply_contains")
|
||
if expected_contains is None:
|
||
return True
|
||
if isinstance(expected_contains, str):
|
||
expected_contains = [expected_contains]
|
||
if not isinstance(expected_contains, list):
|
||
return True
|
||
text = str(reply or "")
|
||
return all(str(token) in text for token in expected_contains)
|
||
|
||
|
||
def _build_report(
|
||
*,
|
||
results: list[dict[str, Any]],
|
||
started_at: datetime,
|
||
) -> dict[str, Any]:
|
||
total = len(results)
|
||
passed = sum(1 for row in results if row.get("passed"))
|
||
failed = total - passed
|
||
return {
|
||
"version": "feishu-replay-v1",
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"started_at": started_at.isoformat(),
|
||
"metrics": {
|
||
"total": total,
|
||
"passed": passed,
|
||
"failed": failed,
|
||
"pass_rate": round((passed / total) if total else 0.0, 4),
|
||
},
|
||
"cases": results,
|
||
}
|
||
|
||
|
||
def _write_report(report: dict[str, Any], output_dir: Path) -> Path:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||
report_path = output_dir / f"feishu_replay_eval_{ts}.json"
|
||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
latest = output_dir / "feishu_replay_latest.json"
|
||
latest.write_text(report_path.read_text(encoding="utf-8"), encoding="utf-8")
|
||
return report_path
|
||
|
||
|
||
async def _run_replay(
|
||
cases: list[ReplayCase],
|
||
*,
|
||
tenant_id: int,
|
||
allow_tenant_1: bool = False,
|
||
) -> dict[str, Any]:
|
||
if int(tenant_id) == 1 and not allow_tenant_1:
|
||
raise RuntimeError("tenant_id=1 is blocked for replay cleanup; use dedicated replay tenant")
|
||
|
||
started_at = datetime.now(timezone.utc)
|
||
patcher = ReplayPatcher(cases)
|
||
patcher.install()
|
||
case_results: list[dict[str, Any]] = []
|
||
try:
|
||
async with AsyncSessionLocal() as db:
|
||
await _cleanup_tenant_replay_data(db, tenant_id=tenant_id)
|
||
project_id_map = await _seed_minimal_data(db, tenant_id=tenant_id)
|
||
patcher.set_project_id_map(project_id_map)
|
||
pipeline = FeishuSignalPipelineService(db, tenant_id=tenant_id)
|
||
integration = FeishuIntegrationService(db)
|
||
integration.tenant_id = tenant_id
|
||
|
||
for case in cases:
|
||
token = CURRENT_CASE_ID.set(case.id)
|
||
before_count = await _count_items_by_message_id(
|
||
db,
|
||
tenant_id=tenant_id,
|
||
message_id=case.feishu_message_id,
|
||
)
|
||
mismatches: list[str] = []
|
||
try:
|
||
result = await pipeline.process_message(
|
||
message_content=case.message_content,
|
||
sender=case.sender,
|
||
source_chat_id=case.source_chat_id,
|
||
source_sender_open_id="ou_replay_user",
|
||
feishu_message_id=case.feishu_message_id,
|
||
captured_at=datetime.now(timezone.utc),
|
||
)
|
||
except Exception as exc:
|
||
result = {"status": "error", "classification": "error", "error": str(exc)}
|
||
if str(case.expected.get("status") or "").strip().lower() != "error":
|
||
mismatches.append(f"process_exception:{exc}")
|
||
finally:
|
||
CURRENT_CASE_ID.reset(token)
|
||
|
||
after_count = await _count_items_by_message_id(
|
||
db,
|
||
tenant_id=tenant_id,
|
||
message_id=case.feishu_message_id,
|
||
)
|
||
item_delta = after_count - before_count
|
||
route_status = None
|
||
result_rows = result.get("results") if isinstance(result, dict) else None
|
||
result_count = len(result_rows) if isinstance(result_rows, list) else 0
|
||
result_levels = sorted(
|
||
{
|
||
str((row or {}).get("level") or "").strip().lower()
|
||
for row in (result_rows or [])
|
||
if isinstance(row, dict) and str((row or {}).get("level") or "").strip()
|
||
}
|
||
)
|
||
if isinstance(result_rows, list) and result_rows:
|
||
first = result_rows[0] if isinstance(result_rows[0], dict) else {}
|
||
route_status = str((first or {}).get("route_status") or "").strip() or None
|
||
if route_status is None:
|
||
route_status = await _latest_item_route_status(
|
||
db,
|
||
tenant_id=tenant_id,
|
||
message_id=case.feishu_message_id,
|
||
)
|
||
|
||
scene = integration._classify_message_scene(capture_result=result if isinstance(result, dict) else {})
|
||
reply_text = _build_reply_text(
|
||
service=integration,
|
||
message_content=case.message_content,
|
||
capture_result=result if isinstance(result, dict) else {},
|
||
trace_id=f"replay-{case.id}",
|
||
)
|
||
|
||
expected = case.expected
|
||
expected_status = expected.get("status")
|
||
if expected_status is not None and str(result.get("status")) != str(expected_status):
|
||
mismatches.append(f"status expected={expected_status} actual={result.get('status')}")
|
||
|
||
expected_classification = expected.get("classification")
|
||
if expected_classification is not None and str(result.get("classification")) != str(expected_classification):
|
||
mismatches.append(
|
||
f"classification expected={expected_classification} actual={result.get('classification')}"
|
||
)
|
||
|
||
expected_reason = expected.get("reason")
|
||
if expected_reason is not None and str(result.get("reason")) != str(expected_reason):
|
||
mismatches.append(f"reason expected={expected_reason} actual={result.get('reason')}")
|
||
|
||
expected_scene = expected.get("scene")
|
||
if expected_scene is not None and str(scene) != str(expected_scene):
|
||
mismatches.append(f"scene expected={expected_scene} actual={scene}")
|
||
|
||
expected_route_status = expected.get("route_status")
|
||
if expected_route_status is not None and str(route_status) != str(expected_route_status):
|
||
mismatches.append(f"route_status expected={expected_route_status} actual={route_status}")
|
||
|
||
expected_item_delta = expected.get("item_delta")
|
||
if expected_item_delta is not None and int(item_delta) != int(expected_item_delta):
|
||
mismatches.append(f"item_delta expected={expected_item_delta} actual={item_delta}")
|
||
|
||
expected_result_count = expected.get("result_count")
|
||
if expected_result_count is not None and int(result_count) != int(expected_result_count):
|
||
mismatches.append(f"result_count expected={expected_result_count} actual={result_count}")
|
||
|
||
expected_levels = expected.get("result_levels")
|
||
if isinstance(expected_levels, list):
|
||
expected_levels_set = sorted({str(x).strip().lower() for x in expected_levels if str(x).strip()})
|
||
if expected_levels_set != result_levels:
|
||
mismatches.append(f"result_levels expected={expected_levels_set} actual={result_levels}")
|
||
|
||
if bool(expected.get("multi_write")):
|
||
if result_count < 2:
|
||
mismatches.append(f"multi_write expected=true actual_result_count={result_count}")
|
||
if len(result_levels) < 2:
|
||
mismatches.append(f"multi_write expected mixed levels actual_levels={result_levels}")
|
||
|
||
if not _match_reply(expected, reply_text):
|
||
mismatches.append(f"reply mismatch expected={expected.get('reply_text') or expected.get('reply_contains')} actual={reply_text}")
|
||
|
||
case_results.append(
|
||
{
|
||
"id": case.id,
|
||
"passed": len(mismatches) == 0,
|
||
"mismatches": mismatches,
|
||
"expected": expected,
|
||
"actual": {
|
||
"status": result.get("status"),
|
||
"classification": result.get("classification"),
|
||
"reason": result.get("reason"),
|
||
"scene": scene,
|
||
"route_status": route_status,
|
||
"item_delta": item_delta,
|
||
"result_count": result_count,
|
||
"result_levels": result_levels,
|
||
"reply_text": reply_text,
|
||
},
|
||
}
|
||
)
|
||
|
||
await _cleanup_tenant_replay_data(db, tenant_id=tenant_id)
|
||
finally:
|
||
patcher.restore()
|
||
|
||
return _build_report(results=case_results, started_at=started_at)
|
||
|
||
|
||
async def _main_async(args: argparse.Namespace) -> int:
|
||
cases = _load_cases(Path(args.input))
|
||
if args.case_ids:
|
||
tokens = [token.strip() for token in str(args.case_ids).split(",") if token.strip()]
|
||
wanted = set(tokens)
|
||
filtered = [row for row in cases if row.id in wanted]
|
||
missing = sorted(wanted - {row.id for row in filtered})
|
||
if missing:
|
||
print(json.dumps({"error": "case_not_found", "missing": missing}, ensure_ascii=False, indent=2))
|
||
return 2
|
||
cases = filtered
|
||
lint = _lint_cases(cases)
|
||
if args.lint_only:
|
||
print(json.dumps({"lint": lint}, ensure_ascii=False, indent=2))
|
||
return 0 if lint["valid"] else 2
|
||
|
||
if not lint["valid"] and not args.case_ids:
|
||
print(json.dumps({"lint": lint}, ensure_ascii=False, indent=2))
|
||
return 2
|
||
|
||
tenant_id = int(args.tenant_id)
|
||
if tenant_id <= 0:
|
||
print(json.dumps({"error": "invalid_tenant_id", "tenant_id": args.tenant_id}, ensure_ascii=False, indent=2))
|
||
return 2
|
||
if tenant_id == 1 and not bool(args.allow_tenant_1):
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"error": "tenant_id_blocked",
|
||
"message": "tenant_id=1 默认禁用，避免误删主租户数据；请改用回放专用租户（例如 9901）。",
|
||
},
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
)
|
||
)
|
||
return 2
|
||
|
||
report = await _run_replay(
|
||
cases,
|
||
tenant_id=tenant_id,
|
||
allow_tenant_1=bool(args.allow_tenant_1),
|
||
)
|
||
report_path = _write_report(report, Path(args.output_dir))
|
||
metrics = report.get("metrics", {})
|
||
print(
|
||
f"[feishu-replay] total={metrics.get('total')} passed={metrics.get('passed')} "
|
||
f"failed={metrics.get('failed')} pass_rate={metrics.get('pass_rate')} "
|
||
f"report={report_path}"
|
||
)
|
||
if int(metrics.get("failed") or 0) > 0 and args.strict:
|
||
return 2
|
||
return 0
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(description="Feishu replay eval")
|
||
parser.add_argument(
|
||
"--input",
|
||
default="backend/evals/golden_feishu_replay_v1.jsonl",
|
||
help="Replay dataset path (.jsonl)",
|
||
)
|
||
parser.add_argument(
|
||
"--output-dir",
|
||
default="backend/evals/reports",
|
||
help="Report output directory",
|
||
)
|
||
parser.add_argument(
|
||
"--tenant-id",
|
||
default=str(DEFAULT_REPLAY_TENANT_ID),
|
||
help="Replay tenant id",
|
||
)
|
||
parser.add_argument(
|
||
"--allow-tenant-1",
|
||
action="store_true",
|
||
help="Allow tenant_id=1 (dangerous, only for isolated test db).",
|
||
)
|
||
parser.add_argument(
|
||
"--lint-only",
|
||
action="store_true",
|
||
help="Only validate dataset structure",
|
||
)
|
||
parser.add_argument(
|
||
"--strict",
|
||
action="store_true",
|
||
help="Exit non-zero when replay has failures",
|
||
)
|
||
parser.add_argument(
|
||
"--case-ids",
|
||
default="",
|
||
help="Only run selected case ids, comma separated",
|
||
)
|
||
return parser.parse_args()
|
||
|
||
|
||
def main() -> None:
|
||
args = parse_args()
|
||
code = asyncio.run(_main_async(args))
|
||
raise SystemExit(code)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|