import argparse import asyncio import json import time import urllib.parse import urllib.request from datetime import datetime from pathlib import Path from typing import Any, Optional from sqlalchemy import func, select from app.core.database import AsyncSessionLocal from app.models import AICallLog, AIOrchestrationRun, AIProcessingLog, FeishuReplyOutbox BASE = "http://127.0.0.1:8000" CHAT_ID = "oc_c487d87563e48c2f9566ccd5a758212c" SENDER = "P1验收机器人" BANNED_PREFIXES = ["让我先", "让我尝试", "我来看看", "我先查", "让我看看", "我需要先"] DEFAULT_LOG_PATH = Path(__file__).resolve().parent / "tmp_acceptance" / "consultant_dispatch_feishu_runs.jsonl" TERMINAL_RUN_STATUSES = {"succeeded", "failed", "cancelled", "answered", "completed"} def login_token() -> str: data = urllib.parse.urlencode({"username": "admin", "password": "admin123"}).encode() req = urllib.request.Request( f"{BASE}/auth/login", data=data, headers={"Content-Type": "application/x-www-form-urlencoded"}, ) with urllib.request.urlopen(req, timeout=30) as resp: payload = json.loads(resp.read().decode("utf-8")) return str(payload.get("access_token") or "") def post_capture(token: str, message: str, source_message_id: str) -> dict[str, Any]: payload = { "group_chat_id": CHAT_ID, "message_content": message, "sender": SENDER, "feishu_message_id": source_message_id, } body = json.dumps(payload, ensure_ascii=False).encode("utf-8") req = urllib.request.Request( f"{BASE}/ai-steward/signal/capture-feishu", data=body, headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, ) with urllib.request.urlopen(req, timeout=120) as resp: text = resp.read().decode("utf-8") try: return json.loads(text) except Exception: return {"raw": text} def detect_prompt_scene(prompt: str) -> Optional[str]: p = str(prompt or "") if "你是 PMFlow 的团队协作助手。" in p: return "member_workload" if "你是 PMFlow 的市场情报助手。" in p: return "external_trend" if "用户消息同时包含问题和需求。" in p: return "mixed" if "你是 PMFlow 的 AI 项目管理官。" in p: return "ai_pmo" if "用户通过飞书直接提问。" in p: return "feishu_direct" if "你是 PMFlow 的 RAG 问答助手" in p: return "default" return None def infer_scene_from_reply(reply: str) -> str: t = str(reply or "") if "[回答]" in t and "[已收到需求]" in t: return "mixed" if "[行业动态]" in t and "[建议]" in t: return "external_trend" if ("负载" in t or "工作量" in t) and "%" in t and "项目列表" not in t: return "member_workload" if "完成率" in t: return "default" return "unknown" def has_thinking_leak(text: str) -> bool: for line in str(text or "").splitlines(): s = line.strip() if not s: continue if any(s.startswith(prefix) for prefix in BANNED_PREFIXES): return True return False def append_jsonl_log(log_path: str, payload: dict[str, Any]) -> str: target = Path(log_path).expanduser() if not target.is_absolute(): target = (Path.cwd() / target).resolve() target.parent.mkdir(parents=True, exist_ok=True) with target.open("a", encoding="utf-8") as fh: fh.write(json.dumps(payload, ensure_ascii=False) + "\n") return str(target) def resolve_consultant_dispatch(payload: dict[str, Any]) -> Optional[dict[str, Any]]: direct = payload.get("consultant_dispatch") if isinstance(direct, dict): return direct rag_result = payload.get("rag_result") if isinstance(rag_result, dict) and isinstance(rag_result.get("consultant_dispatch"), dict): return dict(rag_result.get("consultant_dispatch") or {}) return None async def fetch_max_ai_call_id(db) -> int: row = (await db.execute(select(func.max(AICallLog.id)))).scalar() return int(row or 0) async def wait_run(db, source_message_id: str, timeout_sec: int = 720) -> Optional[AIOrchestrationRun]: deadline = time.time() + timeout_sec last = None while time.time() < deadline: row = ( await db.execute( select(AIOrchestrationRun) .where(AIOrchestrationRun.source_message_id == source_message_id) .order_by(AIOrchestrationRun.id.desc()) .limit(1) ) ).scalar_one_or_none() if row is not None: last = row st = str(row.status or "").lower() if st in {"succeeded", "failed", "cancelled"}: return row await asyncio.sleep(2) return last async def wait_outbox(db, source_message_id: str, run_status: str, timeout_sec: int = 300) -> list[FeishuReplyOutbox]: deadline = time.time() + timeout_sec expected = 2 if run_status == "succeeded" else 1 latest: list[FeishuReplyOutbox] = [] while time.time() < deadline: latest = ( await db.execute( select(FeishuReplyOutbox) .where(FeishuReplyOutbox.source_message_id == source_message_id) .order_by(FeishuReplyOutbox.id.asc()) ) ).scalars().all() sent = [r for r in latest if str(r.status or "") == "sent"] if len(sent) >= expected: return latest await asyncio.sleep(2) return latest async def wait_capture_artifacts( db, *, source_message_id: str, timeout_sec: int = 300, ) -> tuple[Optional[AIOrchestrationRun], list[FeishuReplyOutbox]]: deadline = time.time() + timeout_sec latest_run: Optional[AIOrchestrationRun] = None latest_outbox: list[FeishuReplyOutbox] = [] while time.time() < deadline: latest_run = ( await db.execute( select(AIOrchestrationRun) .where(AIOrchestrationRun.source_message_id == source_message_id) .order_by(AIOrchestrationRun.id.desc()) .limit(1) ) ).scalar_one_or_none() latest_outbox = ( await db.execute( select(FeishuReplyOutbox) .where(FeishuReplyOutbox.source_message_id == source_message_id) .order_by(FeishuReplyOutbox.id.asc()) ) ).scalars().all() sent_rows = [row for row in latest_outbox if str(row.status or "").lower() == "sent"] run_status = str(getattr(latest_run, "status", "") or "").strip().lower() if len(sent_rows) >= 2: return latest_run, latest_outbox if sent_rows and (run_status in TERMINAL_RUN_STATUSES or latest_run is None): return latest_run, latest_outbox await asyncio.sleep(2) return latest_run, latest_outbox async def fetch_processing(db, source_message_id: str) -> Optional[AIProcessingLog]: return ( await db.execute( select(AIProcessingLog) .where(AIProcessingLog.feishu_message_id == source_message_id) .order_by(AIProcessingLog.id.desc()) .limit(1) ) ).scalar_one_or_none() async def fetch_prompt_scene(db, min_call_id: int, keys: list[str]) -> tuple[str, list[dict[str, Any]]]: rows = ( await db.execute( select(AICallLog) .where( AICallLog.id > min_call_id, AICallLog.function_name.in_(["adaptive_rag_answer_normal", "adaptive_rag_answer_deep"]), ) .order_by(AICallLog.id.asc()) .limit(200) ) ).scalars().all() lowered = [k.lower() for k in keys if k] picked = "unknown" candidates: list[dict[str, Any]] = [] for row in rows: prompt = str(row.prompt or "") pl = prompt.lower() if lowered and not any(k in pl for k in lowered): continue scene = detect_prompt_scene(prompt) candidates.append( { "id": int(row.id), "scene": scene or "unknown", "function": str(row.function_name or ""), "created_at": str(row.created_at or ""), } ) if scene: picked = scene if picked == "unknown" and candidates: picked = str(candidates[-1].get("scene") or "unknown") return picked, candidates def evaluate(idx: int, actual_scene: str, final_reply: str, run_status: str) -> tuple[bool, str]: leak = has_thinking_leak(final_reply) if idx == 1: ok = actual_scene == "member_workload" and "%" in final_reply and "项目列表" not in final_reply and not leak return ok, "member_workload+百分比+无项目列表" if idx == 2: sec = "[行业动态]" in final_reply and ("[对项目影响]" in final_reply or "[对项目的影响]" in final_reply) and "[建议]" in final_reply src = "未接入实时数据" in final_reply or "训练数据" in final_reply or "数据来源" in final_reply ok = actual_scene == "external_trend" and sec and src and not leak return ok, "external_trend+三段输出+来源限制" if idx == 3: mixed = "[回答]" in final_reply and "[已收到需求]" in final_reply prog = "进行中" in final_reply ok = actual_scene == "mixed" and run_status == "succeeded" and mixed and prog and not leak return ok, "mixed+创建成功+进行中数量" if idx == 4: ok = actual_scene == "default" and "完成率" in final_reply and "%" in final_reply and not leak return ok, "default+项目状态完成率" if idx == 5: ok = actual_scene == "external_trend" and not leak return ok, "external_trend+无思考外漏" return False, "unknown" async def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--idx", type=int, required=True) parser.add_argument("--message", type=str, required=True) parser.add_argument("--keys", type=str, default="") parser.add_argument("--prefix", type=str, default="real-scene-p1-manual") parser.add_argument("--log-path", type=str, default=str(DEFAULT_LOG_PATH)) args = parser.parse_args() keys = [k.strip() for k in str(args.keys).split("|") if k.strip()] source_message_id = f"{args.prefix}-{args.idx}-{int(time.time())}" token = login_token() if not token: raise RuntimeError("token empty") async with AsyncSessionLocal() as db: before_call_id = await fetch_max_ai_call_id(db) capture = post_capture(token, args.message, source_message_id) async with AsyncSessionLocal() as db: run_row, outbox_rows = await wait_capture_artifacts(db, source_message_id=source_message_id) run_status = str(getattr(run_row, "status", "missing") or "missing") if run_row is not None else "missing" processing = await fetch_processing(db, source_message_id) if keys: prompt_scene, candidates = await fetch_prompt_scene(db, before_call_id, keys) else: prompt_scene, candidates = "unknown", [] sent_rows = [r for r in outbox_rows if str(r.status or "") == "sent"] ack = str(sent_rows[0].message_text or "").strip() if sent_rows else "" final_reply = str(sent_rows[-1].message_text or "").strip() if sent_rows else "" if not final_reply: final_reply = str(capture.get("direct_reply_text") or "").strip() inferred_scene = infer_scene_from_reply(final_reply) actual_scene = prompt_scene if prompt_scene != "unknown" else inferred_scene passed, pass_rule = evaluate(args.idx, actual_scene, final_reply, run_status) payload = { "generated_at": datetime.now().isoformat(), "idx": args.idx, "source_message_id": source_message_id, "question": args.message, "message": args.message, "capture_status": str(capture.get("status") or ""), "capture_direct_reply_text": str(capture.get("direct_reply_text") or ""), "consultant_dispatch": resolve_consultant_dispatch(capture), "run_status": run_status, "ack": ack, "final_reply": final_reply, "prompt_scene": prompt_scene, "inferred_scene": inferred_scene, "actual_scene": actual_scene, "thinking_leak": has_thinking_leak(final_reply), "has_capacity_warning": "⚠ 产能提示" in final_reply, "processing": { "route": str(getattr(processing, "route", "") or ""), "intent": str(getattr(processing, "intent", "") or ""), "status": str(getattr(processing, "processing_status", "") or ""), "duration_ms": int(getattr(processing, "duration_ms", 0) or 0), "error_message": str(getattr(processing, "error_message", "") or ""), "reply_source": str(getattr(processing, "reply_source", "") or ""), }, "outbox": [ { "id": int(r.id), "status": str(r.status or ""), "attempts": int(r.attempts or 0), "created_at": str(r.created_at or ""), "text": str(r.message_text or ""), } for r in outbox_rows ], "call_log_candidates": candidates, "pass": passed, "pass_rule": pass_rule, } payload["log_path"] = append_jsonl_log(args.log_path, payload) print(json.dumps(payload, ensure_ascii=False)) if __name__ == "__main__": asyncio.run(main())