356 lines
13 KiB
Python
356 lines
13 KiB
Python
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from app.core.database import AsyncSessionLocal
|
|
from app.models import AICallLog, AIOrchestrationRun, AIProcessingLog, FeishuReplyOutbox
|
|
|
|
BASE = "http://127.0.0.1:8000"
|
|
CHAT_ID = "oc_c487d87563e48c2f9566ccd5a758212c"
|
|
SENDER = "P1验收机器人"
|
|
DEFAULT_LOG_PATH = Path(__file__).resolve().parent / "tmp_acceptance" / "consultant_dispatch_feishu_runs.jsonl"
|
|
TERMINAL_RUN_STATUSES = {"succeeded", "failed", "cancelled", "answered", "completed"}
|
|
|
|
|
|
def login_token() -> str:
|
|
data = urllib.parse.urlencode({"username": "admin", "password": "admin123"}).encode()
|
|
req = urllib.request.Request(
|
|
f"{BASE}/auth/login",
|
|
data=data,
|
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
payload = json.loads(resp.read().decode("utf-8"))
|
|
return str(payload.get("access_token") or "")
|
|
|
|
|
|
def post_capture(token: str, message: str, source_message_id: str) -> dict[str, Any]:
|
|
payload = {
|
|
"group_chat_id": CHAT_ID,
|
|
"message_content": message,
|
|
"sender": SENDER,
|
|
"feishu_message_id": source_message_id,
|
|
}
|
|
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
f"{BASE}/ai-steward/signal/capture-feishu",
|
|
data=body,
|
|
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=120) as resp:
|
|
text = resp.read().decode("utf-8")
|
|
try:
|
|
return json.loads(text)
|
|
except Exception:
|
|
return {"raw": text}
|
|
|
|
|
|
def detect_prompt_scene(prompt: str) -> Optional[str]:
|
|
p = str(prompt or "")
|
|
if "你是 PMFlow 的团队协作助手。" in p:
|
|
return "member_workload"
|
|
if "你是 PMFlow 的市场情报助手。" in p:
|
|
return "external_trend"
|
|
if "用户消息同时包含问题和需求。" in p:
|
|
return "mixed"
|
|
if "你是 PMFlow 的 AI 项目管理官。" in p:
|
|
return "ai_pmo"
|
|
if "用户通过飞书直接提问。" in p:
|
|
return "feishu_direct"
|
|
if "你是 PMFlow 的 RAG 问答助手" in p:
|
|
return "default"
|
|
return None
|
|
|
|
|
|
def infer_scene_from_reply(reply: str) -> str:
|
|
t = str(reply or "")
|
|
if "[回答]" in t and "[已收到需求]" in t:
|
|
return "mixed"
|
|
if "[行业动态]" in t and "[建议]" in t:
|
|
return "external_trend"
|
|
if ("负载" in t or "工作量" in t) and "%" in t and "项目列表" not in t:
|
|
return "member_workload"
|
|
if "完成率" in t:
|
|
return "default"
|
|
return "unknown"
|
|
|
|
|
|
def has_thinking_leak(text: str) -> bool:
|
|
for line in str(text or "").splitlines():
|
|
s = line.strip()
|
|
if not s:
|
|
continue
|
|
if len(s) <= 12 and s.startswith("我") and "先" in s:
|
|
return True
|
|
return False
|
|
|
|
|
|
def append_jsonl_log(log_path: str, payload: dict[str, Any]) -> str:
|
|
target = Path(log_path).expanduser()
|
|
if not target.is_absolute():
|
|
target = (Path.cwd() / target).resolve()
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with target.open("a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
|
return str(target)
|
|
|
|
|
|
def resolve_consultant_dispatch(payload: dict[str, Any]) -> Optional[dict[str, Any]]:
|
|
direct = payload.get("consultant_dispatch")
|
|
if isinstance(direct, dict):
|
|
return direct
|
|
rag_result = payload.get("rag_result")
|
|
if isinstance(rag_result, dict) and isinstance(rag_result.get("consultant_dispatch"), dict):
|
|
return dict(rag_result.get("consultant_dispatch") or {})
|
|
return None
|
|
|
|
|
|
async def fetch_max_ai_call_id(db) -> int:
|
|
row = (await db.execute(select(func.max(AICallLog.id)))).scalar()
|
|
return int(row or 0)
|
|
|
|
|
|
async def wait_run(db, source_message_id: str, timeout_sec: int = 720) -> Optional[AIOrchestrationRun]:
|
|
deadline = time.time() + timeout_sec
|
|
last = None
|
|
while time.time() < deadline:
|
|
row = (
|
|
await db.execute(
|
|
select(AIOrchestrationRun)
|
|
.where(AIOrchestrationRun.source_message_id == source_message_id)
|
|
.order_by(AIOrchestrationRun.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is not None:
|
|
last = row
|
|
st = str(row.status or "").lower()
|
|
if st in {"succeeded", "failed", "cancelled"}:
|
|
return row
|
|
await asyncio.sleep(2)
|
|
return last
|
|
|
|
|
|
async def wait_outbox(db, source_message_id: str, run_status: str, timeout_sec: int = 300) -> list[FeishuReplyOutbox]:
|
|
deadline = time.time() + timeout_sec
|
|
expected = 2 if run_status == "succeeded" else 1
|
|
latest: list[FeishuReplyOutbox] = []
|
|
while time.time() < deadline:
|
|
latest = (
|
|
await db.execute(
|
|
select(FeishuReplyOutbox)
|
|
.where(FeishuReplyOutbox.source_message_id == source_message_id)
|
|
.order_by(FeishuReplyOutbox.id.asc())
|
|
)
|
|
).scalars().all()
|
|
sent = [r for r in latest if str(r.status or "") == "sent"]
|
|
if len(sent) >= expected:
|
|
return latest
|
|
await asyncio.sleep(2)
|
|
return latest
|
|
|
|
|
|
async def wait_capture_artifacts(
|
|
db,
|
|
*,
|
|
source_message_id: str,
|
|
timeout_sec: int = 300,
|
|
) -> tuple[Optional[AIOrchestrationRun], list[FeishuReplyOutbox]]:
|
|
deadline = time.time() + timeout_sec
|
|
latest_run: Optional[AIOrchestrationRun] = None
|
|
latest_outbox: list[FeishuReplyOutbox] = []
|
|
while time.time() < deadline:
|
|
latest_run = (
|
|
await db.execute(
|
|
select(AIOrchestrationRun)
|
|
.where(AIOrchestrationRun.source_message_id == source_message_id)
|
|
.order_by(AIOrchestrationRun.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
latest_outbox = (
|
|
await db.execute(
|
|
select(FeishuReplyOutbox)
|
|
.where(FeishuReplyOutbox.source_message_id == source_message_id)
|
|
.order_by(FeishuReplyOutbox.id.asc())
|
|
)
|
|
).scalars().all()
|
|
sent_rows = [row for row in latest_outbox if str(row.status or "").lower() == "sent"]
|
|
run_status = str(getattr(latest_run, "status", "") or "").strip().lower()
|
|
if len(sent_rows) >= 2:
|
|
return latest_run, latest_outbox
|
|
if sent_rows and (run_status in TERMINAL_RUN_STATUSES or latest_run is None):
|
|
return latest_run, latest_outbox
|
|
await asyncio.sleep(2)
|
|
return latest_run, latest_outbox
|
|
|
|
|
|
async def fetch_processing(db, source_message_id: str) -> Optional[AIProcessingLog]:
|
|
return (
|
|
await db.execute(
|
|
select(AIProcessingLog)
|
|
.where(AIProcessingLog.feishu_message_id == source_message_id)
|
|
.order_by(AIProcessingLog.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
|
|
|
|
async def fetch_prompt_scene(db, min_call_id: int, keys: list[str]) -> tuple[str, list[dict[str, Any]]]:
|
|
rows = (
|
|
await db.execute(
|
|
select(AICallLog)
|
|
.where(
|
|
AICallLog.id > min_call_id,
|
|
AICallLog.function_name.in_(["adaptive_rag_answer_normal", "adaptive_rag_answer_deep"]),
|
|
)
|
|
.order_by(AICallLog.id.asc())
|
|
.limit(200)
|
|
)
|
|
).scalars().all()
|
|
|
|
lowered = [k.lower() for k in keys if k]
|
|
picked = "unknown"
|
|
candidates: list[dict[str, Any]] = []
|
|
|
|
for row in rows:
|
|
prompt = str(row.prompt or "")
|
|
pl = prompt.lower()
|
|
if lowered and not any(k in pl for k in lowered):
|
|
continue
|
|
scene = detect_prompt_scene(prompt)
|
|
candidates.append(
|
|
{
|
|
"id": int(row.id),
|
|
"scene": scene or "unknown",
|
|
"function": str(row.function_name or ""),
|
|
"created_at": str(row.created_at or ""),
|
|
}
|
|
)
|
|
if scene:
|
|
picked = scene
|
|
|
|
if picked == "unknown" and candidates:
|
|
picked = str(candidates[-1].get("scene") or "unknown")
|
|
|
|
return picked, candidates
|
|
|
|
|
|
def evaluate(idx: int, actual_scene: str, final_reply: str, run_status: str) -> tuple[bool, str]:
|
|
leak = has_thinking_leak(final_reply)
|
|
if idx == 1:
|
|
ok = actual_scene == "member_workload" and "%" in final_reply and "项目列表" not in final_reply and not leak
|
|
return ok, "member_workload+百分比+无项目列表"
|
|
if idx == 2:
|
|
sec = "[行业动态]" in final_reply and ("[对项目影响]" in final_reply or "[对项目的影响]" in final_reply) and "[建议]" in final_reply
|
|
src = "未接入实时数据" in final_reply or "训练数据" in final_reply or "数据来源" in final_reply
|
|
ok = actual_scene == "external_trend" and sec and src and not leak
|
|
return ok, "external_trend+三段输出+来源限制"
|
|
if idx == 3:
|
|
mixed = "[回答]" in final_reply and "[已收到需求]" in final_reply
|
|
prog = "进行中" in final_reply
|
|
ok = actual_scene == "mixed" and run_status == "succeeded" and mixed and prog and not leak
|
|
return ok, "mixed+创建成功+进行中数量"
|
|
if idx == 4:
|
|
ok = actual_scene == "default" and "完成率" in final_reply and "%" in final_reply and not leak
|
|
return ok, "default+项目状态完成率"
|
|
if idx == 5:
|
|
ok = actual_scene == "external_trend" and not leak
|
|
return ok, "external_trend+无思考外漏"
|
|
return False, "unknown"
|
|
|
|
|
|
async def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--idx", type=int, required=True)
|
|
parser.add_argument("--message", type=str, required=True)
|
|
parser.add_argument("--keys", type=str, default="")
|
|
parser.add_argument("--prefix", type=str, default="real-scene-p1-manual")
|
|
parser.add_argument("--log-path", type=str, default=str(DEFAULT_LOG_PATH))
|
|
args = parser.parse_args()
|
|
|
|
keys = [k.strip() for k in str(args.keys).split("|") if k.strip()]
|
|
source_message_id = f"{args.prefix}-{args.idx}-{int(time.time())}"
|
|
|
|
token = login_token()
|
|
if not token:
|
|
raise RuntimeError("token empty")
|
|
|
|
async with AsyncSessionLocal() as db:
|
|
before_call_id = await fetch_max_ai_call_id(db)
|
|
|
|
capture = post_capture(token, args.message, source_message_id)
|
|
|
|
async with AsyncSessionLocal() as db:
|
|
run_row, outbox_rows = await wait_capture_artifacts(db, source_message_id=source_message_id)
|
|
run_status = str(getattr(run_row, "status", "missing") or "missing") if run_row is not None else "missing"
|
|
processing = await fetch_processing(db, source_message_id)
|
|
if keys:
|
|
prompt_scene, candidates = await fetch_prompt_scene(db, before_call_id, keys)
|
|
else:
|
|
prompt_scene, candidates = "unknown", []
|
|
|
|
sent_rows = [r for r in outbox_rows if str(r.status or "") == "sent"]
|
|
ack = str(sent_rows[0].message_text or "").strip() if sent_rows else ""
|
|
final_reply = str(sent_rows[-1].message_text or "").strip() if sent_rows else ""
|
|
if not final_reply:
|
|
final_reply = str(capture.get("direct_reply_text") or "").strip()
|
|
|
|
inferred_scene = infer_scene_from_reply(final_reply)
|
|
actual_scene = prompt_scene if prompt_scene != "unknown" else inferred_scene
|
|
passed, pass_rule = evaluate(args.idx, actual_scene, final_reply, run_status)
|
|
|
|
payload = {
|
|
"generated_at": datetime.now().isoformat(),
|
|
"idx": args.idx,
|
|
"source_message_id": source_message_id,
|
|
"question": args.message,
|
|
"message": args.message,
|
|
"capture_status": str(capture.get("status") or ""),
|
|
"capture_direct_reply_text": str(capture.get("direct_reply_text") or ""),
|
|
"consultant_dispatch": resolve_consultant_dispatch(capture),
|
|
"run_status": run_status,
|
|
"ack": ack,
|
|
"final_reply": final_reply,
|
|
"prompt_scene": prompt_scene,
|
|
"inferred_scene": inferred_scene,
|
|
"actual_scene": actual_scene,
|
|
"thinking_leak": has_thinking_leak(final_reply),
|
|
"has_capacity_warning": "⚠ 产能提示" in final_reply,
|
|
"processing": {
|
|
"route": str(getattr(processing, "route", "") or ""),
|
|
"intent": str(getattr(processing, "intent", "") or ""),
|
|
"status": str(getattr(processing, "processing_status", "") or ""),
|
|
"duration_ms": int(getattr(processing, "duration_ms", 0) or 0),
|
|
"error_message": str(getattr(processing, "error_message", "") or ""),
|
|
"reply_source": str(getattr(processing, "reply_source", "") or ""),
|
|
},
|
|
"outbox": [
|
|
{
|
|
"id": int(r.id),
|
|
"status": str(r.status or ""),
|
|
"attempts": int(r.attempts or 0),
|
|
"created_at": str(r.created_at or ""),
|
|
"text": str(r.message_text or ""),
|
|
}
|
|
for r in outbox_rows
|
|
],
|
|
"call_log_candidates": candidates,
|
|
"pass": passed,
|
|
"pass_rule": pass_rule,
|
|
}
|
|
payload["log_path"] = append_jsonl_log(args.log_path, payload)
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|