292 lines
11 KiB
Python
292 lines
11 KiB
Python
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import datetime
|
|
from typing import Any, Optional
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from app.core.database import AsyncSessionLocal
|
|
from app.models import AICallLog, AIOrchestrationRun, AIProcessingLog, FeishuReplyOutbox
|
|
|
|
BASE = "http://127.0.0.1:8000"
|
|
CHAT_ID = "oc_c487d87563e48c2f9566ccd5a758212c"
|
|
SENDER = "P1验收机器人"
|
|
BANNED_PREFIXES = ["让我先", "让我尝试", "我来看看", "我先查", "让我看看", "我需要先"]
|
|
|
|
|
|
def login_token() -> str:
|
|
data = urllib.parse.urlencode({"username": "admin", "password": "admin123"}).encode()
|
|
req = urllib.request.Request(
|
|
f"{BASE}/auth/login",
|
|
data=data,
|
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
payload = json.loads(resp.read().decode("utf-8"))
|
|
return str(payload.get("access_token") or "")
|
|
|
|
|
|
def post_capture(token: str, message: str, source_message_id: str) -> dict[str, Any]:
|
|
payload = {
|
|
"group_chat_id": CHAT_ID,
|
|
"message_content": message,
|
|
"sender": SENDER,
|
|
"feishu_message_id": source_message_id,
|
|
}
|
|
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
f"{BASE}/ai-steward/ai-steward/signal/capture-feishu",
|
|
data=body,
|
|
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=120) as resp:
|
|
text = resp.read().decode("utf-8")
|
|
try:
|
|
return json.loads(text)
|
|
except Exception:
|
|
return {"raw": text}
|
|
|
|
|
|
def detect_prompt_scene(prompt: str) -> Optional[str]:
|
|
p = str(prompt or "")
|
|
if "你是 PMFlow 的团队协作助手。" in p:
|
|
return "member_workload"
|
|
if "你是 PMFlow 的市场情报助手。" in p:
|
|
return "external_trend"
|
|
if "用户消息同时包含问题和需求。" in p:
|
|
return "mixed"
|
|
if "你是 PMFlow 的 AI 项目管理官。" in p:
|
|
return "ai_pmo"
|
|
if "用户通过飞书直接提问。" in p:
|
|
return "feishu_direct"
|
|
if "你是 PMFlow 的 RAG 问答助手" in p:
|
|
return "default"
|
|
return None
|
|
|
|
|
|
def infer_scene_from_reply(reply: str) -> str:
|
|
t = str(reply or "")
|
|
if "[回答]" in t and "[已收到需求]" in t:
|
|
return "mixed"
|
|
if "[行业动态]" in t and "[建议]" in t:
|
|
return "external_trend"
|
|
if ("负载" in t or "工作量" in t) and "%" in t and "项目列表" not in t:
|
|
return "member_workload"
|
|
if "完成率" in t:
|
|
return "default"
|
|
return "unknown"
|
|
|
|
|
|
def has_thinking_leak(text: str) -> bool:
|
|
for line in str(text or "").splitlines():
|
|
s = line.strip()
|
|
if not s:
|
|
continue
|
|
if any(s.startswith(prefix) for prefix in BANNED_PREFIXES):
|
|
return True
|
|
return False
|
|
|
|
|
|
async def fetch_max_ai_call_id(db) -> int:
|
|
row = (await db.execute(select(func.max(AICallLog.id)))).scalar()
|
|
return int(row or 0)
|
|
|
|
|
|
async def wait_run(db, source_message_id: str, timeout_sec: int = 720) -> Optional[AIOrchestrationRun]:
|
|
deadline = time.time() + timeout_sec
|
|
last = None
|
|
while time.time() < deadline:
|
|
row = (
|
|
await db.execute(
|
|
select(AIOrchestrationRun)
|
|
.where(AIOrchestrationRun.source_message_id == source_message_id)
|
|
.order_by(AIOrchestrationRun.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
if row is not None:
|
|
last = row
|
|
st = str(row.status or "").lower()
|
|
if st in {"succeeded", "failed", "cancelled"}:
|
|
return row
|
|
await asyncio.sleep(2)
|
|
return last
|
|
|
|
|
|
async def wait_outbox(db, source_message_id: str, run_status: str, timeout_sec: int = 300) -> list[FeishuReplyOutbox]:
|
|
deadline = time.time() + timeout_sec
|
|
expected = 2 if run_status == "succeeded" else 1
|
|
latest: list[FeishuReplyOutbox] = []
|
|
while time.time() < deadline:
|
|
latest = (
|
|
await db.execute(
|
|
select(FeishuReplyOutbox)
|
|
.where(FeishuReplyOutbox.source_message_id == source_message_id)
|
|
.order_by(FeishuReplyOutbox.id.asc())
|
|
)
|
|
).scalars().all()
|
|
sent = [r for r in latest if str(r.status or "") == "sent"]
|
|
if len(sent) >= expected:
|
|
return latest
|
|
await asyncio.sleep(2)
|
|
return latest
|
|
|
|
|
|
async def fetch_processing(db, source_message_id: str) -> Optional[AIProcessingLog]:
|
|
return (
|
|
await db.execute(
|
|
select(AIProcessingLog)
|
|
.where(AIProcessingLog.feishu_message_id == source_message_id)
|
|
.order_by(AIProcessingLog.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
|
|
|
|
async def fetch_prompt_scene(db, min_call_id: int, keys: list[str]) -> tuple[str, list[dict[str, Any]]]:
|
|
rows = (
|
|
await db.execute(
|
|
select(AICallLog)
|
|
.where(
|
|
AICallLog.id > min_call_id,
|
|
AICallLog.function_name.in_(["adaptive_rag_answer_normal", "adaptive_rag_answer_deep"]),
|
|
)
|
|
.order_by(AICallLog.id.asc())
|
|
.limit(200)
|
|
)
|
|
).scalars().all()
|
|
|
|
lowered = [k.lower() for k in keys if k]
|
|
picked = "unknown"
|
|
candidates: list[dict[str, Any]] = []
|
|
|
|
for row in rows:
|
|
prompt = str(row.prompt or "")
|
|
pl = prompt.lower()
|
|
if lowered and not any(k in pl for k in lowered):
|
|
continue
|
|
scene = detect_prompt_scene(prompt)
|
|
candidates.append(
|
|
{
|
|
"id": int(row.id),
|
|
"scene": scene or "unknown",
|
|
"function": str(row.function_name or ""),
|
|
"created_at": str(row.created_at or ""),
|
|
}
|
|
)
|
|
if scene:
|
|
picked = scene
|
|
|
|
if picked == "unknown" and candidates:
|
|
picked = str(candidates[-1].get("scene") or "unknown")
|
|
|
|
return picked, candidates
|
|
|
|
|
|
def evaluate(idx: int, actual_scene: str, final_reply: str, run_status: str) -> tuple[bool, str]:
|
|
leak = has_thinking_leak(final_reply)
|
|
if idx == 1:
|
|
ok = actual_scene == "member_workload" and "%" in final_reply and "项目列表" not in final_reply and not leak
|
|
return ok, "member_workload+百分比+无项目列表"
|
|
if idx == 2:
|
|
sec = "[行业动态]" in final_reply and ("[对项目影响]" in final_reply or "[对项目的影响]" in final_reply) and "[建议]" in final_reply
|
|
src = "未接入实时数据" in final_reply or "训练数据" in final_reply or "数据来源" in final_reply
|
|
ok = actual_scene == "external_trend" and sec and src and not leak
|
|
return ok, "external_trend+三段输出+来源限制"
|
|
if idx == 3:
|
|
mixed = "[回答]" in final_reply and "[已收到需求]" in final_reply
|
|
prog = "进行中" in final_reply
|
|
ok = actual_scene == "mixed" and run_status == "succeeded" and mixed and prog and not leak
|
|
return ok, "mixed+创建成功+进行中数量"
|
|
if idx == 4:
|
|
ok = actual_scene == "default" and "完成率" in final_reply and "%" in final_reply and not leak
|
|
return ok, "default+项目状态完成率"
|
|
if idx == 5:
|
|
ok = actual_scene == "external_trend" and not leak
|
|
return ok, "external_trend+无思考外漏"
|
|
return False, "unknown"
|
|
|
|
|
|
async def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--idx", type=int, required=True)
|
|
parser.add_argument("--message", type=str, required=True)
|
|
parser.add_argument("--keys", type=str, default="")
|
|
parser.add_argument("--prefix", type=str, default="real-scene-p1-manual")
|
|
args = parser.parse_args()
|
|
|
|
keys = [k.strip() for k in str(args.keys).split("|") if k.strip()]
|
|
source_message_id = f"{args.prefix}-{args.idx}-{int(time.time())}"
|
|
|
|
token = login_token()
|
|
if not token:
|
|
raise RuntimeError("token empty")
|
|
|
|
async with AsyncSessionLocal() as db:
|
|
before_call_id = await fetch_max_ai_call_id(db)
|
|
|
|
capture = post_capture(token, args.message, source_message_id)
|
|
|
|
async with AsyncSessionLocal() as db:
|
|
run_row = await wait_run(db, source_message_id)
|
|
run_status = str(getattr(run_row, "status", "missing") or "missing") if run_row is not None else "missing"
|
|
outbox_rows = await wait_outbox(db, source_message_id, run_status)
|
|
processing = await fetch_processing(db, source_message_id)
|
|
prompt_scene, candidates = await fetch_prompt_scene(db, before_call_id, keys)
|
|
|
|
sent_rows = [r for r in outbox_rows if str(r.status or "") == "sent"]
|
|
ack = str(sent_rows[0].message_text or "").strip() if sent_rows else ""
|
|
final_reply = str(sent_rows[-1].message_text or "").strip() if sent_rows else ""
|
|
if not final_reply:
|
|
final_reply = str(capture.get("direct_reply_text") or "").strip()
|
|
|
|
inferred_scene = infer_scene_from_reply(final_reply)
|
|
actual_scene = prompt_scene if prompt_scene != "unknown" else inferred_scene
|
|
passed, pass_rule = evaluate(args.idx, actual_scene, final_reply, run_status)
|
|
|
|
payload = {
|
|
"generated_at": datetime.now().isoformat(),
|
|
"idx": args.idx,
|
|
"source_message_id": source_message_id,
|
|
"message": args.message,
|
|
"capture_status": str(capture.get("status") or ""),
|
|
"run_status": run_status,
|
|
"ack": ack,
|
|
"final_reply": final_reply,
|
|
"prompt_scene": prompt_scene,
|
|
"inferred_scene": inferred_scene,
|
|
"actual_scene": actual_scene,
|
|
"thinking_leak": has_thinking_leak(final_reply),
|
|
"has_capacity_warning": "⚠ 产能提示" in final_reply,
|
|
"processing": {
|
|
"route": str(getattr(processing, "route", "") or ""),
|
|
"intent": str(getattr(processing, "intent", "") or ""),
|
|
"status": str(getattr(processing, "processing_status", "") or ""),
|
|
"duration_ms": int(getattr(processing, "duration_ms", 0) or 0),
|
|
"error_message": str(getattr(processing, "error_message", "") or ""),
|
|
"reply_source": str(getattr(processing, "reply_source", "") or ""),
|
|
},
|
|
"outbox": [
|
|
{
|
|
"id": int(r.id),
|
|
"status": str(r.status or ""),
|
|
"attempts": int(r.attempts or 0),
|
|
"created_at": str(r.created_at or ""),
|
|
"text": str(r.message_text or ""),
|
|
}
|
|
for r in outbox_rows
|
|
],
|
|
"call_log_candidates": candidates,
|
|
"pass": passed,
|
|
"pass_rule": pass_rule,
|
|
}
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|