152 lines
5.2 KiB
Python
152 lines
5.2 KiB
Python
"""Project understanding regression eval for AdaptiveRAGService.
|
|
|
|
Usage:
|
|
python backend/evals/rag_project_understanding_eval.py --tenant-id 1
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def _bootstrap_path() -> None:
|
|
this_file = Path(__file__).resolve()
|
|
backend_root = this_file.parents[1]
|
|
if str(backend_root) not in sys.path:
|
|
sys.path.insert(0, str(backend_root))
|
|
|
|
|
|
_bootstrap_path()
|
|
|
|
from app.database import AsyncSessionLocal # noqa: E402
|
|
from app.services.adaptive_rag_service import AdaptiveRAGService # noqa: E402
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class EvalCase:
|
|
question: str
|
|
expected_keywords: list[str]
|
|
|
|
|
|
CASES: list[EvalCase] = [
|
|
EvalCase(
|
|
question="智能配置功能是用来干嘛的？",
|
|
expected_keywords=["智能配置", "配置", "/ai-config", "知识库"],
|
|
),
|
|
EvalCase(
|
|
question="能力管理页面主要做什么？",
|
|
expected_keywords=["能力", "技能", "/skills", "推荐"],
|
|
),
|
|
EvalCase(
|
|
question="AI 管家页面有哪些用途？",
|
|
expected_keywords=["AI 管家", "/ai-steward", "冲突", "问答"],
|
|
),
|
|
EvalCase(
|
|
question="项目列表页面入口在哪，负责什么？",
|
|
expected_keywords=["项目", "/projects", "状态"],
|
|
),
|
|
EvalCase(
|
|
question="团队负载功能的目标是什么？",
|
|
expected_keywords=["负载", "/workloads", "调配"],
|
|
),
|
|
]
|
|
|
|
|
|
def _keyword_hit_ratio(text: str, keywords: list[str]) -> float:
|
|
haystack = str(text or "").lower()
|
|
if not keywords:
|
|
return 1.0
|
|
hit = 0
|
|
for kw in keywords:
|
|
if str(kw).strip().lower() in haystack:
|
|
hit += 1
|
|
return round(hit / max(len(keywords), 1), 4)
|
|
|
|
|
|
async def _run_eval(tenant_id: int) -> dict[str, Any]:
|
|
async with AsyncSessionLocal() as db:
|
|
service = AdaptiveRAGService(db, tenant_id=tenant_id)
|
|
system_snapshot = await service.context_cache.get_system_snapshot(db=db, tenant_id=tenant_id)
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
evidence_hits: list[float] = []
|
|
for case in CASES:
|
|
feature_matches = service._match_feature_catalog_entries(
|
|
question=case.question,
|
|
system_snapshot=system_snapshot,
|
|
limit=2,
|
|
)
|
|
direct_answer = service._build_snapshot_direct_answer(
|
|
case.question,
|
|
system_snapshot=system_snapshot,
|
|
user_snapshot={},
|
|
) or ""
|
|
retrieval = await service.multi_step_search(
|
|
question=case.question,
|
|
project_id=None,
|
|
source_chat_id=None,
|
|
session_state=None,
|
|
)
|
|
retrieval_text = "\n".join(
|
|
str(item.get("label") or "") + " " + str(item.get("metadata") or "")
|
|
for item in retrieval.get("items") or []
|
|
if isinstance(item, dict)
|
|
)
|
|
merged_text = f"{direct_answer}\n{retrieval_text}"
|
|
hit_ratio = _keyword_hit_ratio(merged_text, case.expected_keywords)
|
|
evidence_ratio = 0.0
|
|
if feature_matches:
|
|
top = feature_matches[0]
|
|
expected_evidence: list[str] = []
|
|
route = str(top.get("route") or "").strip()
|
|
page_file = str(top.get("page_file") or "").strip()
|
|
apis = top.get("apis") if isinstance(top.get("apis"), list) else []
|
|
if route:
|
|
expected_evidence.append(route)
|
|
if page_file:
|
|
expected_evidence.append(page_file)
|
|
for api in apis[:2]:
|
|
if str(api).strip():
|
|
expected_evidence.append(str(api).strip())
|
|
evidence_ratio = _keyword_hit_ratio(direct_answer, expected_evidence[:3]) if expected_evidence else 0.0
|
|
evidence_hits.append(evidence_ratio)
|
|
rows.append(
|
|
{
|
|
"question": case.question,
|
|
"expected_keywords": case.expected_keywords,
|
|
"keyword_hit_ratio": hit_ratio,
|
|
"feature_evidence_ratio": evidence_ratio,
|
|
"direct_answer_preview": direct_answer[:240],
|
|
"retrieval_item_count": len(retrieval.get("items") or []),
|
|
}
|
|
)
|
|
|
|
avg_score = round(sum(float(x["keyword_hit_ratio"]) for x in rows) / max(len(rows), 1), 4)
|
|
avg_feature_evidence_ratio = round(sum(evidence_hits) / max(len(evidence_hits), 1), 4)
|
|
return {
|
|
"tenant_id": tenant_id,
|
|
"case_count": len(rows),
|
|
"avg_keyword_hit_ratio": avg_score,
|
|
"avg_feature_evidence_ratio": avg_feature_evidence_ratio,
|
|
"cases": rows,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Run project understanding eval.")
|
|
parser.add_argument("--tenant-id", type=int, default=1, help="Tenant ID")
|
|
args = parser.parse_args()
|
|
|
|
result = asyncio.run(_run_eval(args.tenant_id))
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|