"""Project understanding regression eval for AdaptiveRAGService.

Usage:
  python backend/evals/rag_project_understanding_eval.py --tenant-id 1
"""

from __future__ import annotations

import argparse
import asyncio
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any


def _bootstrap_path() -> None:
    this_file = Path(__file__).resolve()
    backend_root = this_file.parents[1]
    if str(backend_root) not in sys.path:
        sys.path.insert(0, str(backend_root))


_bootstrap_path()

from app.database import AsyncSessionLocal  # noqa: E402
from app.services.adaptive_rag_service import AdaptiveRAGService  # noqa: E402


@dataclass(slots=True)
class EvalCase:
    question: str
    expected_keywords: list[str]


CASES: list[EvalCase] = [
    EvalCase(
        question="智能配置功能是用来干嘛的？",
        expected_keywords=["智能配置", "配置", "/ai-config", "知识库"],
    ),
    EvalCase(
        question="能力管理页面主要做什么？",
        expected_keywords=["能力", "技能", "/skills", "推荐"],
    ),
    EvalCase(
        question="AI 管家页面有哪些用途？",
        expected_keywords=["AI 管家", "/ai-steward", "冲突", "问答"],
    ),
    EvalCase(
        question="项目列表页面入口在哪，负责什么？",
        expected_keywords=["项目", "/projects", "状态"],
    ),
    EvalCase(
        question="团队负载功能的目标是什么？",
        expected_keywords=["负载", "/workloads", "调配"],
    ),
]


def _keyword_hit_ratio(text: str, keywords: list[str]) -> float:
    haystack = str(text or "").lower()
    if not keywords:
        return 1.0
    hit = 0
    for kw in keywords:
        if str(kw).strip().lower() in haystack:
            hit += 1
    return round(hit / max(len(keywords), 1), 4)


async def _run_eval(tenant_id: int) -> dict[str, Any]:
    async with AsyncSessionLocal() as db:
        service = AdaptiveRAGService(db, tenant_id=tenant_id)
        system_snapshot = await service.context_cache.get_system_snapshot(db=db, tenant_id=tenant_id)

        rows: list[dict[str, Any]] = []
        evidence_hits: list[float] = []
        for case in CASES:
            feature_matches = service._match_feature_catalog_entries(
                question=case.question,
                system_snapshot=system_snapshot,
                limit=2,
            )
            direct_answer = service._build_snapshot_direct_answer(
                case.question,
                system_snapshot=system_snapshot,
                user_snapshot={},
            ) or ""
            retrieval = await service.multi_step_search(
                question=case.question,
                project_id=None,
                source_chat_id=None,
                session_state=None,
            )
            retrieval_text = "\n".join(
                str(item.get("label") or "") + " " + str(item.get("metadata") or "")
                for item in retrieval.get("items") or []
                if isinstance(item, dict)
            )
            merged_text = f"{direct_answer}\n{retrieval_text}"
            hit_ratio = _keyword_hit_ratio(merged_text, case.expected_keywords)
            evidence_ratio = 0.0
            if feature_matches:
                top = feature_matches[0]
                expected_evidence: list[str] = []
                route = str(top.get("route") or "").strip()
                page_file = str(top.get("page_file") or "").strip()
                apis = top.get("apis") if isinstance(top.get("apis"), list) else []
                if route:
                    expected_evidence.append(route)
                if page_file:
                    expected_evidence.append(page_file)
                for api in apis[:2]:
                    if str(api).strip():
                        expected_evidence.append(str(api).strip())
                evidence_ratio = _keyword_hit_ratio(direct_answer, expected_evidence[:3]) if expected_evidence else 0.0
            evidence_hits.append(evidence_ratio)
            rows.append(
                {
                    "question": case.question,
                    "expected_keywords": case.expected_keywords,
                    "keyword_hit_ratio": hit_ratio,
                    "feature_evidence_ratio": evidence_ratio,
                    "direct_answer_preview": direct_answer[:240],
                    "retrieval_item_count": len(retrieval.get("items") or []),
                }
            )

        avg_score = round(sum(float(x["keyword_hit_ratio"]) for x in rows) / max(len(rows), 1), 4)
        avg_feature_evidence_ratio = round(sum(evidence_hits) / max(len(evidence_hits), 1), 4)
        return {
            "tenant_id": tenant_id,
            "case_count": len(rows),
            "avg_keyword_hit_ratio": avg_score,
            "avg_feature_evidence_ratio": avg_feature_evidence_ratio,
            "cases": rows,
        }


def main() -> None:
    parser = argparse.ArgumentParser(description="Run project understanding eval.")
    parser.add_argument("--tenant-id", type=int, default=1, help="Tenant ID")
    args = parser.parse_args()

    result = asyncio.run(_run_eval(args.tenant_id))
    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()