54 lines
2.8 KiB
Python
54 lines
2.8 KiB
Python
"""Replay runner for PMFlow AI context retention contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from ..config.pmflow_ai_context_replay_cases import PMFLOW_AI_CONTEXT_REPLAY_CASES
|
|
from .pmflow_ai_context_pack_service import PMFlowAIContextPackBuilder, evaluate_context_replay_case
|
|
|
|
|
|
class PMFlowAIContextReplayRunner:
|
|
"""Run deterministic context-pack replay cases."""
|
|
|
|
def run_cases(self, cases: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
rows = []
|
|
source_cases = cases if isinstance(cases, list) else PMFLOW_AI_CONTEXT_REPLAY_CASES
|
|
for case in source_cases:
|
|
if not isinstance(case, dict):
|
|
continue
|
|
rows.append(self.run_case(case))
|
|
return {
|
|
"packet_version": "pmflow_ai_context_replay_runner.v1",
|
|
"case_count": len(rows),
|
|
"passed_count": len([row for row in rows if row.get("passed")]),
|
|
"failed_count": len([row for row in rows if not row.get("passed")]),
|
|
"cases": rows,
|
|
}
|
|
|
|
def run_case(self, case: dict[str, Any]) -> dict[str, Any]:
|
|
builder = PMFlowAIContextPackBuilder(budget_chars=int(case.get("budget_chars") or 12000))
|
|
context_pack = builder.build(
|
|
user_question=str(case.get("user_question") or ""),
|
|
thread_context_pack=case.get("thread_context_pack") if isinstance(case.get("thread_context_pack"), dict) else {},
|
|
conversation_context=case.get("conversation_context") if isinstance(case.get("conversation_context"), dict) else {},
|
|
tenant_project_context=case.get("tenant_project_context") if isinstance(case.get("tenant_project_context"), dict) else {},
|
|
memory_context_pack=case.get("memory_context_pack") if isinstance(case.get("memory_context_pack"), dict) else {},
|
|
)
|
|
expected = [str(item) for item in list(case.get("expected_evidence") or []) if str(item or "").strip()]
|
|
forbidden = [str(item) for item in list(case.get("forbidden_evidence") or []) if str(item or "").strip()]
|
|
evaluation = evaluate_context_replay_case(context_pack=context_pack, expected_evidence=expected)
|
|
haystack = str(context_pack or "")
|
|
forbidden_checks = [{"forbidden": item[:160], "absent": item not in haystack} for item in forbidden]
|
|
passed = bool(evaluation.get("passed")) and all(item["absent"] for item in forbidden_checks)
|
|
return {
|
|
"case_id": str(case.get("case_id") or "")[:120],
|
|
"passed": passed,
|
|
"expected_checks": list(evaluation.get("checks") or []),
|
|
"forbidden_checks": forbidden_checks,
|
|
"observability": evaluation.get("observability") if isinstance(evaluation.get("observability"), dict) else {},
|
|
}
|
|
|
|
|
|
__all__ = ["PMFlowAIContextReplayRunner"]
|