"""Replay runner for PMFlow AI context retention contracts.""" from __future__ import annotations from typing import Any from ..config.pmflow_ai_context_replay_cases import PMFLOW_AI_CONTEXT_REPLAY_CASES from .pmflow_ai_context_pack_service import PMFlowAIContextPackBuilder, evaluate_context_replay_case class PMFlowAIContextReplayRunner: """Run deterministic context-pack replay cases.""" def run_cases(self, cases: list[dict[str, Any]] | None = None) -> dict[str, Any]: rows = [] source_cases = cases if isinstance(cases, list) else PMFLOW_AI_CONTEXT_REPLAY_CASES for case in source_cases: if not isinstance(case, dict): continue rows.append(self.run_case(case)) return { "packet_version": "pmflow_ai_context_replay_runner.v1", "case_count": len(rows), "passed_count": len([row for row in rows if row.get("passed")]), "failed_count": len([row for row in rows if not row.get("passed")]), "cases": rows, } def run_case(self, case: dict[str, Any]) -> dict[str, Any]: builder = PMFlowAIContextPackBuilder(budget_chars=int(case.get("budget_chars") or 12000)) context_pack = builder.build( user_question=str(case.get("user_question") or ""), thread_context_pack=case.get("thread_context_pack") if isinstance(case.get("thread_context_pack"), dict) else {}, conversation_context=case.get("conversation_context") if isinstance(case.get("conversation_context"), dict) else {}, tenant_project_context=case.get("tenant_project_context") if isinstance(case.get("tenant_project_context"), dict) else {}, memory_context_pack=case.get("memory_context_pack") if isinstance(case.get("memory_context_pack"), dict) else {}, ) expected = [str(item) for item in list(case.get("expected_evidence") or []) if str(item or "").strip()] forbidden = [str(item) for item in list(case.get("forbidden_evidence") or []) if str(item or "").strip()] evaluation = evaluate_context_replay_case(context_pack=context_pack, expected_evidence=expected) haystack = str(context_pack or "") forbidden_checks = [{"forbidden": item[:160], "absent": item not in haystack} for item in forbidden] passed = bool(evaluation.get("passed")) and all(item["absent"] for item in forbidden_checks) return { "case_id": str(case.get("case_id") or "")[:120], "passed": passed, "expected_checks": list(evaluation.get("checks") or []), "forbidden_checks": forbidden_checks, "observability": evaluation.get("observability") if isinstance(evaluation.get("observability"), dict) else {}, } __all__ = ["PMFlowAIContextReplayRunner"]