"""Deterministic golden replay for the PMFlow AI turn runtime.""" from __future__ import annotations from dataclasses import dataclass from typing import Any from ..config.pmflow_ai_turn_runtime_golden_cases import PMFLOW_AI_TURN_RUNTIME_GOLDEN_CASES from ..schemas.rag import RAGAskRequest from .pmflow_ai_turn_runtime_service import PMFlowAITurnRuntimeService @dataclass(frozen=True) class PMFlowAITurnRuntimeGoldenResult: case_id: str passed: bool failures: tuple[str, ...] observed: dict[str, Any] class PMFlowAITurnRuntimeGoldenReplay: """Run behavior-level golden cases against TurnState and the action loop.""" def __init__(self, db: Any, *, tenant_id: int) -> None: self.runtime = PMFlowAITurnRuntimeService(db, tenant_id=tenant_id) def run_all(self, cases: list[dict[str, Any]] | None = None) -> list[PMFlowAITurnRuntimeGoldenResult]: return [self.run_case(case) for case in list(cases or PMFLOW_AI_TURN_RUNTIME_GOLDEN_CASES)] def run_case(self, case: dict[str, Any]) -> PMFlowAITurnRuntimeGoldenResult: request = RAGAskRequest( question=str(case.get("question") or ""), surface_context=dict(case.get("surface_context") or {}), ) state = self.runtime.build_initial_turn_state( request=request, route_plan=dict(case.get("legacy_route_plan") or {}), conversation_context=dict(case.get("conversation_context") or {}), turn_hypothesis=dict(case.get("turn_hypothesis") or {}), ) decision = self.runtime.assess_sufficiency(state) observed = { "answer_shape": state.answer_state.answer_shape, "next_action": decision.next_action_type, "candidate_actions": [item.action_type for item in state.action_state.candidate_actions], "must_ask_user": decision.must_ask_user, "can_answer_with_assumptions": state.sufficiency_state.can_answer_with_assumptions, "needs_external": state.sufficiency_state.needs_external_evidence, "needs_internal": state.sufficiency_state.needs_internal_evidence, "primary_binding": (state.object_state.primary_object.binding_strength if state.object_state.primary_object else ""), "context_titles": [item.title for item in state.object_state.context_objects if item.title], "route_hypothesis_only": bool(state.route_plan_input.get("route_hypothesis_only")), } failures = tuple(_case_failures(expect=dict(case.get("expect") or {}), observed=observed)) return PMFlowAITurnRuntimeGoldenResult( case_id=str(case.get("case_id") or "unnamed"), passed=not failures, failures=failures, observed=observed, ) def _case_failures(*, expect: dict[str, Any], observed: dict[str, Any]) -> list[str]: failures: list[str] = [] for key in ("answer_shape", "next_action", "must_ask_user", "can_answer_with_assumptions", "needs_external", "needs_internal", "primary_binding"): if key in expect and observed.get(key) != expect.get(key): failures.append(f"{key}: expected {expect.get(key)!r}, got {observed.get(key)!r}") for action in list(expect.get("actions") or []): if action not in list(observed.get("candidate_actions") or []): failures.append(f"candidate_actions missing {action!r}: got {observed.get('candidate_actions')!r}") for action in list(expect.get("forbid_actions") or []): if action in list(observed.get("candidate_actions") or []): failures.append(f"candidate_actions should not contain {action!r}: got {observed.get('candidate_actions')!r}") for title in list(expect.get("context_titles") or []): if title not in list(observed.get("context_titles") or []): failures.append(f"context_titles missing {title!r}: got {observed.get('context_titles')!r}") if observed.get("route_hypothesis_only") is not True: failures.append("route_plan_input must stay hypothesis-only") return failures __all__ = [ "PMFlowAITurnRuntimeGoldenReplay", "PMFlowAITurnRuntimeGoldenResult", ]