86 lines
4.1 KiB
Python
86 lines
4.1 KiB
Python
"""Deterministic golden replay for the PMFlow AI turn runtime."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
from ..config.pmflow_ai_turn_runtime_golden_cases import PMFLOW_AI_TURN_RUNTIME_GOLDEN_CASES
|
|
from ..schemas.rag import RAGAskRequest
|
|
from .pmflow_ai_turn_runtime_service import PMFlowAITurnRuntimeService
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PMFlowAITurnRuntimeGoldenResult:
|
|
case_id: str
|
|
passed: bool
|
|
failures: tuple[str, ...]
|
|
observed: dict[str, Any]
|
|
|
|
|
|
class PMFlowAITurnRuntimeGoldenReplay:
|
|
"""Run behavior-level golden cases against TurnState and the action loop."""
|
|
|
|
def __init__(self, db: Any, *, tenant_id: int) -> None:
|
|
self.runtime = PMFlowAITurnRuntimeService(db, tenant_id=tenant_id)
|
|
|
|
def run_all(self, cases: list[dict[str, Any]] | None = None) -> list[PMFlowAITurnRuntimeGoldenResult]:
|
|
return [self.run_case(case) for case in list(cases or PMFLOW_AI_TURN_RUNTIME_GOLDEN_CASES)]
|
|
|
|
def run_case(self, case: dict[str, Any]) -> PMFlowAITurnRuntimeGoldenResult:
|
|
request = RAGAskRequest(
|
|
question=str(case.get("question") or ""),
|
|
surface_context=dict(case.get("surface_context") or {}),
|
|
)
|
|
state = self.runtime.build_initial_turn_state(
|
|
request=request,
|
|
route_plan=dict(case.get("legacy_route_plan") or {}),
|
|
conversation_context=dict(case.get("conversation_context") or {}),
|
|
turn_hypothesis=dict(case.get("turn_hypothesis") or {}),
|
|
)
|
|
decision = self.runtime.assess_sufficiency(state)
|
|
observed = {
|
|
"answer_shape": state.answer_state.answer_shape,
|
|
"next_action": decision.next_action_type,
|
|
"candidate_actions": [item.action_type for item in state.action_state.candidate_actions],
|
|
"must_ask_user": decision.must_ask_user,
|
|
"can_answer_with_assumptions": state.sufficiency_state.can_answer_with_assumptions,
|
|
"needs_external": state.sufficiency_state.needs_external_evidence,
|
|
"needs_internal": state.sufficiency_state.needs_internal_evidence,
|
|
"primary_binding": (state.object_state.primary_object.binding_strength if state.object_state.primary_object else ""),
|
|
"context_titles": [item.title for item in state.object_state.context_objects if item.title],
|
|
"route_hypothesis_only": bool(state.route_plan_input.get("route_hypothesis_only")),
|
|
}
|
|
failures = tuple(_case_failures(expect=dict(case.get("expect") or {}), observed=observed))
|
|
return PMFlowAITurnRuntimeGoldenResult(
|
|
case_id=str(case.get("case_id") or "unnamed"),
|
|
passed=not failures,
|
|
failures=failures,
|
|
observed=observed,
|
|
)
|
|
|
|
|
|
def _case_failures(*, expect: dict[str, Any], observed: dict[str, Any]) -> list[str]:
|
|
failures: list[str] = []
|
|
for key in ("answer_shape", "next_action", "must_ask_user", "can_answer_with_assumptions", "needs_external", "needs_internal", "primary_binding"):
|
|
if key in expect and observed.get(key) != expect.get(key):
|
|
failures.append(f"{key}: expected {expect.get(key)!r}, got {observed.get(key)!r}")
|
|
for action in list(expect.get("actions") or []):
|
|
if action not in list(observed.get("candidate_actions") or []):
|
|
failures.append(f"candidate_actions missing {action!r}: got {observed.get('candidate_actions')!r}")
|
|
for action in list(expect.get("forbid_actions") or []):
|
|
if action in list(observed.get("candidate_actions") or []):
|
|
failures.append(f"candidate_actions should not contain {action!r}: got {observed.get('candidate_actions')!r}")
|
|
for title in list(expect.get("context_titles") or []):
|
|
if title not in list(observed.get("context_titles") or []):
|
|
failures.append(f"context_titles missing {title!r}: got {observed.get('context_titles')!r}")
|
|
if observed.get("route_hypothesis_only") is not True:
|
|
failures.append("route_plan_input must stay hypothesis-only")
|
|
return failures
|
|
|
|
|
|
__all__ = [
|
|
"PMFlowAITurnRuntimeGoldenReplay",
|
|
"PMFlowAITurnRuntimeGoldenResult",
|
|
]
|