"""Verification helpers for context bundle consumption.""" from __future__ import annotations from .shared import Any, AICallLog, AsyncSession, CONTEXT_BUNDLE_MARKER, ModelRoutingService, Optional, TARGET_FUNCTION_CONTEXT, and_, select, uuid4 class ModelContextBundleVerifyMixin: """Run canary calls and verify prompt context evidence.""" @classmethod async def _trigger_canary_call( cls, db: AsyncSession, *, tenant_id: int, function_name: str, probe_id: str, ) -> dict[str, Any]: prompt = cls._canary_prompt_for_function(function_name=function_name, probe_id=probe_id) try: from ..llm_service import llm_service result = await llm_service.generate_structured_object( prompt, db=db, tenant_id=int(tenant_id), function_name=function_name, use_prompt_profile=False, temperature=0.0, max_tokens=220, response_mode="fast", ) return { "called": True, "error": None, "error_code": str(result.error_code or ""), "status": str(result.error_code or "success"), "provider": str(result.provider or ""), "model": str(result.model or ""), } except Exception as exc: return { "called": False, "error": str(exc)[:500], "error_code": "canary_call_failed", "status": "failed", "provider": "", "model": "", } @classmethod async def _find_prompt_evidence( cls, db: AsyncSession, *, tenant_id: int, function_name: str, probe_id: Optional[str], ) -> Optional[AICallLog]: stmt = select(AICallLog).where( and_( AICallLog.tenant_id == int(tenant_id), AICallLog.function_name == str(function_name), AICallLog.prompt.like(f"%[{CONTEXT_BUNDLE_MARKER}]%"), ) ) if probe_id: stmt = stmt.where(AICallLog.prompt.like(f"%{probe_id}%")) stmt = stmt.order_by(AICallLog.id.desc()).limit(1) return (await db.execute(stmt)).scalar_one_or_none() @classmethod async def verify_context_consumption( cls, db: AsyncSession, *, tenant_id: int, run_canary: bool = True, ) -> dict[str, Any]: probe_id = uuid4().hex[:12] if run_canary else "" function_results: list[dict[str, Any]] = [] for function_name in cls.target_functions(): target = TARGET_FUNCTION_CONTEXT.get(function_name) or {} canary = { "called": False, "status": "skipped", "error": None, "error_code": "", "provider": "", "model": "", } if run_canary: canary = await cls._trigger_canary_call( db, tenant_id=int(tenant_id), function_name=function_name, probe_id=probe_id, ) evidence = await cls._find_prompt_evidence( db, tenant_id=int(tenant_id), function_name=function_name, probe_id=probe_id if run_canary else None, ) prompt_text = str(evidence.prompt or "") if evidence is not None else "" checks = { "marker": f"[{CONTEXT_BUNDLE_MARKER}]" in prompt_text, "knowledge_graph": '"knowledge_graph"' in prompt_text, "task_template": '"task_template"' in prompt_text, "contract_sync": '"contract_sync"' in prompt_text, } route = await ModelRoutingService.get_effective_routing( db, tenant_id=int(tenant_id), function_name=function_name, ) function_results.append( { "function_name": function_name, "tier": str(target.get("tier") or ""), "template_ids": [ str(x) for x in target.get("template_ids", []) if str(x or "").strip() ], "routing_snapshot": { "provider": str(route.get("provider") or ""), "model": str(route.get("model") or ""), "tier": str(route.get("tier") or ""), "source": str(route.get("source") or ""), }, "canary": canary, "checks": checks, "pass": all(checks.values()), "log_evidence": ( { "id": int(evidence.id), "status": str(evidence.status or ""), "created_at": evidence.created_at.isoformat() if evidence.created_at else None, "prompt_excerpt": prompt_text[:800], } if evidence is not None else None ), } ) tier_result: dict[str, dict[str, Any]] = { "low": {"pass": False, "functions": []}, "high": {"pass": False, "functions": []}, } for tier_name in ("low", "high"): tier_functions = [row for row in function_results if str(row.get("tier") or "") == tier_name] tier_result[tier_name] = { "pass": bool(tier_functions) and all(bool(row.get("pass")) for row in tier_functions), "functions": tier_functions, } overall_pass = bool(tier_result["low"]["pass"]) and bool(tier_result["high"]["pass"]) return { "marker": CONTEXT_BUNDLE_MARKER, "run_canary": bool(run_canary), "probe_id": probe_id or None, "overall_pass": overall_pass, "tiers": tier_result, "functions": function_results, }