171 lines
6.2 KiB
Python
171 lines
6.2 KiB
Python
"""Verification helpers for context bundle consumption."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from .shared import Any, AICallLog, AsyncSession, CONTEXT_BUNDLE_MARKER, ModelRoutingService, Optional, TARGET_FUNCTION_CONTEXT, and_, select, uuid4
|
|
|
|
|
|
class ModelContextBundleVerifyMixin:
|
|
"""Run canary calls and verify prompt context evidence."""
|
|
|
|
@classmethod
|
|
async def _trigger_canary_call(
|
|
cls,
|
|
db: AsyncSession,
|
|
*,
|
|
tenant_id: int,
|
|
function_name: str,
|
|
probe_id: str,
|
|
) -> dict[str, Any]:
|
|
prompt = cls._canary_prompt_for_function(function_name=function_name, probe_id=probe_id)
|
|
try:
|
|
from ..llm import llm_service
|
|
|
|
result = await llm_service.generate_structured_object(
|
|
prompt,
|
|
db=db,
|
|
tenant_id=int(tenant_id),
|
|
function_name=function_name,
|
|
use_prompt_profile=False,
|
|
temperature=0.0,
|
|
max_tokens=220,
|
|
response_mode="fast",
|
|
)
|
|
return {
|
|
"called": True,
|
|
"error": None,
|
|
"error_code": str(result.error_code or ""),
|
|
"status": str(result.error_code or "success"),
|
|
"provider": str(result.provider or ""),
|
|
"model": str(result.model or ""),
|
|
}
|
|
except Exception as exc:
|
|
return {
|
|
"called": False,
|
|
"error": str(exc)[:500],
|
|
"error_code": "canary_call_failed",
|
|
"status": "failed",
|
|
"provider": "",
|
|
"model": "",
|
|
}
|
|
|
|
@classmethod
|
|
async def _find_prompt_evidence(
|
|
cls,
|
|
db: AsyncSession,
|
|
*,
|
|
tenant_id: int,
|
|
function_name: str,
|
|
probe_id: Optional[str],
|
|
) -> Optional[AICallLog]:
|
|
stmt = select(AICallLog).where(
|
|
and_(
|
|
AICallLog.tenant_id == int(tenant_id),
|
|
AICallLog.function_name == str(function_name),
|
|
AICallLog.prompt.like(f"%[{CONTEXT_BUNDLE_MARKER}]%"),
|
|
)
|
|
)
|
|
if probe_id:
|
|
stmt = stmt.where(AICallLog.prompt.like(f"%{probe_id}%"))
|
|
stmt = stmt.order_by(AICallLog.id.desc()).limit(1)
|
|
return (await db.execute(stmt)).scalar_one_or_none()
|
|
|
|
@classmethod
|
|
async def verify_context_consumption(
|
|
cls,
|
|
db: AsyncSession,
|
|
*,
|
|
tenant_id: int,
|
|
run_canary: bool = True,
|
|
) -> dict[str, Any]:
|
|
probe_id = uuid4().hex[:12] if run_canary else ""
|
|
function_results: list[dict[str, Any]] = []
|
|
|
|
for function_name in cls.target_functions():
|
|
target = TARGET_FUNCTION_CONTEXT.get(function_name) or {}
|
|
canary = {
|
|
"called": False,
|
|
"status": "skipped",
|
|
"error": None,
|
|
"error_code": "",
|
|
"provider": "",
|
|
"model": "",
|
|
}
|
|
if run_canary:
|
|
canary = await cls._trigger_canary_call(
|
|
db,
|
|
tenant_id=int(tenant_id),
|
|
function_name=function_name,
|
|
probe_id=probe_id,
|
|
)
|
|
|
|
evidence = await cls._find_prompt_evidence(
|
|
db,
|
|
tenant_id=int(tenant_id),
|
|
function_name=function_name,
|
|
probe_id=probe_id if run_canary else None,
|
|
)
|
|
prompt_text = str(evidence.prompt or "") if evidence is not None else ""
|
|
checks = {
|
|
"marker": f"[{CONTEXT_BUNDLE_MARKER}]" in prompt_text,
|
|
"knowledge_graph": '"knowledge_graph"' in prompt_text,
|
|
"task_template": '"task_template"' in prompt_text,
|
|
"contract_sync": '"contract_sync"' in prompt_text,
|
|
}
|
|
route = await ModelRoutingService.get_effective_routing(
|
|
db,
|
|
tenant_id=int(tenant_id),
|
|
function_name=function_name,
|
|
)
|
|
function_results.append(
|
|
{
|
|
"function_name": function_name,
|
|
"tier": str(target.get("tier") or ""),
|
|
"template_ids": [
|
|
str(x)
|
|
for x in target.get("template_ids", [])
|
|
if str(x or "").strip()
|
|
],
|
|
"routing_snapshot": {
|
|
"provider": str(route.get("provider") or ""),
|
|
"model": str(route.get("model") or ""),
|
|
"tier": str(route.get("tier") or ""),
|
|
"source": str(route.get("source") or ""),
|
|
},
|
|
"canary": canary,
|
|
"checks": checks,
|
|
"pass": all(checks.values()),
|
|
"log_evidence": (
|
|
{
|
|
"id": int(evidence.id),
|
|
"status": str(evidence.status or ""),
|
|
"created_at": evidence.created_at.isoformat() if evidence.created_at else None,
|
|
"prompt_excerpt": prompt_text[:800],
|
|
}
|
|
if evidence is not None
|
|
else None
|
|
),
|
|
}
|
|
)
|
|
|
|
tier_result: dict[str, dict[str, Any]] = {
|
|
"low": {"pass": False, "functions": []},
|
|
"high": {"pass": False, "functions": []},
|
|
}
|
|
for tier_name in ("low", "high"):
|
|
tier_functions = [row for row in function_results if str(row.get("tier") or "") == tier_name]
|
|
tier_result[tier_name] = {
|
|
"pass": bool(tier_functions) and all(bool(row.get("pass")) for row in tier_functions),
|
|
"functions": tier_functions,
|
|
}
|
|
|
|
overall_pass = bool(tier_result["low"]["pass"]) and bool(tier_result["high"]["pass"])
|
|
return {
|
|
"marker": CONTEXT_BUNDLE_MARKER,
|
|
"run_canary": bool(run_canary),
|
|
"probe_id": probe_id or None,
|
|
"overall_pass": overall_pass,
|
|
"tiers": tier_result,
|
|
"functions": function_results,
|
|
}
|