167 lines
6.2 KiB
Python
167 lines
6.2 KiB
Python
"""Generic context grounding verifier for PMFlow AI answers."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from typing import Any
|
||
|
||
|
||
def verify_answer_context_grounding(
|
||
*,
|
||
answer: str,
|
||
used_context_packet: dict[str, Any] | None,
|
||
min_required_overlap: int = 1,
|
||
) -> dict[str, Any]:
|
||
"""Check whether an answer respects supplied thread/memory context.
|
||
|
||
This verifier is intentionally domain-agnostic: it extracts textual signals
|
||
from the already-selected context packet and checks whether non-empty context
|
||
has any material overlap with the answer. It does not classify the user's
|
||
scene or branch on business object names.
|
||
"""
|
||
|
||
packet = used_context_packet if isinstance(used_context_packet, dict) else {}
|
||
answer_terms = _text_terms(answer)
|
||
required_items = _required_context_items(packet)
|
||
missing = []
|
||
used = []
|
||
for item in required_items:
|
||
terms = _text_terms(item.get("text"))
|
||
if not terms:
|
||
continue
|
||
overlap = sorted(answer_terms & terms)
|
||
row = {
|
||
"source": item.get("source"),
|
||
"ref_id": item.get("ref_id"),
|
||
"overlap_count": len(overlap),
|
||
"preview": item.get("preview"),
|
||
}
|
||
if len(overlap) >= max(1, int(min_required_overlap or 1)):
|
||
used.append(row)
|
||
else:
|
||
missing.append(row)
|
||
dropped_risks = list(packet.get("dropped_risks") or []) if isinstance(packet.get("dropped_risks"), list) else []
|
||
budget_dropped_core = [
|
||
item
|
||
for item in dropped_risks
|
||
if isinstance(item, dict) and str(item.get("section") or "") in {"thread_context", "memory_context"}
|
||
]
|
||
required_count = len(required_items)
|
||
used_required_count = len(used)
|
||
minimum_required = min(3, max(1, required_count // 2)) if required_count else 0
|
||
passed = not required_items or used_required_count >= minimum_required or not answer_terms
|
||
if budget_dropped_core:
|
||
passed = False
|
||
findings = []
|
||
if required_items and not used and answer_terms:
|
||
findings.append(
|
||
{
|
||
"finding_type": "context_not_reflected",
|
||
"severity": "blocking",
|
||
"summary": "Answer has supplied context but does not reflect any selected context signal.",
|
||
}
|
||
)
|
||
if required_items and used_required_count < minimum_required and answer_terms:
|
||
findings.append(
|
||
{
|
||
"finding_type": "context_obligation_underused",
|
||
"severity": "blocking",
|
||
"summary": "Answer did not satisfy enough required thread, memory or topic context obligations.",
|
||
}
|
||
)
|
||
if budget_dropped_core:
|
||
findings.append(
|
||
{
|
||
"finding_type": "critical_context_dropped",
|
||
"severity": "blocking",
|
||
"summary": "Thread or memory context was dropped by budget before answer generation.",
|
||
}
|
||
)
|
||
return {
|
||
"packet_version": "pmflow_ai_context_grounding_verification.v1",
|
||
"grounding_passed": passed,
|
||
"used_context_count": len(used),
|
||
"missing_context_count": len(missing),
|
||
"used_context_refs": used[:12],
|
||
"missing_required_context": missing[:12],
|
||
"dropped_risks": dropped_risks[:12],
|
||
"findings": findings[:8],
|
||
}
|
||
|
||
|
||
def revise_answer_with_context_boundary(*, answer: str, grounding_packet: dict[str, Any]) -> str:
|
||
if grounding_packet.get("grounding_passed"):
|
||
return str(answer or "").strip()
|
||
missing = list(grounding_packet.get("missing_required_context") or [])
|
||
if not missing:
|
||
return str(answer or "").strip()
|
||
previews = [
|
||
str(item.get("preview") or "").strip()
|
||
for item in missing[:3]
|
||
if isinstance(item, dict) and str(item.get("preview") or "").strip()
|
||
]
|
||
if not previews:
|
||
return str(answer or "").strip()
|
||
boundary = "这轮回答需要先对齐已选上下文：" + "；".join(previews)
|
||
original = str(answer or "").strip()
|
||
if not original:
|
||
return boundary
|
||
return f"{original}\n\n{boundary}"
|
||
|
||
|
||
def _required_context_items(packet: dict[str, Any]) -> list[dict[str, str]]:
|
||
rows = []
|
||
for item in list(packet.get("thread_message_refs") or [])[:8]:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
text = str(item.get("content_preview") or "").strip()
|
||
if text:
|
||
rows.append(
|
||
{
|
||
"source": "thread_message",
|
||
"ref_id": str(item.get("message_id") or item.get("message_index") or "")[:80],
|
||
"text": text,
|
||
"preview": text[:180],
|
||
}
|
||
)
|
||
evidence = packet.get("context_evidence") if isinstance(packet.get("context_evidence"), dict) else {}
|
||
for item in list(evidence.get("claims") or [])[:12]:
|
||
if not isinstance(item, dict) or not bool(item.get("required_for_answer")):
|
||
continue
|
||
text = str(item.get("claim_preview") or "").strip()
|
||
if text:
|
||
rows.insert(
|
||
0,
|
||
{
|
||
"source": f"context_evidence:{str(item.get('semantic_role') or '')[:60]}",
|
||
"ref_id": str(item.get("claim_id") or item.get("source_ref") or "")[:80],
|
||
"text": text,
|
||
"preview": text[:180],
|
||
},
|
||
)
|
||
for item in list(packet.get("memory_refs") or [])[:8]:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
text = str(item.get("summary_preview") or "").strip()
|
||
if text:
|
||
rows.append(
|
||
{
|
||
"source": "memory",
|
||
"ref_id": str(item.get("memory_id") or "")[:80],
|
||
"text": text,
|
||
"preview": text[:180],
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def _text_terms(value: Any) -> set[str]:
|
||
text = str(value or "").lower()
|
||
alnum = {item for item in re.findall(r"[a-z0-9_]{2,}", text)}
|
||
cjk = re.findall(r"[\u4e00-\u9fff]", text)
|
||
cjk_terms = {"".join(cjk[index : index + 2]) for index in range(max(0, len(cjk) - 1))}
|
||
return {item for item in [*alnum, *cjk_terms] if item}
|
||
|
||
|
||
__all__ = ["revise_answer_with_context_boundary", "verify_answer_context_grounding"]
|