112 lines
4.7 KiB
Python
112 lines
4.7 KiB
Python
"""Prompt evaluation rubrics for platform Prompt Operations."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from copy import deepcopy
|
||
from typing import Any
|
||
|
||
|
||
DEFAULT_PROMPT_EVAL_RUBRIC: dict[str, Any] = {
|
||
"rubric_key": "default_prompt_eval_v1",
|
||
"label": "通用 Prompt 发布评估",
|
||
"dimensions": {
|
||
"contract_adherence": "是否遵守输入/输出契约、变量约束和格式要求。",
|
||
"business_correctness": "是否围绕业务场景给出准确、可执行且不过度扩展的结果。",
|
||
"evidence_grounding": "是否明确证据、上下文与不确定性，不把无依据内容包装成事实。",
|
||
"safety_boundary": "是否遵守 PMFlow 主链边界、权限边界和 fail-closed 规则。",
|
||
"operator_readability": "是否便于运营、审查和后续沉淀样本。",
|
||
},
|
||
"release_policy": {
|
||
"recommend_release": "平均表现优于或不低于 baseline，且无关键风险。",
|
||
"needs_revision": "存在可修订问题，但不构成强阻塞。",
|
||
"block_release": "违反核心契约、边界或高风险样本失败。",
|
||
"manual_review": "证据不足或模型评估不稳定，需要人工复核。",
|
||
},
|
||
}
|
||
|
||
|
||
FAMILY_PROMPT_EVAL_RUBRICS: dict[str, dict[str, Any]] = {
|
||
"classification": {
|
||
"rubric_key": "classification_prompt_eval_v1",
|
||
"label": "分类/路由 Prompt 评估",
|
||
"focus": [
|
||
"分类字段是否稳定、可解释。",
|
||
"不确定时是否澄清或返回 uncertain，而不是强行裁决。",
|
||
"是否避免在执行层恢复关键词/对象词表二次拍板。",
|
||
],
|
||
},
|
||
"extraction": {
|
||
"rubric_key": "extraction_prompt_eval_v1",
|
||
"label": "结构化抽取 Prompt 评估",
|
||
"focus": [
|
||
"字段完整度、JSON 可解析性和 pending 标注是否可靠。",
|
||
"证据不足时是否显式失败或待确认。",
|
||
"是否避免模板句、默认句冒充抽取成功。",
|
||
],
|
||
},
|
||
"planning": {
|
||
"rubric_key": "planning_prompt_eval_v1",
|
||
"label": "规划/拆解 Prompt 评估",
|
||
"focus": [
|
||
"任务拆解是否可执行、顺序清晰且边界明确。",
|
||
"是否保留人工确认和高风险审批边界。",
|
||
"是否避免自动写入项目真相层或越过主链。",
|
||
],
|
||
},
|
||
"response": {
|
||
"rubric_key": "response_prompt_eval_v1",
|
||
"label": "回答呈现 Prompt 评估",
|
||
"focus": [
|
||
"来源合同是否清楚，内部/外部/不确定是否分明。",
|
||
"是否使用客户可理解中文，不泄露 raw source/internal label。",
|
||
"是否避免过度自信、泛化结论和内部术语外露。",
|
||
],
|
||
},
|
||
"governance": {
|
||
"rubric_key": "governance_prompt_eval_v1",
|
||
"label": "治理/安全 Prompt 评估",
|
||
"focus": [
|
||
"是否严格遵守权限、审计和 fail-closed 边界。",
|
||
"风险、阻塞和人工确认原因是否清楚。",
|
||
"是否避免为了完整回答而绕过运行时护栏。",
|
||
],
|
||
},
|
||
}
|
||
|
||
|
||
PROMPT_KEY_EVAL_RUBRICS: dict[str, dict[str, Any]] = {
|
||
"PROMPT_RAG_QUERY_SEMANTICS": {
|
||
"rubric_key": "rag_query_semantics_eval_v1",
|
||
"label": "Query Route Gateway 语义路由评估",
|
||
"focus": [
|
||
"route_mode / decision_origin / reply_contract 是否由结构化语义证据支撑。",
|
||
"internal/external/uncertain 分流是否遵守 Query Route Gateway，而不是靠对象词表补判。",
|
||
"证据不足时是否澄清或 external-first，禁止默认混合答案。",
|
||
],
|
||
},
|
||
"PROMPT_EXTERNAL_DIRECT_FACT_QA": {
|
||
"rubric_key": "external_direct_fact_eval_v1",
|
||
"label": "外部实时问答评估",
|
||
"focus": [
|
||
"是否仅使用外部 verified evidence，不混入 Feishu/PMFlow 内部来源。",
|
||
"外部证据不足时是否显式说明限制。",
|
||
"来源摘要是否客户可读且不暴露 raw provider/source label。",
|
||
],
|
||
},
|
||
}
|
||
|
||
|
||
def get_prompt_eval_rubric(*, prompt_key: str, prompt_family: str) -> dict[str, Any]:
|
||
"""Return a merged rubric payload for a prompt key/family."""
|
||
|
||
rubric = deepcopy(DEFAULT_PROMPT_EVAL_RUBRIC)
|
||
family = FAMILY_PROMPT_EVAL_RUBRICS.get(str(prompt_family or "").strip())
|
||
if family:
|
||
rubric.update(deepcopy(family))
|
||
rubric["base_rubric_key"] = DEFAULT_PROMPT_EVAL_RUBRIC["rubric_key"]
|
||
prompt_specific = PROMPT_KEY_EVAL_RUBRICS.get(str(prompt_key or "").strip())
|
||
if prompt_specific:
|
||
rubric.update(deepcopy(prompt_specific))
|
||
rubric["family_rubric_key"] = family.get("rubric_key") if family else None
|
||
return rubric
|