test: lock ai routing governance boundaries
This commit is contained in:
parent
f48a4d0381
commit
d1c4b71376
|
|
@ -448,6 +448,11 @@ async def test_admin_console_reports_protected_downgrade_and_structured_failures
|
|||
assert "structured_failure_rate" in codes
|
||||
assert data["recent_failures"][0]["status_label"] == "模型返回为空"
|
||||
assert data["scene_routes"][0]["label"] == "项目启动草案"
|
||||
assert data["providers"][0]["api_key_masked"] == "db-l...-key"
|
||||
assert "db-low-key" not in str(data)
|
||||
assert "db-high-key" not in str(data)
|
||||
assert "prompt 0" not in str(data)
|
||||
assert "api_env" not in str(data)
|
||||
launch_row = next(item for item in data["scene_slot_matrix"] if item["scene_key"] == "project_launch")
|
||||
assert launch_row["recent_failure_rate"] == 1.0
|
||||
assert launch_row["recent_failure_count"] == 3
|
||||
|
|
@ -1299,6 +1304,8 @@ async def test_ai_routing_overview_requires_governance_role() -> None:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ai_routing_api_returns_enveloped_payloads(monkeypatch) -> None:
|
||||
operation_log_calls = []
|
||||
|
||||
class FakeService:
|
||||
def __init__(self, db, *, tenant_id: int):
|
||||
assert tenant_id == 9
|
||||
|
|
@ -1381,7 +1388,7 @@ async def test_ai_routing_api_returns_enveloped_payloads(monkeypatch) -> None:
|
|||
return {"items": [], "summary": {"ok": 1}}
|
||||
|
||||
async def fake_log_operation(*args, **kwargs):
|
||||
_ = args, kwargs
|
||||
operation_log_calls.append({"args": args, "kwargs": kwargs})
|
||||
|
||||
monkeypatch.setattr(ai_routing, "AIRoutingGovernanceService", FakeService)
|
||||
monkeypatch.setattr(ai_routing, "log_operation", fake_log_operation)
|
||||
|
|
@ -1409,6 +1416,15 @@ async def test_ai_routing_api_returns_enveloped_payloads(monkeypatch) -> None:
|
|||
assert slot_publish.json()["data"]["active_version"] == "slot-v1"
|
||||
assert slot_rollback.json()["data"]["active_version"] == "slot-v2"
|
||||
assert diagnostics.json()["data"]["summary"]["ok"] == 1
|
||||
assert len(operation_log_calls) == 1
|
||||
log_kwargs = operation_log_calls[0]["kwargs"]
|
||||
assert log_kwargs["tenant_id"] == 9
|
||||
assert log_kwargs["user_id"] == 101
|
||||
assert log_kwargs["entity_type"] == "ai_routing_policy"
|
||||
assert log_kwargs["entity_id"] == "team-policy"
|
||||
assert log_kwargs["action"] == "update"
|
||||
assert "provider" not in str(log_kwargs["after_data"]).lower()
|
||||
assert "model" not in str(log_kwargs["after_data"]).lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -409,7 +409,7 @@ Product language uses workspace/space; the compatibility table name remains `ten
|
|||
- Purpose: platform AI routing overview for PMFlow AI operators.
|
||||
- Auth: PMFlow platform capability `platform.ai_routing.manage`.
|
||||
- Response data:
|
||||
- `control_plane`: `{ version, source, updated_at, baseline_note }`
|
||||
- `config_version`, `config_updated_at`, `source_of_truth`
|
||||
- `slots[]`: low/high slot summaries with Chinese labels, provider/model display summary, connection state, capability coverage, and masked key status.
|
||||
- `capability_slots[]`: configurable `low/high/structured/tool/long_context/external_query` slots and candidate `private/multimodal` slots. LLM slots include `primary_engine_id`, fallback chain, failure policy, protected state, requirements, capability coverage, candidate count, candidate runtime previews, and blocked candidate reasons. `external_query` includes `primary_provider_id`, `fallback_provider_ids`, `verifier_required`, `source_filtering_required`, and `external_provider_registry` status for MCP adapter, bounded live search fallback, verifier, and fail-closed source filtering.
|
||||
- `metrics`: recent 24h/7d calls, failures, failure rate, escalation rate, structured failure rate, average latency, and cost estimate.
|
||||
|
|
@ -437,7 +437,7 @@ Product language uses workspace/space; the compatibility table name remains `ten
|
|||
- Side effects:
|
||||
- persist into `integrations(type='ai_team_policy')`
|
||||
- append `operation_logs(entity_type='ai_routing_policy')`
|
||||
- Boundary: never writes `backend/.control_plane/pmflow-routing-rules.json`, provider secrets, or model-routing overrides.
|
||||
- Boundary: never writes `backend/.control_plane/pmflow-routing-rules.json`, provider secrets, or model-routing overrides; audit payload contains business policy only, not provider/model/runtime fields.
|
||||
|
||||
### GET `/ai-routing/admin-console`
|
||||
- Purpose: AI admin/operator console aggregating provider connection summaries, capability slots, post-publish slot observability, business-scene routes, recent failure samples, protected scenes, and supplier/runtime diagnostics.
|
||||
|
|
@ -452,6 +452,7 @@ Product language uses workspace/space; the compatibility table name remains `ten
|
|||
- `scene_slot_matrix[]`: explainability matrix for operators. Each row includes `scene_key`, `scene_label`, `scene_contract`, `lifecycle`, `scene_family`, `current_slot`, `required_slot`, `current_slot_label`, `required_capabilities[]`, `protected`, `active_runtime`, `active_slot_runtime`, `draft_runtime`, `draft_slot_runtime`, `active_version`, `draft_dirty`, `capability_gap`, `recent_failure_rate`, `recent_failure_count`, `recent_failed_calls`, `recent_total_calls`, `scene_health`, `release_gate`, `impact_summary`, aggregate `risk_level/risk_label`, `slots[]`, and advanced `functions[]`. `scene_contract` includes business goal, input contract, output contract, success criteria, failure policy, confirmation policy, evidence policy, protected reason, expected slot, scene family, owner, and lifecycle. `scene_health` includes `status=healthy|warning|blocked|unknown`, score, blockers, warnings, and evidence summary. `release_gate` includes whether publish is blocked, blocking reason, draft impact, protected-scene regression, and suggested action. `active_runtime` / `active_slot_runtime` and `draft_runtime` / `draft_slot_runtime` summarize slot key/label, release version, configured state, runtime label/source, and primary runtime id. `capability_gap` distinguishes missing capability, missing probe evidence, stale evidence, and human-readable blocking reasons for release review. Each slot row includes `slot_key`, slot label, whether it is required by the scene, active runtime/source, release version, capability coverage, and risk reasons such as missing active binding, missing capability, missing probe evidence, stale probe evidence, or protected-scene downgrade. `functions[]` is diagnostic payload only; business-scene route UI must stay at scene/slot level, consume backend scene contracts, and route function-level inspection to hidden Route Lab.
|
||||
- `recent_failures[]`: sanitized failure samples without raw error or prompt; includes `scene_key / scene_label` when the function belongs to a registered business scene so `/platform/ai-routing-failures?scene=...` can filter deterministically.
|
||||
- `protected_scenes[]`, `diagnostics[]`
|
||||
- Boundary: this is the only user-facing surface that may show provider/model/base URL summaries, and all secrets must remain masked; PMFlow business pages must consume only business policy/mode/trust state.
|
||||
|
||||
### GET `/ai-call-traces`
|
||||
- Purpose: platform AI call trace ledger for PMFlow AI operators to inspect provider/model call history, sanitized request/response evidence, runtime routing metadata, slot usage, latency, tokens, cost estimates, and recent failure context.
|
||||
|
|
|
|||
|
|
@ -252,6 +252,7 @@
|
|||
- AI Steward 浮窗的“当前对象/来源模式/证据状态”必须直接消费 `/rag/ask` 与 QA session API 的显式 contract 字段,禁止在前端靠回答正文反推 internal/external 或 plan 状态。
|
||||
- 网页问答显式锚点 v1 只能消费稳定强锚点:项目页路径中的 `project_id` 与 URL query 中已显式存在的 `project_id/source_chat_id`;前端禁止自行猜测 `source_chat_id`。
|
||||
- `/ai-config` 是空间级 AI 使用设置,技术路由继续保留但用户可见不再叫“AI 运营中心”“管理后台”或泛化 AI Config;`/ai-config` 及其子路由必须使用空间配置壳 `SpaceSidebar(activeLabel="AI 使用设置") + SpaceTopBar`,让用户仍处在个人空间/团队空间的配置上下文,不得切入项目工作台侧栏;`/ai-config?tab=setup` 由 `AISetupWizard` 承接 AI 可用状态和项目生成能力门禁,只展示空间业务影响、团队策略和连接入口,不展示“启用供应商/路由/连接测试/额度诊断”类平台动作;连接、知识、团队策略、MCP 模板使用和 Prompt 策略继续组织客户空间可控能力,并明确这些配置影响当前团队空间内所有项目。`AIReadiness` 必须遵守双层表达:用户端默认客户视图,平台人员在用户端也只能通过显式 `platformDiagnostic` 查看 cause/CTA;`/platform/*` 默认平台诊断视图,展示供应商、全局路由、模型槽位、诊断、失败样本和处理动作。平台供应商、全局路由、route/context、semantic facts、高级函数路由和跨租户运营诊断不在客户空间暴露,统一进入 `/platform/*`,由 `PlatformBoundary` 和 `platform_context.capabilities` 保护;旧 `/ai-config/contract-sync-board`、`/ai-config/query-route-governance`、`/ai-config/ai-routing-console`、`/ai-config/model-routing-config`、`/ai-config/knowledge-graph-board` 和 `/ai-config/semantic-facts-board` 只做兼容跳转到 `/platform/*`,其中 model routing 旧入口跳到隐藏 `/platform/route-lab`,避免平台页套入空间壳。`/platform/*` 使用 `PlatformSidebar + PlatformTopBar` 专属壳,侧栏按运行总览、AI 运行层、工具与外部能力层、上下文与语义层、质量与事件层组织,不复用旧项目工作台导航;隐藏 Route Lab 不进侧栏,只读展示函数级规则、最近执行与历史 override 痕迹。AI 路由与供应商控制台由 `backend/app/api/ai_routing.py` 与 `backend/app/services/ai_routing_governance_service.py` 提供聚合,客户负责人只看业务场景策略,平台 AI Ops 才看供应商连接、快速/深度槽位、场景路由、诊断和审计提示;平台 AI 运行总览已合并进 `/platform`,旧 `/platform/ai-routing-console` 只做兼容跳转;AI 运行层前端保留 `/platform/ai-routing-providers`、`/platform/ai-routing-slots`、`/platform/ai-routing-scenes`、`/platform/ai-routing-costs`、`/platform/ai-routing-failures` 五个明确子菜单,共享同一脱敏 admin-console 合同;质量与事件层新增 `/platform/ai-call-traces` AI 调用审计,只给平台 AI Ops 查看 `ai_call_logs` 的脱敏出入参、完整脱敏 provider 请求/响应 payload、上游尝试链、provider/model/slot/routing/tokens/cost/latency 和错误摘要,未脱敏 provider payload、headers 与密钥不暴露;团队策略只写 `integrations(type='ai_team_policy')`,供应商连接写 `integrations(type='llm_engine')` 并在 engine config 内分层保存声明 capabilities、`capability_probe` live evidence 与后端派生 `effective_capabilities`,旧 flat `capability_evidence` 只做兼容;能力探测必须按 registry 逐项验证,JSON Schema 独立于普通 JSON 输出,长上下文声明窗口只能作为 `declared_only`;供应商页允许平台操作人对高成本长上下文能力执行“确认长上下文”,写入带来源、确认人和过期时间的 `operator_verified/operator_attestation` 证据,满足发布校验但不进入 AI 调用审计;能力槽位写 `integrations(type='ai_capability_slots')`,但只有发布后的 `active.slots` 会由 `llm_service` low/high/structured tier 解析消费,draft 只用于平台编辑、校验、发布和回滚;能力槽位发布校验按 `SLOT_REQUIREMENTS` 对 draft primary/fallback 的能力矩阵、缺证据和证据过期做阻塞检查,low 缺 probe 只 warning,reserved 槽当前只展示候选/缺口,除非 draft 显式配置运行时,否则不接 runtime 消费;`backend/.control_plane/pmflow-routing-rules.json` 只作为 baseline rule pack 与冷启动 fallback。MCP 页负责模板化配置、后端直连/Local Bridge 状态展示、runtime probe、命令预览与本机确认;contract sync、route/context、knowledge graph 和 semantic facts 页属于平台 AI Ops 排障治理入口。
|
||||
- AI Routing Governance P5 已把 AI 路由治理验收锁进平台 AI 运营中心:后端 `backend/app/api/ai_routing.py` 提供 `overview / team-policy / admin-console / diagnostics` 包络接口,`backend/app/services/ai_routing_governance_service.py` 负责脱敏 provider/model/base URL 摘要、scene-slot matrix 与 recent failure 样本聚合;`PUT /ai-routing/team-policy` 只写 `integrations(type='ai_team_policy')` 并记录 `operation_logs(entity_type='ai_routing_policy')`,不改 PMFlow OS 主链、Space、Account Settings、control-plane routing rules 或运行时模型真源。`backend/tests/test_ai_routing_governance_service.py` 与 `frontend/src/pages/ai-routing/AIRoutingPages.test.tsx` 锁定负责人视图不出现 provider/model、admin console 默认隐藏 `function_name` 且 recent failure 不泄露 prompt/key。
|
||||
- `/platform/prompt-assets` 属于平台质量与事件层,不是客户空间 Prompt 策略页。它可展示平台内部 prompt 正文、默认模板 hash、active/draft release、release gate、replay run、质量归因、golden case、调用点和迁移缺口;客户空间、Space、Project Launch 和项目级 AI 副驾不得暴露 prompt body。标记为 `inline_discovered` / `needs_migration` 的资产只代表已完成清单梳理,运行时仍需单独迁移调用点到 PromptRuntimeService/PromptReleaseService 后才算可运营生效;`guarded_release` 可创建候选但激活必须通过 replay/schema/golden case 门禁,当前 golden replay 无 active golden case 或无最新 replay run 时保持阻塞;LLM 回放只允许通过本地路由层和显式 routing_meta 执行,不在运营页硬选模型。
|
||||
- Workspace / Platform 账号边界:当前新增兼容层 `tenant_memberships / tenant_invitations`,把产品语义从“企业租户账号”收敛到 `solo-first, team-ready` 的个人空间/团队空间。手机号是登录、自注册与邀请绑定的主身份,后端通过 `phone_verification_codes`、`users.phone_e164` 和 `tenant_invitations.phone_e164` 串起“验证手机号 -> 创建或绑定账号 -> 创建个人空间 -> 接受团队邀请”的链路。用户账号不归团队所有;注册或受邀创建账号时必须有个人空间,接受邀请只是增加团队空间 membership,退出团队空间后仍保留个人空间。PMFlow 内部平台身份由 `platform_memberships` 单独承载,并在登录响应和 `/users/me` 返回 `platform_context`,客户空间 `OWNER` 不自动拥有平台能力。历史数据修复由 `backend/alembic/versions/20260428_repair_admin_yangyu_workspace_state.py` 承接:`tenant_id=1` 保留为旧团队空间,admin/yangyu 的当前激活 `users.tenant_id` 分别指向自己的 personal workspace。
|
||||
- AI 路由与供应商用户可见层必须脱敏并分平面:成员视图不得出现 low/high、provider、model、`function_name`、`api_env`、raw routing meta 或明文 key;客户空间负责人只看“质量优先 / 成本均衡 / 速度优先”和项目启动、需求结构化、任务拆解、人员匹配、内部问答、外部查询、飞书理解、飞书回复等场景策略;平台 AI Ops 的业务场景路由也不展示 `function_name`,函数级路线只允许隐藏 Route Lab 只读诊断。
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@
|
|||
- 项目创建接口:参考 `backend/app/api/ai_steward_api/routes_capture.py`
|
||||
- 模型路由配置接口:参考 `backend/app/api/model_routing.py`;作为 platform AI Operations Console 隐藏 Route Lab 的诊断接口,要求 `platform.ai_routing.manage`,`API_ROUTE_PROFILE=lean` 下也必须由核心路由加载;运营导航不暴露函数级编辑入口
|
||||
- AI 路由治理聚合服务:参考 `backend/app/services/ai_routing_governance_service.py`;聚合 `integrations(type='llm_engine'|'model_routing'|'ai_team_policy'|'ai_capability_slots'|'ai_function_runtime_profiles')`、`ai_call_logs` 与 `backend/.control_plane/pmflow-routing-rules.json`,维护 `SLOT_REQUIREMENTS`、`SCENE_DEFAULT_SLOT`、槽位能力覆盖、函数运行 profile 摘要、缺能力/缺证据发布阻塞项、scene-slot matrix、active slot observability 和函数级输出 token 观测;函数运行 profile 摘要必须包含 token 预算、temperature、前台等待、provider/job 超时、最大重试和大输入策略,用于 `/platform/ai-routing-scenes` 右侧“生成运行合同”编辑;观测聚合只输出产品化状态、fallback/degraded 计数、估算成本/延迟来源、输出 token p95、截断/结构化失败计数与脱敏失败样本,只返回中文展示字段、能力标签、probe status、last verified 与 masked key,不返回明文 key、`api_env`、prompt、raw error 或 raw routing meta
|
||||
- AI Routing Governance P5 验收锁定:`backend/tests/test_ai_routing_governance_service.py` 覆盖 OWNER/REVIEWER 访问边界、OWNER 保存团队策略审计、admin console 敏感字段脱敏、recent failure 不泄露 prompt/key;`frontend/src/pages/ai-routing/AIRoutingPages.test.tsx` 覆盖 `/ai-config/team-ai-policy` 负责人视图不出现 provider/model、`/platform` 与 AI 运营中心入口存在、平台管理视图默认隐藏 `function_name`。
|
||||
- 回复质量闭环接口:参考 `backend/app/api/ai_steward_api/routes_reply_iteration.py`
|
||||
- 训练/微调管线接口:参考 `backend/app/api/ai_steward_api/routes_training_pipeline.py`
|
||||
- Mounted Runtime 控制台接口:参考 `backend/app/api/ai_steward_api/routes_mounted_ai.py`
|
||||
|
|
|
|||
|
|
@ -930,6 +930,10 @@ describe("AI routing productized pages", () => {
|
|||
renderWithProviders(<TeamAIPolicyPage />);
|
||||
|
||||
expect(await screen.findByText("项目启动草案")).not.toBeNull();
|
||||
expect(screen.queryByText("qwen-plus")).toBeNull();
|
||||
expect(screen.queryByText("GLM-5")).toBeNull();
|
||||
expect(screen.queryByText(/DashScope/)).toBeNull();
|
||||
expect(screen.queryByText(/OpenAI 兼容网关/)).toBeNull();
|
||||
fireEvent.click(screen.getByRole("button", { name: /质量优先/ }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "保存团队策略" }));
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue