1547 lines
67 KiB
Python
1547 lines
67 KiB
Python
from __future__ import annotations
|
||
|
||
import ast
|
||
import json
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
import pytest_asyncio
|
||
from fastapi import FastAPI, HTTPException
|
||
from httpx import ASGITransport, AsyncClient
|
||
from sqlalchemy import JSON, select
|
||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||
|
||
from app.api import ai_routing
|
||
from app.core.database import get_db
|
||
from app.dependencies.auth import get_current_user
|
||
from app.dependencies.permissions import require_platform_ai_ops
|
||
from app.dependencies.tenant import get_tenant_id
|
||
from app.models.ai_call_log import AICallLog
|
||
from app.models.base import Base
|
||
from app.models.integration import Integration
|
||
from app.services.ai_routing_governance_service import AIRoutingGovernanceService
|
||
from app.services.ai_routing_governance_service import _summarize_capability_probe_status as summarize_governance_probe_status
|
||
from app.services.ai_function_runtime_profile_service import AIFunctionRuntimeProfileService
|
||
from app.services.llm_service import ChatCompletionResult, llm_service
|
||
from app.services.llm_service.mixin_engine_admin import _summarize_capability_probe_status as summarize_engine_probe_status
|
||
from app.services.model.model_routing import ModelRoutingService
|
||
|
||
|
||
def _use_sqlite_json_types(*tables) -> None:
|
||
for table in tables:
|
||
for column in table.columns:
|
||
if column.type.__class__.__name__ in {"JSONB", "ARRAY", "Vector"}:
|
||
column.type = JSON()
|
||
|
||
|
||
@pytest_asyncio.fixture
|
||
async def db_session():
|
||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||
tables = [Integration.__table__, AICallLog.__table__]
|
||
_use_sqlite_json_types(*tables)
|
||
async with engine.begin() as conn:
|
||
await conn.run_sync(lambda sync_conn: Base.metadata.create_all(sync_conn, tables=tables))
|
||
maker = async_sessionmaker(engine, expire_on_commit=False)
|
||
async with maker() as session:
|
||
yield session
|
||
await engine.dispose()
|
||
|
||
|
||
def _engine_bundle() -> dict:
|
||
return {
|
||
"active_engine_id": "low-1",
|
||
"settings": {"enable_local_kb_qa": False, "local_kb_qa_scope": "kb_qa_only"},
|
||
"engines": [
|
||
{
|
||
"id": "low-1",
|
||
"name": "Qwen 低阶引擎",
|
||
"provider": "dashscope",
|
||
"model": "qwen-plus",
|
||
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||
"api_key": "db-low-key",
|
||
"enabled": True,
|
||
"connection_status": "available",
|
||
"last_test_status": "success",
|
||
},
|
||
{
|
||
"id": "high-1",
|
||
"name": "OpenAI 高阶引擎",
|
||
"provider": "openai",
|
||
"model": "GLM-5",
|
||
"base_url": "https://api.edgefn.net/v1",
|
||
"api_key": "db-high-key",
|
||
"enabled": True,
|
||
"connection_status": "untested",
|
||
},
|
||
],
|
||
}
|
||
|
||
|
||
def test_declared_unverified_candidate_capabilities_keep_probe_status_partial() -> None:
|
||
declared = {
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"tool_calling": True,
|
||
"vision": True,
|
||
"audio": True,
|
||
"private_deployment": True,
|
||
"external_search_ready": True,
|
||
}
|
||
verified = {
|
||
"status": "verified",
|
||
"ok": True,
|
||
"evidence_strength": "live_probe",
|
||
}
|
||
probe = {
|
||
"evidence": {
|
||
"structured_output": dict(verified),
|
||
"json_schema": dict(verified),
|
||
"tool_calling": dict(verified),
|
||
}
|
||
}
|
||
|
||
assert summarize_engine_probe_status(declared=declared, probe=probe) == "partial"
|
||
assert summarize_governance_probe_status(declared=declared, probe=probe) == "partial"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_overview_masks_keys_and_ignores_env_placeholders(db_session, monkeypatch) -> None:
|
||
monkeypatch.setenv("DASHSCOPE_API_KEY", "local-placeholder")
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
data = await AIRoutingGovernanceService(db_session, tenant_id=1).get_overview()
|
||
|
||
assert data["config_version"]
|
||
assert data["slots"][0]["label"] == "快速槽"
|
||
assert data["slots"][0]["api_key_masked"] == "db-l...-key"
|
||
assert not any(item["code"] == "env_key_override" for item in data["diagnostics"])
|
||
assert data["slots"][0]["config_source"] == "供应商配置"
|
||
assert "api_env" not in str(data)
|
||
assert "local-placeholder" not in str(data)
|
||
assert "db-low-key" not in str(data)
|
||
|
||
console = await AIRoutingGovernanceService(db_session, tenant_id=1).get_admin_console()
|
||
assert console["providers"][0]["status_label"] == "可用"
|
||
assert console["providers"][0]["is_default_fallback"] is True
|
||
assert console["providers"][0]["capabilities"]["structured_output"] is True
|
||
assert console["providers"][1]["status_label"] == "未测试"
|
||
assert console["capability_slots"][0]["primary_engine_id"] == "low-1"
|
||
assert console["capability_slots"][0]["requirements"]["summary"]
|
||
assert console["scene_slot_matrix"][0]["scene_key"] == "project_launch"
|
||
assert console["scene_slot_matrix"][0]["scene_label"] == "项目启动草案"
|
||
assert console["scene_slot_matrix"][0]["current_slot"] == "structured"
|
||
assert console["scene_slot_matrix"][0]["required_slot"] == "structured"
|
||
assert console["scene_slot_matrix"][0]["active_runtime"]["slot_key"] == "structured"
|
||
assert console["scene_slot_matrix"][0]["active_slot_runtime"]["slot_key"] == "structured"
|
||
assert console["scene_slot_matrix"][0]["draft_slot_runtime"]["slot_key"] == "structured"
|
||
assert console["scene_slot_matrix"][0]["scene_contract"]["output_contract"] == "结构化草案 + 人工确认"
|
||
launch_profile = console["scene_slot_matrix"][0]["function_runtime_profiles"][0]
|
||
assert launch_profile["function_name"] == "pmflow_launch_draft_v1"
|
||
assert launch_profile["active"]["max_tokens"] == 10000
|
||
assert launch_profile["active"]["provider_timeout_seconds"] == 90
|
||
assert launch_profile["active"]["job_timeout_seconds"] == 180
|
||
assert launch_profile["draft"]["repair_max_tokens"] == 3600
|
||
assert launch_profile["draft"]["large_input_policy"] == "preprocess_then_generate"
|
||
change_review_row = next(item for item in console["scene_slot_matrix"] if item["scene_key"] == "change_review")
|
||
assert change_review_row["scene_contract"]["lifecycle"] == "active"
|
||
assert change_review_row["scene_contract"]["protected"] is True
|
||
change_review_profile = change_review_row["function_runtime_profiles"][0]
|
||
assert change_review_profile["function_name"] == "pmflow_change_review_extraction_v1"
|
||
assert change_review_profile["prompt_key"] == "PROMPT_PMFLOW_CHANGE_REVIEW_EXTRACTION"
|
||
assert change_review_profile["active"]["max_tokens"] == 10000
|
||
answer_presentation_row = next(item for item in console["scene_slot_matrix"] if item["scene_key"] == "pmflow_ai_answer_presentation")
|
||
answer_profiles = {
|
||
item["function_name"]: item
|
||
for item in answer_presentation_row["function_runtime_profiles"]
|
||
}
|
||
assert answer_profiles["customer_answer_composer"]["active"]["max_tokens"] == 10000
|
||
assert answer_profiles["customer_answer_verifier"]["active"]["max_tokens"] == 10000
|
||
operating_row = next(item for item in console["scene_slot_matrix"] if item["scene_key"] == "space_pmflow_ai_operating_runtime")
|
||
assert operating_row["scene_label"] == "Space AI Operating Mission"
|
||
assert operating_row["current_slot"] == "low"
|
||
operating_profiles = {
|
||
item["function_name"]: item
|
||
for item in operating_row["function_runtime_profiles"]
|
||
}
|
||
assert operating_profiles["pmflow_ai_operating_mission"]["active"]["provider_timeout_seconds"] == 60
|
||
assert operating_profiles["pmflow_ai_operating_mission"]["active"]["job_timeout_seconds"] == 90
|
||
agent_runtime_row = next(item for item in console["scene_slot_matrix"] if item["scene_key"] == "space_pmflow_ai_agent_runtime")
|
||
assert agent_runtime_row["scene_label"] == "Space AI Agent Runtime 节点"
|
||
agent_runtime_profiles = {
|
||
item["function_name"]: item
|
||
for item in agent_runtime_row["function_runtime_profiles"]
|
||
}
|
||
assert agent_runtime_profiles["pmflow_ai_agent_evidence_normalizer"]["active"]["max_tokens"] == 2800
|
||
assert agent_runtime_profiles["pmflow_ai_agent_answer_composer"]["active"]["provider_timeout_seconds"] == 90
|
||
external_query_row = next(item for item in console["scene_slot_matrix"] if item["scene_key"] == "external_query")
|
||
external_profiles = {
|
||
item["function_name"]: item
|
||
for item in external_query_row["function_runtime_profiles"]
|
||
}
|
||
assert external_profiles["external_query_dashscope_responses"]["required_slot"] == "external_query"
|
||
assert external_profiles["external_query_dashscope_responses"]["active"]["provider_timeout_seconds"] == 120
|
||
assert console["scene_slot_matrix"][0]["scene_contract"]["lifecycle"] == "active"
|
||
assert console["scene_slot_matrix"][0]["scene_health"]["status"] in {"blocked", "warning", "unknown", "healthy"}
|
||
assert console["scene_slot_matrix"][0]["release_gate"]["suggested_action"] in {"configure_slot", "probe_capability", "add_provider", "inspect_failures", "none"}
|
||
assert "draft_dirty" in console["scene_slot_matrix"][0]
|
||
assert isinstance(console["scene_slot_matrix"][0]["capability_gap"], dict)
|
||
assert isinstance(console["scene_slot_matrix"][0]["required_capabilities"], list)
|
||
assert any(item["slot_key"] == "structured" for item in console["scene_slot_matrix"][0]["slots"])
|
||
assert console["scene_slot_matrix"][0]["runtime_version"]
|
||
candidate = next(item for item in console["scene_slot_matrix"] if item["scene_key"] == "release_review")
|
||
assert candidate["scene_contract"]["lifecycle"] == "candidate"
|
||
assert candidate["scene_health"]["status"] == "unknown"
|
||
assert candidate["release_gate"]["publish_blocked"] is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_function_runtime_profile_draft_smoke_and_publish_gate(db_session) -> None:
|
||
service = AIFunctionRuntimeProfileService(db_session)
|
||
|
||
await service.update_profile(
|
||
function_name="pmflow_launch_draft_v1",
|
||
payload={
|
||
"max_tokens": 7600,
|
||
"repair_max_tokens": 3600,
|
||
"temperature": 0,
|
||
"foreground_wait_seconds": 12,
|
||
"provider_timeout_seconds": 120,
|
||
"job_timeout_seconds": 240,
|
||
"max_retries": 2,
|
||
"large_input_policy": "preprocess_then_generate",
|
||
},
|
||
updated_by=101,
|
||
)
|
||
blocked = await service.get_profiles()
|
||
assert blocked["dirty"] is True
|
||
assert blocked["validation"]["can_publish"] is False
|
||
assert any(item["code"] == "smoke_required" for item in blocked["validation"]["items"])
|
||
with pytest.raises(ValueError, match="pmflow_launch_draft_v1.*smoke_required"):
|
||
await service.publish_profiles(published_by=101, summary="publish profile")
|
||
|
||
await service.record_smoke(
|
||
function_name="pmflow_launch_draft_v1",
|
||
status="success",
|
||
summary="unit smoke",
|
||
checked_by=101,
|
||
)
|
||
await service.record_smoke(
|
||
function_name="pmflow_change_review_extraction_v1",
|
||
status="success",
|
||
summary="unit smoke",
|
||
checked_by=101,
|
||
)
|
||
published = await service.publish_profiles(published_by=101, summary="publish profile")
|
||
|
||
assert published["dirty"] is False
|
||
assert published["active_profiles"]["pmflow_launch_draft_v1"]["max_tokens"] == 7600
|
||
assert published["active_profiles"]["pmflow_launch_draft_v1"]["provider_timeout_seconds"] == 120
|
||
resolved = await service.resolve_profile(
|
||
function_name="pmflow_launch_draft_v1",
|
||
scene_key="project_launch",
|
||
prompt_key="PROMPT_PMFLOW_LAUNCH_DRAFT",
|
||
)
|
||
assert resolved.max_tokens == 7600
|
||
assert resolved.provider_timeout_seconds == 120
|
||
assert resolved.job_timeout_seconds == 240
|
||
assert resolved.profile_source == "active"
|
||
external_query_profile = await service.resolve_profile(
|
||
function_name="external_query_dashscope_responses",
|
||
scene_key="external_query",
|
||
)
|
||
assert external_query_profile.required_slot == "external_query"
|
||
assert external_query_profile.provider_timeout_seconds == 120
|
||
assert external_query_profile.routing_meta()["timeout_policy_source"] == "function_runtime_profile"
|
||
agent_answer_profile = await service.resolve_profile(
|
||
function_name="pmflow_ai_agent_answer_composer",
|
||
scene_key="space_pmflow_ai_agent_runtime",
|
||
prompt_key="PROMPT_PMFLOW_AI_AGENT_ANSWER_COMPOSER",
|
||
)
|
||
assert agent_answer_profile.max_tokens == 2600
|
||
assert agent_answer_profile.provider_timeout_seconds == 90
|
||
await service.update_profile(
|
||
function_name="pmflow_ai_agent_answer_composer",
|
||
payload={"runtime_mode": "active", "quality_gate_mode": "off"},
|
||
updated_by=101,
|
||
)
|
||
agent_answer_draft = await service.resolve_profile(
|
||
function_name="pmflow_ai_agent_answer_composer",
|
||
scene_key="space_pmflow_ai_agent_runtime",
|
||
prompt_key="PROMPT_PMFLOW_AI_AGENT_ANSWER_COMPOSER",
|
||
stage="draft",
|
||
)
|
||
assert agent_answer_draft.quality_gate_mode == "off"
|
||
|
||
|
||
def test_function_runtime_profile_update_request_keeps_runtime_controls() -> None:
|
||
body = ai_routing.FunctionRuntimeProfileUpdateRequest(
|
||
runtime_mode="bypass",
|
||
quality_gate_mode="off",
|
||
)
|
||
|
||
payload = body.model_dump(exclude_unset=True)
|
||
|
||
assert payload["runtime_mode"] == "bypass"
|
||
assert payload["quality_gate_mode"] == "off"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_platform_ai_ops_reads_provider_bundle_from_platform_runtime_scope(db_session) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=9,
|
||
type="llm_engine",
|
||
config={"active_engine_id": "wrong", "engines": []},
|
||
is_active=True,
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
data = await AIRoutingGovernanceService(db_session, tenant_id=9).get_admin_console()
|
||
|
||
assert data["providers"][0]["name"] == "Qwen 低阶引擎"
|
||
assert data["providers"][0]["enabled"] is True
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_provider_actions_follow_engine_state(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
bundle["default_fallback_engine_id"] = "low-1"
|
||
bundle["engines"].append(
|
||
{
|
||
"id": "disabled-1",
|
||
"name": "停用备用引擎",
|
||
"provider": "dashscope",
|
||
"model": "qwen-plus",
|
||
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||
"api_key": "db-disabled-key",
|
||
"enabled": False,
|
||
"connection_status": "available",
|
||
}
|
||
)
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
await db_session.commit()
|
||
|
||
data = await AIRoutingGovernanceService(db_session, tenant_id=99).get_admin_console()
|
||
providers = {item["id"]: item for item in data["providers"]}
|
||
|
||
assert providers["low-1"]["available_actions"] == {
|
||
"enable": False,
|
||
"disable": True,
|
||
"edit": True,
|
||
"test": True,
|
||
"capability_probe": True,
|
||
"set_default_fallback": False,
|
||
"delete": False,
|
||
}
|
||
assert providers["high-1"]["available_actions"]["test"] is True
|
||
assert providers["high-1"]["available_actions"]["capability_probe"] is False
|
||
assert providers["high-1"]["available_actions"]["set_default_fallback"] is False
|
||
assert providers["high-1"]["available_actions"]["delete"] is False
|
||
assert providers["disabled-1"]["available_actions"] == {
|
||
"enable": True,
|
||
"disable": False,
|
||
"edit": True,
|
||
"test": False,
|
||
"capability_probe": False,
|
||
"set_default_fallback": False,
|
||
"delete": True,
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_llm_engine_state_guards_block_invalid_provider_operations(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
bundle["default_fallback_engine_id"] = "low-1"
|
||
bundle["engines"].append(
|
||
{
|
||
"id": "disabled-1",
|
||
"name": "停用备用引擎",
|
||
"provider": "dashscope",
|
||
"model": "qwen-plus",
|
||
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||
"api_key": "db-disabled-key",
|
||
"enabled": False,
|
||
"connection_status": "available",
|
||
}
|
||
)
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
await db_session.commit()
|
||
|
||
with pytest.raises(ValueError, match="未通过连接测试"):
|
||
await llm_service.set_default_fallback_engine(db_session, tenant_id=99, engine_id="high-1")
|
||
|
||
with pytest.raises(ValueError, match="测试未通过"):
|
||
await llm_service.probe_engine_capabilities(db_session, tenant_id=99, engine_id="high-1")
|
||
|
||
with pytest.raises(ValueError, match="兜底默认"):
|
||
await llm_service.delete_engine(db_session, tenant_id=99, engine_id="low-1")
|
||
|
||
with pytest.raises(ValueError, match="启用中"):
|
||
await llm_service.delete_engine(db_session, tenant_id=99, engine_id="high-1")
|
||
|
||
with pytest.raises(ValueError, match="已停用"):
|
||
await llm_service.test_engine_connection(db_session, tenant_id=99, engine_id="disabled-1")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_admin_console_reports_slot_observability_from_ai_call_logs(db_session) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
success = AICallLog(
|
||
tenant_id=1,
|
||
function_name="pmflow_launch_draft_v1",
|
||
prompt="prompt",
|
||
response="{}",
|
||
tokens_used=12,
|
||
cost=0.003,
|
||
status="success",
|
||
planned_tier="structured",
|
||
executed_tier="structured",
|
||
routing_decision="active_slot",
|
||
route_source="active",
|
||
route_rule_name="structured",
|
||
routing_config_version="slot-v-test",
|
||
routing_config_updated_at="2026-04-29T00:00:00+08:00",
|
||
)
|
||
success.routing_meta = {"slot_key": "structured", "slot_runtime_source": "active_slot", "slot_fallback_index": 1}
|
||
success.latency_ms = 120
|
||
second_success = AICallLog(
|
||
tenant_id=1,
|
||
function_name="generate_task_breakdown",
|
||
prompt="prompt",
|
||
response="{}",
|
||
tokens_used=4,
|
||
cost=0.001,
|
||
status="success",
|
||
planned_tier="structured",
|
||
executed_tier="structured",
|
||
routing_decision="active_slot",
|
||
route_source="active",
|
||
route_rule_name="structured",
|
||
routing_config_version="slot-v-test",
|
||
routing_config_updated_at="2026-04-29T00:00:00+08:00",
|
||
)
|
||
second_success.routing_meta = {"slot_key": "structured", "slot_runtime_source": "active_slot", "slot_fallback_index": 0}
|
||
degraded = AICallLog(
|
||
tenant_id=1,
|
||
function_name="pmflow_requirement_plan_draft",
|
||
prompt="prompt",
|
||
response="",
|
||
tokens_used=8,
|
||
cost=0.002,
|
||
status="structured_invalid",
|
||
planned_tier="structured",
|
||
executed_tier="structured",
|
||
routing_decision="structured_slot_degraded",
|
||
route_source="active",
|
||
route_rule_name="structured",
|
||
routing_config_version="slot-v-test",
|
||
routing_config_updated_at="2026-04-29T00:00:00+08:00",
|
||
)
|
||
degraded.routing_meta = {"slot_key": "structured", "slot_runtime_source": "active_slot_degraded", "slot_degraded": True}
|
||
degraded.latency_ms = 180
|
||
db_session.add_all([success, second_success, degraded])
|
||
await db_session.commit()
|
||
|
||
data = await AIRoutingGovernanceService(db_session, tenant_id=1).get_admin_console()
|
||
by_slot = {item["slot_key"]: item for item in data["slot_observability"]}
|
||
structured = by_slot["structured"]["windows"]["last_24h"]
|
||
high = by_slot["high"]["windows"]["last_24h"]
|
||
|
||
assert structured["status"] == "sampled"
|
||
assert structured["total_calls"] == 3
|
||
assert structured["failed_calls"] == 1
|
||
assert structured["fallback_hits"] == 1
|
||
assert structured["degraded_calls"] == 1
|
||
assert structured["active_slot_calls"] == 2
|
||
assert structured["active_slot_degraded_calls"] == 1
|
||
assert structured["average_latency_ms"] == 150
|
||
assert structured["estimated_cost"] == 0.006
|
||
assert structured["recent_failure_samples"][0]["slot_runtime_source"] == "active_slot_degraded"
|
||
assert high["status"] == "empty"
|
||
assert high["sample_state_label"] == "暂无运行样本"
|
||
recommendations = data["slot_recommendations"]
|
||
codes = {item["code"] for item in recommendations}
|
||
assert "structured_high_failure_rate" in codes
|
||
assert "structured_fallback_primary_candidate" in codes
|
||
assert "structured_active_slot_degraded" in codes
|
||
degraded_recommendation = next(item for item in recommendations if item["code"] == "structured_active_slot_degraded")
|
||
assert degraded_recommendation["severity"] == "critical"
|
||
assert "7d degraded 调用 1 次" in degraded_recommendation["evidence_summary"]
|
||
scene_row = next(item for item in data["scene_slot_matrix"] if item["scene_key"] == "project_launch")
|
||
assert scene_row["scene_health"]["blockers"]
|
||
assert scene_row["release_gate"]["affected_by_draft"] in {True, False}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_admin_console_uses_function_runtime_slots_over_legacy_low_high_routes(db_session) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=1,
|
||
type="model_routing",
|
||
config={
|
||
"mappings": {
|
||
"adaptive_rag_answer_deep": {"tier_alias": "low", "enabled": True},
|
||
"pmflow_requirement_plan_draft": {"tier_alias": "low", "enabled": True},
|
||
}
|
||
},
|
||
is_active=True,
|
||
)
|
||
)
|
||
for idx in range(3):
|
||
db_session.add(
|
||
AICallLog(
|
||
tenant_id=1,
|
||
function_name="pmflow_launch_draft_v1",
|
||
prompt=f"prompt {idx}",
|
||
response="",
|
||
tokens_used=1,
|
||
cost=0,
|
||
status="structured_empty",
|
||
planned_tier="low",
|
||
executed_tier="low",
|
||
routing_decision="rule_match",
|
||
route_source="default",
|
||
route_rule_name="engine_low_fallback",
|
||
routing_config_version="v3",
|
||
routing_config_updated_at="2026-03-26T21:30:00+08:00",
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
data = await AIRoutingGovernanceService(db_session, tenant_id=1).get_admin_console()
|
||
|
||
codes = {item["code"] for item in data["diagnostics"]}
|
||
assert "protected_function_downgraded" not in codes
|
||
assert "structured_failure_rate" in codes
|
||
assert data["recent_failures"][0]["status_label"] == "模型返回为空"
|
||
assert data["scene_routes"][0]["label"] == "项目启动草案"
|
||
assert data["providers"][0]["api_key_masked"] == "db-l...-key"
|
||
assert "db-low-key" not in str(data)
|
||
assert "db-high-key" not in str(data)
|
||
assert "prompt 0" not in str(data)
|
||
assert "api_env" not in str(data)
|
||
launch_row = next(item for item in data["scene_slot_matrix"] if item["scene_key"] == "project_launch")
|
||
assert launch_row["recent_failure_rate"] == 1.0
|
||
assert launch_row["recent_failure_count"] == 3
|
||
assert launch_row["recent_failed_calls"] == 3
|
||
assert "项目启动草案" in launch_row["impact_summary"]
|
||
requirement_row = next(item for item in data["scene_slot_matrix"] if item["scene_key"] == "requirement_structuring")
|
||
requirement_functions = {item["function_name"]: item for item in requirement_row["functions"]}
|
||
assert requirement_functions["pmflow_requirement_plan_draft"]["tier"] == "structured"
|
||
assert requirement_functions["pmflow_requirement_plan_draft"]["source"] == "runtime_profile"
|
||
assert not any(
|
||
"pmflow_requirement_plan_draft 被路由到 快速槽" in reason
|
||
for slot in requirement_row["slots"]
|
||
for reason in slot.get("risk_reasons", [])
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_function_runtime_profile_required_slot_is_configurable_with_scene_contract(db_session) -> None:
|
||
service = AIFunctionRuntimeProfileService(db_session)
|
||
profiles = await service.get_profiles()
|
||
draft = profiles["draft_profiles"]["pmflow_requirement_plan_draft"]
|
||
|
||
assert draft["required_slot"] == "structured"
|
||
|
||
updated = await service.update_profile(
|
||
function_name="pmflow_requirement_plan_draft",
|
||
payload={"required_slot": "high"},
|
||
updated_by=101,
|
||
)
|
||
assert updated["draft_profiles"]["pmflow_requirement_plan_draft"]["required_slot"] == "high"
|
||
|
||
with pytest.raises(ValueError, match="required_slot"):
|
||
await service.update_profile(
|
||
function_name="pmflow_requirement_plan_draft",
|
||
payload={"required_slot": "low"},
|
||
updated_by=101,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_routing_blocks_protected_scene_downgrade_to_low_slot(db_session) -> None:
|
||
with pytest.raises(ValueError, match="保护场景不能降级"):
|
||
await ModelRoutingService.upsert_override(
|
||
db_session,
|
||
tenant_id=9,
|
||
function_name="pmflow_requirement_plan_draft",
|
||
tier_alias="low",
|
||
updated_by=101,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_team_policy_persists_integration_without_runtime_route_changes(db_session) -> None:
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=1)
|
||
payload = {
|
||
"strategy": "quality_first",
|
||
"scenes": [{"key": "project_launch", "policy": "深度起草并进入确认", "mode": "深度优先"}],
|
||
"automation_boundary": "所有写操作进入待确认。",
|
||
}
|
||
|
||
saved = await service.update_team_policy(payload=payload, updated_by=7)
|
||
await db_session.commit()
|
||
loaded = await service.get_team_policy()
|
||
|
||
assert saved["strategy_label"] == "质量优先"
|
||
assert loaded["scenes"][0]["policy"] == "深度起草并进入确认"
|
||
assert loaded["automation_boundary"] == "所有写操作进入待确认。"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_capability_slot_binding_persists_in_platform_runtime_scope(db_session) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=9)
|
||
saved = await service.update_capability_slot(
|
||
slot_key="high",
|
||
payload={
|
||
"primary_engine_id": "high-1",
|
||
"fallback_engine_ids": ["low-1"],
|
||
"failure_policy": "try_fallback_then_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
await db_session.commit()
|
||
|
||
high_slot = next(item for item in saved["slots"] if item["slot_key"] == "high")
|
||
assert high_slot["primary_engine_id"] == "high-1"
|
||
assert high_slot["fallback_engine_ids"] == ["low-1"]
|
||
row = (
|
||
await db_session.execute(
|
||
select(Integration).where(
|
||
Integration.tenant_id == 1,
|
||
Integration.type == "ai_capability_slots",
|
||
)
|
||
)
|
||
).scalar_one()
|
||
assert row.config["draft"]["slots"]["high"]["primary_engine_id"] == "high-1"
|
||
assert row.config["active"]["slots"].get("high") is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_runtime_activation_slots_are_configurable_and_candidates_explain_gaps(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "high-1":
|
||
engine["capabilities"] = {
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"tool_calling": False,
|
||
"long_context": True,
|
||
"max_context_tokens": 8192,
|
||
"capability_source": "manual",
|
||
"capability_probe_status": "untested",
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
await db_session.commit()
|
||
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=9)
|
||
saved = await service.update_capability_slot(
|
||
slot_key="structured",
|
||
payload={
|
||
"primary_engine_id": "high-1",
|
||
"fallback_engine_ids": [],
|
||
"failure_policy": "explicit_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
|
||
structured_slot = next(item for item in saved["draft_slots"] if item["slot_key"] == "structured")
|
||
tool_slot = next(item for item in saved["draft_slots"] if item["slot_key"] == "tool")
|
||
long_slot = next(item for item in saved["draft_slots"] if item["slot_key"] == "long_context")
|
||
external_slot = next(item for item in saved["draft_slots"] if item["slot_key"] == "external_query")
|
||
assert structured_slot["configurable"] is True
|
||
assert structured_slot["primary_engine_id"] == "high-1"
|
||
assert tool_slot["configurable"] is True
|
||
assert long_slot["configurable"] is True
|
||
assert external_slot["configurable"] is True
|
||
assert external_slot["external_provider_registry"]["mcp_adapter_available"] is True
|
||
assert tool_slot["blocked_candidate_engine_count"] >= 1
|
||
assert any("缺能力" in "；".join(item.get("gap_reasons") or []) for item in tool_slot["blocked_candidate_engines"])
|
||
assert long_slot["blocked_candidate_engine_count"] >= 1
|
||
|
||
saved = await service.update_capability_slot(
|
||
slot_key="external_query",
|
||
payload={
|
||
"primary_engine_id": "live_search",
|
||
"fallback_engine_ids": ["mcp"],
|
||
"failure_policy": "explicit_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
external_slot = next(item for item in saved["draft_slots"] if item["slot_key"] == "external_query")
|
||
assert external_slot["primary_provider_id"] == "live_search"
|
||
assert external_slot["fallback_provider_ids"] == ["mcp"]
|
||
assert external_slot["verifier_required"] is True
|
||
assert external_slot["source_filtering_required"] is True
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_capability_slot_publish_promotes_draft_to_active_and_keeps_history(db_session) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=9)
|
||
await service.update_capability_slot(
|
||
slot_key="low",
|
||
payload={
|
||
"primary_engine_id": "low-1",
|
||
"fallback_engine_ids": [],
|
||
"failure_policy": "upgrade_to_high",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
await service.update_capability_slot(
|
||
slot_key="high",
|
||
payload={
|
||
"primary_engine_id": "high-1",
|
||
"fallback_engine_ids": ["low-1"],
|
||
"failure_policy": "try_fallback_then_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
draft = await service.get_capability_slots()
|
||
assert draft["dirty"] is True
|
||
assert draft["validation"]["can_publish"] is True
|
||
assert any(item["code"] == "low_primary_capability_evidence_untested" and item["severity"] == "warning" for item in draft["validation"]["items"])
|
||
|
||
published = await service.publish_capability_slots(published_by=101, summary="unit test publish")
|
||
await db_session.commit()
|
||
|
||
high_active = next(item for item in published["active_slots"] if item["slot_key"] == "high")
|
||
assert high_active["primary_engine_id"] == "high-1"
|
||
assert published["dirty"] is False
|
||
assert published["history"][0]["summary"] == "unit test publish"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_external_query_slot_publish_requires_provider_verifier_and_source_filtering(db_session, monkeypatch) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=9)
|
||
await service.update_capability_slot(
|
||
slot_key="low",
|
||
payload={
|
||
"primary_engine_id": "low-1",
|
||
"fallback_engine_ids": [],
|
||
"failure_policy": "upgrade_to_high",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
await service.update_capability_slot(
|
||
slot_key="high",
|
||
payload={
|
||
"primary_engine_id": "high-1",
|
||
"fallback_engine_ids": [],
|
||
"failure_policy": "explicit_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
await service.update_capability_slot(
|
||
slot_key="external_query",
|
||
payload={
|
||
"primary_engine_id": "live_search",
|
||
"fallback_engine_ids": ["mcp"],
|
||
"failure_policy": "explicit_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
|
||
validation = await service.validate_capability_slots()
|
||
assert validation["can_publish"] is True
|
||
|
||
monkeypatch.setenv("AI_EXTERNAL_QUERY_VERIFIER_ENABLED", "0")
|
||
blocked = await service.validate_capability_slots()
|
||
assert blocked["can_publish"] is False
|
||
assert any(item["code"] == "external_query_verifier_unavailable" for item in blocked["items"])
|
||
|
||
monkeypatch.setenv("AI_EXTERNAL_QUERY_VERIFIER_ENABLED", "1")
|
||
monkeypatch.setenv("AI_EXTERNAL_QUERY_SOURCE_FILTERING_FAIL_CLOSED", "0")
|
||
blocked = await service.validate_capability_slots()
|
||
assert blocked["can_publish"] is False
|
||
assert any(item["code"] == "external_query_source_filtering_open" for item in blocked["items"])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_capability_slot_validation_blocks_runtime_that_misses_capability_matrix(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "high-1":
|
||
engine["capabilities"] = {"local_only": True, "structured_output": True, "json_schema": True}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
await db_session.commit()
|
||
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=9)
|
||
await service.update_capability_slot(
|
||
slot_key="low",
|
||
payload={
|
||
"primary_engine_id": "low-1",
|
||
"fallback_engine_ids": [],
|
||
"failure_policy": "upgrade_to_high",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
await service.update_capability_slot(
|
||
slot_key="high",
|
||
payload={
|
||
"primary_engine_id": "high-1",
|
||
"fallback_engine_ids": [],
|
||
"failure_policy": "explicit_fail",
|
||
"allow_scene_downgrade": False,
|
||
},
|
||
updated_by=101,
|
||
)
|
||
|
||
validation = await service.validate_capability_slots()
|
||
|
||
assert validation["can_publish"] is False
|
||
assert any(item["code"] == "high_主运行时_capability_missing" for item in validation["items"])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_reserved_slot_validation_applies_requirements_when_enabled(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "low-1":
|
||
engine["capabilities"] = {"structured_output": True, "json_schema": False}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=1,
|
||
type="ai_capability_slots",
|
||
config={
|
||
"active_version": "unpublished",
|
||
"draft": {
|
||
"slots": {
|
||
"low": {"primary_engine_id": "low-1", "fallback_engine_ids": []},
|
||
"high": {"primary_engine_id": "high-1", "fallback_engine_ids": []},
|
||
"structured": {"primary_engine_id": "low-1", "fallback_engine_ids": []},
|
||
}
|
||
},
|
||
},
|
||
is_active=True,
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
validation = await AIRoutingGovernanceService(db_session, tenant_id=9).validate_capability_slots()
|
||
|
||
assert validation["can_publish"] is False
|
||
assert any(item["code"] == "structured_主运行时_capability_missing" for item in validation["items"])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_reserved_slot_validation_distinguishes_missing_probe_evidence(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "low-1":
|
||
engine["capabilities"] = {
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"capability_source": "manual",
|
||
"capability_probe_status": "untested",
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=1,
|
||
type="ai_capability_slots",
|
||
config={
|
||
"active_version": "unpublished",
|
||
"draft": {
|
||
"slots": {
|
||
"low": {"primary_engine_id": "low-1", "fallback_engine_ids": []},
|
||
"high": {"primary_engine_id": "high-1", "fallback_engine_ids": []},
|
||
"structured": {"primary_engine_id": "low-1", "fallback_engine_ids": []},
|
||
}
|
||
},
|
||
},
|
||
is_active=True,
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
validation = await AIRoutingGovernanceService(db_session, tenant_id=9).validate_capability_slots()
|
||
|
||
assert validation["can_publish"] is False
|
||
assert any(item["code"] == "structured_主运行时_capability_evidence_missing" for item in validation["items"])
|
||
assert not any(item["code"] == "structured_主运行时_capability_missing" for item in validation["items"])
|
||
structured_slot = next(item for item in (await AIRoutingGovernanceService(db_session, tenant_id=9).get_capability_slots())["draft_slots"] if item["slot_key"] == "structured")
|
||
assert structured_slot["capability_coverage"]["capabilities"]["capability_probe"]["source"] == "manual"
|
||
assert structured_slot["capability_coverage"]["capabilities"]["capability_probe"]["status"] == "untested"
|
||
with pytest.raises(ValueError, match="草稿存在阻塞项"):
|
||
await AIRoutingGovernanceService(db_session, tenant_id=9).publish_capability_slots(
|
||
published_by=101,
|
||
summary="should block without probe evidence",
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_tool_and_long_context_slots_block_without_probe_evidence_when_enabled(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "high-1":
|
||
engine["capabilities"] = {
|
||
"tool_calling": True,
|
||
"long_context": True,
|
||
"max_context_tokens": 128000,
|
||
"capability_source": "manual",
|
||
"capability_probe_status": "untested",
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=1,
|
||
type="ai_capability_slots",
|
||
config={
|
||
"active_version": "unpublished",
|
||
"draft": {
|
||
"slots": {
|
||
"tool": {"primary_engine_id": "high-1", "fallback_engine_ids": []},
|
||
"long_context": {"primary_engine_id": "high-1", "fallback_engine_ids": []},
|
||
}
|
||
},
|
||
},
|
||
is_active=True,
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
service = AIRoutingGovernanceService(db_session, tenant_id=9)
|
||
validation = await service.validate_capability_slots()
|
||
|
||
assert validation["can_publish"] is False
|
||
assert any(item["code"] == "tool_主运行时_capability_evidence_missing" and item["severity"] == "critical" for item in validation["items"])
|
||
assert any(item["code"] == "long_context_主运行时_capability_evidence_missing" and item["severity"] == "critical" for item in validation["items"])
|
||
with pytest.raises(ValueError, match="草稿存在阻塞项"):
|
||
await service.publish_capability_slots(published_by=101, summary="should block tool long-context evidence gaps")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_operator_attestation_marks_long_context_as_verified_slot_evidence(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "high-1":
|
||
engine["connection_status"] = "available"
|
||
engine["capabilities"] = {
|
||
"long_context": True,
|
||
"max_context_tokens": 128000,
|
||
"capability_source": "manual",
|
||
"capability_probe_status": "partial",
|
||
"capability_probe": {
|
||
"status": "partial",
|
||
"last_verified_at": "2026-04-30T12:00:00+08:00",
|
||
"source": "probed",
|
||
"verified_capabilities": {},
|
||
"evidence": {
|
||
"model_self_report": {"ok": True, "status": "declared_only", "evidence_strength": "model_self_report"},
|
||
"long_context_128k": {
|
||
"ok": False,
|
||
"status": "declared_only",
|
||
"evidence_strength": "declared_only",
|
||
"tested_tokens": 128000,
|
||
},
|
||
},
|
||
},
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=1,
|
||
type="ai_capability_slots",
|
||
config={
|
||
"active_version": "unpublished",
|
||
"draft": {
|
||
"slots": {
|
||
"long_context": {"primary_engine_id": "high-1", "fallback_engine_ids": []},
|
||
}
|
||
},
|
||
},
|
||
is_active=True,
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
result = await llm_service.attest_engine_capability(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="high-1",
|
||
capability_key="long_context",
|
||
tested_tokens=1_048_576,
|
||
source_type="official_doc",
|
||
source_url="https://example.com/qwen-plus",
|
||
source_note="官方规格显示 1M 上下文",
|
||
user_id=101,
|
||
)
|
||
|
||
evidence = result["capabilities"]["capability_probe"]["evidence"]["long_context"]
|
||
assert evidence["ok"] is True
|
||
assert evidence["evidence_strength"] == "operator_verified"
|
||
assert evidence["method"] == "operator_attestation"
|
||
assert evidence["tested_tokens"] == 1_048_576
|
||
assert result["capabilities"]["effective_capabilities"]["long_context"] is True
|
||
assert result["capabilities"]["effective_capabilities"]["long_context_128k"] is True
|
||
assert result["capabilities"]["capability_probe"]["status"] == "passed"
|
||
assert "long_context_128k" not in result["capabilities"]["capability_probe"]["evidence"]
|
||
|
||
validation = await AIRoutingGovernanceService(db_session, tenant_id=9).validate_capability_slots()
|
||
assert not any(item["code"] == "long_context_主运行时_capability_evidence_missing" for item in validation["items"])
|
||
assert not any(item["code"] == "long_context_主运行时_capability_evidence_stale" for item in validation["items"])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_reserved_slot_validation_blocks_stale_probe_evidence(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "low-1":
|
||
engine["capabilities"] = {
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"capability_probe": {
|
||
"status": "passed",
|
||
"source": "probed",
|
||
"last_verified_at": "2025-01-01T00:00:00",
|
||
"verified_capabilities": {
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"tool_calling": False,
|
||
"long_context": False,
|
||
},
|
||
"evidence": {
|
||
"structured_output": {"ok": True, "checked_at": "2025-01-01T00:00:00", "summary": "ok"},
|
||
"json_schema": {"ok": True, "checked_at": "2025-01-01T00:00:00", "summary": "ok"},
|
||
},
|
||
},
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
db_session.add(
|
||
Integration(
|
||
tenant_id=1,
|
||
type="ai_capability_slots",
|
||
config={
|
||
"active_version": "unpublished",
|
||
"draft": {
|
||
"slots": {
|
||
"structured": {"primary_engine_id": "low-1", "fallback_engine_ids": []},
|
||
}
|
||
},
|
||
},
|
||
is_active=True,
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
validation = await AIRoutingGovernanceService(db_session, tenant_id=9).validate_capability_slots()
|
||
|
||
assert validation["can_publish"] is False
|
||
assert any(item["code"] == "structured_主运行时_capability_evidence_stale" for item in validation["items"])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_engine_capability_probe_writes_evidence_into_engine_config(db_session, monkeypatch) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
async def fake_structured_probe(**kwargs):
|
||
_ = kwargs
|
||
return ChatCompletionResult(text='{"ok": true, "label": "pmflow"}', raw_response={})
|
||
|
||
async def fake_tool_probe(*, engine):
|
||
_ = engine
|
||
return {"status": "verified", "message": "tool ok", "latency_ms": 11, "evidence_strength": "live_probe"}
|
||
|
||
async def fake_long_context_probe(*, engine, min_context_tokens=32768):
|
||
_ = engine
|
||
return {
|
||
"status": "verified",
|
||
"message": "long context ok",
|
||
"latency_ms": 12,
|
||
"max_context_tokens": min_context_tokens,
|
||
"evidence_strength": "live_probe",
|
||
}
|
||
|
||
async def fake_json_schema_probe(*, engine):
|
||
_ = engine
|
||
return {"status": "verified", "message": "json schema ok", "latency_ms": 7, "evidence_strength": "live_probe"}
|
||
|
||
monkeypatch.setattr(llm_service, "_chat_completion_raw", fake_structured_probe)
|
||
monkeypatch.setattr(llm_service, "_probe_json_schema_capability", fake_json_schema_probe)
|
||
monkeypatch.setattr(llm_service, "_probe_tool_calling_capability", fake_tool_probe)
|
||
monkeypatch.setattr(llm_service, "_probe_long_context_capability", fake_long_context_probe)
|
||
|
||
result = await llm_service.probe_engine_capabilities(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="low-1",
|
||
targets=["structured", "tool", "long_context"],
|
||
)
|
||
|
||
assert result["status"] == "passed"
|
||
assert result["capabilities"]["capability_source"] == "probe"
|
||
assert result["capabilities"]["capability_probe"]["source"] == "probed"
|
||
assert result["capabilities"]["capability_probe"]["status"] == "passed"
|
||
assert result["capabilities"]["capability_probe"]["verified_capabilities"]["tool_calling"] is True
|
||
assert result["capabilities"]["capability_probe"]["evidence"]["tool_calling"]["ok"] is True
|
||
assert result["capabilities"]["capability_evidence"]["tool_calling"]["status"] == "verified"
|
||
row = (
|
||
await db_session.execute(
|
||
select(Integration).where(
|
||
Integration.tenant_id == 1,
|
||
Integration.type == "llm_engine",
|
||
)
|
||
)
|
||
).scalar_one()
|
||
saved = next(item for item in row.config["engines"] if item["id"] == "low-1")
|
||
assert saved["capabilities"]["capability_probe_status"] == "passed"
|
||
assert saved["capabilities"]["capability_probe"]["status"] == "passed"
|
||
assert saved["capabilities"]["capability_evidence"]["long_context"]["status"] == "verified"
|
||
assert saved["capabilities"]["capability_probe"]["evidence"]["long_context"]["tested_tokens"] == 32768
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_long_context_probe_uses_declared_window_not_32k_default(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "low-1":
|
||
engine["capabilities"] = {
|
||
"long_context": True,
|
||
"max_context_tokens": 16384,
|
||
"capability_source": "manual",
|
||
"capability_probe_status": "untested",
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
await db_session.commit()
|
||
|
||
result = await llm_service.probe_engine_capabilities(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="low-1",
|
||
targets=["long_context"],
|
||
)
|
||
|
||
evidence = result["capabilities"]["capability_probe"]["evidence"]["long_context"]
|
||
assert result["status"] == "passed"
|
||
assert evidence["status"] == "verified"
|
||
assert evidence["evidence_strength"] == "provider_contract"
|
||
assert evidence["tested_tokens"] == 16384
|
||
assert "32768" not in evidence["error_summary"]
|
||
assert result["capabilities"]["effective_capabilities"]["long_context"] is True
|
||
assert result["capabilities"]["effective_capabilities"]["long_context_32k"] is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_self_report_probe_is_auxiliary_not_verified_slot_evidence(db_session, monkeypatch) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
async def fake_self_report_completion(**kwargs):
|
||
assert kwargs["function_name"] == "probe_capability_model_self_report"
|
||
payload = {
|
||
"model_name": "qwen-plus",
|
||
"context_window_tokens": 1000000,
|
||
"max_input_tokens": 997952,
|
||
"max_output_tokens": 32768,
|
||
"context_tier": "1m",
|
||
"supports_32k": True,
|
||
"supports_128k": True,
|
||
"supports_1m": True,
|
||
"confidence": "medium",
|
||
"note": "self report only",
|
||
}
|
||
return ChatCompletionResult(
|
||
text=json.dumps(payload),
|
||
raw_request={"model": kwargs["model"], "messages": [{"role": "user", "content": kwargs["prompt"]}]},
|
||
raw_response={"choices": [{"message": {"content": json.dumps(payload)}}]},
|
||
provider_attempts=[{"status": "success", "provider": kwargs["provider"], "model": kwargs["model"]}],
|
||
)
|
||
|
||
monkeypatch.setattr(llm_service, "_chat_completion_raw", fake_self_report_completion)
|
||
|
||
result = await llm_service.probe_engine_capabilities(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="low-1",
|
||
targets=["self_report"],
|
||
)
|
||
|
||
evidence = result["capabilities"]["capability_probe"]["evidence"]["model_self_report"]
|
||
assert result["status"] == "partial"
|
||
assert result["targets"] == ["model_self_report"]
|
||
assert evidence["status"] == "declared_only"
|
||
assert evidence["evidence_strength"] == "model_self_report"
|
||
assert evidence["ok"] is False
|
||
assert evidence["model_self_report"]["context_window_tokens"] == 1000000
|
||
assert result["capabilities"]["capability_probe"]["verified_capabilities"]["model_self_report"] is False
|
||
assert result["capabilities"]["effective_capabilities"]["long_context"] is False
|
||
|
||
def test_default_full_probe_targets_include_model_self_report_reference() -> None:
|
||
targets = llm_service._normalize_capability_probe_targets(
|
||
None,
|
||
mode="full",
|
||
capabilities={"structured_output": True, "json_schema": True, "tool_calling": True, "long_context": False},
|
||
)
|
||
|
||
assert targets[:2] == ["basic_chat", "model_self_report"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_self_report_does_not_downgrade_verified_probe_status(db_session, monkeypatch) -> None:
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=_engine_bundle(), is_active=True))
|
||
await db_session.commit()
|
||
|
||
async def fake_self_report_completion(**kwargs):
|
||
payload = {"model_name": kwargs["model"], "context_window_tokens": 1000000, "confidence": "medium"}
|
||
return ChatCompletionResult(text=json.dumps(payload), raw_request={"model": kwargs["model"]}, raw_response={"content": payload})
|
||
|
||
async def fake_tool_probe(*, engine):
|
||
_ = engine
|
||
return {"status": "verified", "message": "tool ok", "latency_ms": 11, "evidence_strength": "live_probe"}
|
||
|
||
async def fake_long_context_probe(*, engine, min_context_tokens=None):
|
||
_ = engine, min_context_tokens
|
||
return {
|
||
"status": "verified",
|
||
"message": "long ok",
|
||
"latency_ms": None,
|
||
"max_context_tokens": 1000000,
|
||
"declared_ok": True,
|
||
"evidence_strength": "provider_contract",
|
||
}
|
||
|
||
monkeypatch.setattr(llm_service, "_chat_completion_raw", fake_self_report_completion)
|
||
monkeypatch.setattr(llm_service, "_probe_tool_calling_capability", fake_tool_probe)
|
||
monkeypatch.setattr(llm_service, "_probe_long_context_capability", fake_long_context_probe)
|
||
|
||
result = await llm_service.probe_engine_capabilities(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="low-1",
|
||
targets=["self_report", "tool", "long_context"],
|
||
)
|
||
|
||
assert result["status"] == "passed"
|
||
assert result["capabilities"]["capability_probe"]["evidence"]["model_self_report"]["evidence_strength"] == "model_self_report"
|
||
assert result["capabilities"]["capability_probe"]["verified_capabilities"]["tool_calling"] is True
|
||
assert result["capabilities"]["capability_probe"]["verified_capabilities"]["model_self_report"] is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_engine_update_resets_probe_evidence_when_declared_capabilities_change(db_session) -> None:
|
||
bundle = _engine_bundle()
|
||
for engine in bundle["engines"]:
|
||
if engine["id"] == "low-1":
|
||
engine["capabilities"] = {
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"tool_calling": True,
|
||
"long_context": True,
|
||
"max_context_tokens": 32768,
|
||
"capability_source": "probe",
|
||
"capability_probe_status": "passed",
|
||
"capability_last_verified_at": "2026-04-30T12:00:00+08:00",
|
||
"capability_probe": {
|
||
"status": "passed",
|
||
"source": "probed",
|
||
"last_verified_at": "2026-04-30T12:00:00+08:00",
|
||
"verified_capabilities": {"structured_output": True, "json_schema": True, "tool_calling": True, "long_context": True},
|
||
"evidence": {
|
||
"structured_output": {
|
||
"ok": True,
|
||
"status": "verified",
|
||
"evidence_strength": "live_probe",
|
||
"checked_at": "2026-04-30T12:00:00+08:00",
|
||
"summary": "structured ok",
|
||
},
|
||
"json_schema": {
|
||
"ok": True,
|
||
"status": "verified",
|
||
"evidence_strength": "live_probe",
|
||
"checked_at": "2026-04-30T12:00:00+08:00",
|
||
"summary": "json ok",
|
||
},
|
||
"tool_calling": {
|
||
"ok": True,
|
||
"status": "verified",
|
||
"evidence_strength": "live_probe",
|
||
"checked_at": "2026-04-30T12:00:00+08:00",
|
||
"summary": "tool ok",
|
||
},
|
||
"long_context": {
|
||
"ok": True,
|
||
"status": "verified",
|
||
"evidence_strength": "live_probe",
|
||
"checked_at": "2026-04-30T12:00:00+08:00",
|
||
"tested_tokens": 32768,
|
||
"summary": "long ok",
|
||
},
|
||
},
|
||
},
|
||
"capability_evidence": {
|
||
"structured_output": {"status": "verified", "source": "probe", "verified_at": "2026-04-30T12:00:00+08:00", "message": "structured ok", "evidence_strength": "live_probe"},
|
||
"json_schema": {"status": "verified", "source": "probe", "verified_at": "2026-04-30T12:00:00+08:00", "message": "json ok", "evidence_strength": "live_probe"},
|
||
"tool_calling": {"status": "verified", "source": "probe", "verified_at": "2026-04-30T12:00:00+08:00", "message": "tool ok", "evidence_strength": "live_probe"},
|
||
"long_context": {"status": "verified", "source": "probe", "verified_at": "2026-04-30T12:00:00+08:00", "message": "long ok", "evidence_strength": "live_probe", "max_context_tokens": 32768},
|
||
},
|
||
}
|
||
db_session.add(Integration(tenant_id=1, type="llm_engine", config=bundle, is_active=True))
|
||
await db_session.commit()
|
||
|
||
renamed = await llm_service.update_engine(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="low-1",
|
||
name="Qwen renamed",
|
||
)
|
||
assert renamed["capabilities"]["capability_probe_status"] == "passed"
|
||
assert renamed["capabilities"]["capability_probe"]["evidence"]["tool_calling"]["ok"] is True
|
||
|
||
changed = await llm_service.update_engine(
|
||
db_session,
|
||
tenant_id=9,
|
||
engine_id="low-1",
|
||
capabilities={
|
||
"structured_output": True,
|
||
"json_schema": True,
|
||
"tool_calling": True,
|
||
"long_context": True,
|
||
"max_context_tokens": 16384,
|
||
"cost_tier": "low",
|
||
"reliability_tier": "standard",
|
||
},
|
||
)
|
||
assert changed["capabilities"]["capability_probe_status"] == "untested"
|
||
assert changed["capabilities"]["capability_last_verified_at"] is None
|
||
assert changed["capabilities"]["capability_probe"]["evidence"] == {}
|
||
assert changed["capabilities"]["capability_evidence"] == {}
|
||
assert changed["capabilities"]["effective_capabilities"]["tool_calling"] is False
|
||
assert changed["capabilities"]["max_context_tokens"] == 16384
|
||
|
||
|
||
def _build_app(*, role: str = "REVIEWER") -> FastAPI:
|
||
app = FastAPI()
|
||
app.include_router(ai_routing.router)
|
||
fake_user = SimpleNamespace(
|
||
id=101,
|
||
tenant_id=9,
|
||
username="tester",
|
||
role=role,
|
||
)
|
||
|
||
class FakeDb:
|
||
async def commit(self) -> None:
|
||
return None
|
||
|
||
async def _fake_get_db():
|
||
yield FakeDb()
|
||
|
||
async def _fake_platform_ai_ops():
|
||
if role not in {"OWNER", "REVIEWER"}:
|
||
raise HTTPException(status_code=403, detail="Platform capability required: platform.ai_routing.manage")
|
||
return fake_user
|
||
|
||
app.dependency_overrides[get_db] = _fake_get_db
|
||
app.dependency_overrides[get_tenant_id] = lambda: 9
|
||
app.dependency_overrides[get_current_user] = lambda: fake_user
|
||
app.dependency_overrides[require_platform_ai_ops] = _fake_platform_ai_ops
|
||
return app
|
||
|
||
|
||
def test_api_router_exposes_ai_routing_routes() -> None:
|
||
router_source = (Path(__file__).parents[1] / "app" / "api" / "router.py").read_text()
|
||
router_tree = ast.parse(router_source)
|
||
route_assignments = {}
|
||
for node in router_tree.body:
|
||
if not isinstance(node, ast.Assign):
|
||
continue
|
||
for target in node.targets:
|
||
if isinstance(target, ast.Name) and target.id in {"CORE_ROUTE_MODULES", "OPTIONAL_ROUTE_GROUPS"}:
|
||
route_assignments[target.id] = ast.literal_eval(node.value)
|
||
|
||
assert "ai_routing" in route_assignments["CORE_ROUTE_MODULES"]
|
||
assert "model_routing" in route_assignments["CORE_ROUTE_MODULES"]
|
||
assert "llm_engines" in route_assignments["CORE_ROUTE_MODULES"]
|
||
assert "ai_routing" not in route_assignments["OPTIONAL_ROUTE_GROUPS"]["ai_control"]
|
||
assert "model_routing" not in route_assignments["OPTIONAL_ROUTE_GROUPS"]["ai_control"]
|
||
assert "llm_engines" not in route_assignments["OPTIONAL_ROUTE_GROUPS"]["ai_control"]
|
||
route_paths = {str(getattr(route, "path", "")).strip() for route in ai_routing.router.routes}
|
||
assert "/ai-routing/overview" in route_paths
|
||
assert "/ai-routing/team-policy" in route_paths
|
||
assert "/ai-routing/admin-console" in route_paths
|
||
assert "/ai-routing/capability-slots" in route_paths
|
||
assert "/ai-routing/capability-slots/{slot_key}/draft" in route_paths
|
||
assert "/ai-routing/capability-slots/validate" in route_paths
|
||
assert "/ai-routing/capability-slots/publish" in route_paths
|
||
assert "/ai-routing/capability-slots/rollback" in route_paths
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_ai_routing_overview_requires_governance_role() -> None:
|
||
app = _build_app(role="BUILDER")
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||
response = await client.get("/ai-routing/overview")
|
||
|
||
assert response.status_code == 403
|
||
payload = response.json()
|
||
assert payload.get("code") == 403 or "platform.ai_routing.manage" in str(payload.get("detail") or "")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_ai_routing_api_returns_enveloped_payloads(monkeypatch) -> None:
|
||
operation_log_calls = []
|
||
|
||
class FakeService:
|
||
def __init__(self, db, *, tenant_id: int):
|
||
assert tenant_id == 9
|
||
_ = db
|
||
|
||
async def get_overview(self):
|
||
return {"slots": [], "diagnostics": []}
|
||
|
||
async def get_team_policy(self):
|
||
return {"strategy": "balanced", "scenes": []}
|
||
|
||
async def update_team_policy(self, *, payload, updated_by: int):
|
||
assert updated_by == 101
|
||
return {**payload, "updated_by": updated_by}
|
||
|
||
async def get_admin_console(self):
|
||
return {"providers": [], "scene_routes": []}
|
||
|
||
async def get_capability_slots(self):
|
||
return {
|
||
"version": "draft",
|
||
"active_version": "unpublished",
|
||
"draft_version": "draft",
|
||
"dirty": False,
|
||
"validation": {"can_publish": True, "summary": {"ok": 1}, "items": []},
|
||
"history": [],
|
||
"slots": [],
|
||
"active_slots": [],
|
||
"draft_slots": [],
|
||
}
|
||
|
||
async def update_capability_slot(self, *, slot_key, payload, updated_by: int):
|
||
assert slot_key == "high"
|
||
assert updated_by == 101
|
||
return {
|
||
"version": "draft",
|
||
"active_version": "unpublished",
|
||
"draft_version": "draft",
|
||
"dirty": True,
|
||
"validation": {"can_publish": True, "summary": {"ok": 1}, "items": []},
|
||
"history": [],
|
||
"slots": [{**payload, "slot_key": slot_key}],
|
||
"active_slots": [],
|
||
"draft_slots": [{**payload, "slot_key": slot_key}],
|
||
}
|
||
|
||
async def validate_capability_slots(self):
|
||
return {"can_publish": True, "summary": {"ok": 1}, "items": []}
|
||
|
||
async def publish_capability_slots(self, *, published_by: int, summary=None):
|
||
assert published_by == 101
|
||
return {
|
||
"version": "slot-v1",
|
||
"active_version": "slot-v1",
|
||
"draft_version": "draft",
|
||
"dirty": False,
|
||
"validation": {"can_publish": True, "summary": {"ok": 1}, "items": []},
|
||
"history": [{"version": "slot-v1"}],
|
||
"slots": [],
|
||
"active_slots": [],
|
||
"draft_slots": [],
|
||
}
|
||
|
||
async def rollback_capability_slots(self, *, target_version, rolled_back_by: int):
|
||
assert target_version is None
|
||
assert rolled_back_by == 101
|
||
return {
|
||
"version": "slot-v2",
|
||
"active_version": "slot-v2",
|
||
"draft_version": "draft",
|
||
"dirty": False,
|
||
"validation": {"can_publish": True, "summary": {"ok": 1}, "items": []},
|
||
"history": [{"version": "slot-v2"}],
|
||
"slots": [],
|
||
"active_slots": [],
|
||
"draft_slots": [],
|
||
}
|
||
|
||
async def run_diagnostics(self):
|
||
return {"items": [], "summary": {"ok": 1}}
|
||
|
||
async def fake_log_operation(*args, **kwargs):
|
||
operation_log_calls.append({"args": args, "kwargs": kwargs})
|
||
|
||
monkeypatch.setattr(ai_routing, "AIRoutingGovernanceService", FakeService)
|
||
monkeypatch.setattr(ai_routing, "log_operation", fake_log_operation)
|
||
|
||
app = _build_app(role="OWNER")
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||
overview = await client.get("/ai-routing/overview")
|
||
policy = await client.get("/ai-routing/team-policy")
|
||
saved = await client.put("/ai-routing/team-policy", json={"strategy": "quality_first", "scenes": []})
|
||
console = await client.get("/ai-routing/admin-console")
|
||
slots = await client.put("/ai-routing/capability-slots/high", json={"primary_engine_id": "high-1", "fallback_engine_ids": []})
|
||
slot_draft = await client.put("/ai-routing/capability-slots/high/draft", json={"primary_engine_id": "high-1", "fallback_engine_ids": []})
|
||
slot_validation = await client.post("/ai-routing/capability-slots/validate")
|
||
slot_publish = await client.post("/ai-routing/capability-slots/publish", json={"summary": "publish"})
|
||
slot_rollback = await client.post("/ai-routing/capability-slots/rollback", json={})
|
||
diagnostics = await client.post("/ai-routing/diagnostics/run")
|
||
|
||
assert overview.json()["code"] == 0
|
||
assert policy.json()["data"]["strategy"] == "balanced"
|
||
assert saved.json()["data"]["strategy"] == "quality_first"
|
||
assert console.json()["data"]["providers"] == []
|
||
assert slots.json()["data"]["slots"][0]["slot_key"] == "high"
|
||
assert slot_draft.json()["data"]["dirty"] is True
|
||
assert slot_validation.json()["data"]["can_publish"] is True
|
||
assert slot_publish.json()["data"]["active_version"] == "slot-v1"
|
||
assert slot_rollback.json()["data"]["active_version"] == "slot-v2"
|
||
assert diagnostics.json()["data"]["summary"]["ok"] == 1
|
||
assert len(operation_log_calls) >= 1
|
||
log_kwargs = next(
|
||
item["kwargs"]
|
||
for item in operation_log_calls
|
||
if item["kwargs"].get("entity_type") == "ai_routing_policy"
|
||
)
|
||
assert log_kwargs["tenant_id"] == 9
|
||
assert log_kwargs["user_id"] == 101
|
||
assert log_kwargs["entity_type"] == "ai_routing_policy"
|
||
assert log_kwargs["entity_id"] == "team-policy"
|
||
assert log_kwargs["action"] == "update"
|
||
assert "provider" not in str(log_kwargs["after_data"]).lower()
|
||
assert "model" not in str(log_kwargs["after_data"]).lower()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_reviewer_cannot_save_team_policy(monkeypatch) -> None:
|
||
class FakeService:
|
||
pass
|
||
|
||
monkeypatch.setattr(ai_routing, "AIRoutingGovernanceService", FakeService)
|
||
app = _build_app(role="REVIEWER")
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||
response = await client.put("/ai-routing/team-policy", json={"strategy": "quality_first", "scenes": []})
|
||
|
||
assert response.status_code == 403
|
||
assert response.json()["message"] == "仅 OWNER 可保存团队 AI 策略"
|