436 lines
17 KiB
Python
436 lines
17 KiB
Python
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
import pytest_asyncio
|
||
from fastapi import FastAPI
|
||
from httpx import ASGITransport, AsyncClient
|
||
from sqlalchemy import JSON
|
||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||
|
||
os.environ["DATABASE_URL"] = "postgresql+asyncpg://tester:tester@localhost:5432/pmflow_test"
|
||
|
||
from app.api.ai_steward_api import routes_mounted_ai
|
||
from app.core.database import get_db
|
||
from app.dependencies.auth import get_current_user
|
||
from app.dependencies.tenant import get_tenant_id
|
||
from app.models.base import Base
|
||
from app.models.mounted_ai_runtime import (
|
||
MountedAgentTrace,
|
||
MountedConversationRun,
|
||
MountedDraftWorkspace,
|
||
MountedSourceActivation,
|
||
MountedToolCapabilityPolicy,
|
||
MountedTopicWorkspace,
|
||
)
|
||
from app.models.reply_iteration import JudgeRun, QualityAsset, QualityAssetPair
|
||
from app.models.training_pipeline import TrainingDataset, TrainingEvaluation, TrainingJob
|
||
from app.models.training_pipeline import TrainedModelArtifact, TrainedModelDeployment
|
||
|
||
|
||
def _use_sqlite_json_types() -> None:
|
||
for table in (
|
||
MountedTopicWorkspace.__table__,
|
||
MountedDraftWorkspace.__table__,
|
||
MountedConversationRun.__table__,
|
||
MountedAgentTrace.__table__,
|
||
MountedSourceActivation.__table__,
|
||
MountedToolCapabilityPolicy.__table__,
|
||
QualityAsset.__table__,
|
||
QualityAssetPair.__table__,
|
||
JudgeRun.__table__,
|
||
TrainingDataset.__table__,
|
||
TrainingJob.__table__,
|
||
TrainingEvaluation.__table__,
|
||
TrainedModelArtifact.__table__,
|
||
TrainedModelDeployment.__table__,
|
||
):
|
||
for column in table.columns:
|
||
if column.name in {
|
||
"facts_json",
|
||
"goals_json",
|
||
"constraints_json",
|
||
"options_json",
|
||
"open_questions",
|
||
"pending_actions",
|
||
"bound_entities",
|
||
"metadata_json",
|
||
"requirement_card",
|
||
"draft_payload",
|
||
"required_confirmations",
|
||
"context_layers_hit",
|
||
"tool_calls",
|
||
"execution_outcome",
|
||
"input_summary",
|
||
"output_summary",
|
||
"tool_trace",
|
||
"metadata_json",
|
||
"source_scope",
|
||
"tenant_scope",
|
||
"policy_metadata",
|
||
"source_snapshot",
|
||
"tags",
|
||
"findings",
|
||
"llm_evidence",
|
||
"selection_rules",
|
||
"channel_scope",
|
||
"metrics_json",
|
||
"evaluation_summary",
|
||
"effective_scope",
|
||
}:
|
||
column.type = JSON()
|
||
|
||
|
||
@pytest_asyncio.fixture
|
||
async def db_session():
|
||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||
_use_sqlite_json_types()
|
||
async with engine.begin() as conn:
|
||
await conn.run_sync(
|
||
lambda sync_conn: Base.metadata.create_all(
|
||
sync_conn,
|
||
tables=[
|
||
MountedTopicWorkspace.__table__,
|
||
MountedDraftWorkspace.__table__,
|
||
MountedConversationRun.__table__,
|
||
MountedAgentTrace.__table__,
|
||
MountedSourceActivation.__table__,
|
||
MountedToolCapabilityPolicy.__table__,
|
||
QualityAsset.__table__,
|
||
QualityAssetPair.__table__,
|
||
JudgeRun.__table__,
|
||
TrainingDataset.__table__,
|
||
TrainingJob.__table__,
|
||
TrainingEvaluation.__table__,
|
||
TrainedModelArtifact.__table__,
|
||
TrainedModelDeployment.__table__,
|
||
],
|
||
)
|
||
)
|
||
maker = async_sessionmaker(engine, expire_on_commit=False)
|
||
async with maker() as session:
|
||
yield session
|
||
await engine.dispose()
|
||
|
||
|
||
def _build_app(db_session):
|
||
app = FastAPI()
|
||
app.include_router(routes_mounted_ai.router)
|
||
|
||
async def _fake_get_db():
|
||
yield db_session
|
||
|
||
app.dependency_overrides[get_db] = _fake_get_db
|
||
app.dependency_overrides[get_tenant_id] = lambda: 1
|
||
app.dependency_overrides[get_current_user] = lambda: SimpleNamespace(
|
||
id=9001,
|
||
tenant_id=1,
|
||
role="ADMIN",
|
||
name="管理员",
|
||
username="admin",
|
||
)
|
||
return app
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mounted_ai_admin_routes_return_runtime_views(db_session) -> None:
|
||
topic = MountedTopicWorkspace(
|
||
tenant_id=1,
|
||
topic_key="feishu:chat_1:topic_1",
|
||
source_system="feishu",
|
||
channel_ref="chat_1",
|
||
thread_key="thread_1",
|
||
mode="plan",
|
||
status="active",
|
||
open_questions=["负责人是谁"],
|
||
readiness_score=0.4,
|
||
)
|
||
db_session.add(topic)
|
||
await db_session.flush()
|
||
|
||
draft = MountedDraftWorkspace(
|
||
tenant_id=1,
|
||
topic_workspace_id=topic.id,
|
||
state="collecting",
|
||
draft_version=2,
|
||
requirement_card={"goals": ["做挂载 AI"]},
|
||
)
|
||
run = MountedConversationRun(
|
||
tenant_id=1,
|
||
trace_id="mounted-run-1",
|
||
topic_workspace_id=topic.id,
|
||
source_system="feishu",
|
||
source_message_id="om_123",
|
||
mode_before="plan",
|
||
mode_after="clarify",
|
||
active_agent="Clarifier",
|
||
status="completed",
|
||
trigger_reason="missing_owner",
|
||
context_layers_hit={"thread": True, "topic": True},
|
||
final_reply="还差负责人信息。",
|
||
)
|
||
db_session.add_all([draft, run])
|
||
await db_session.flush()
|
||
|
||
trace = MountedAgentTrace(
|
||
tenant_id=1,
|
||
conversation_run_id=run.id,
|
||
agent_name="Clarifier",
|
||
status="completed",
|
||
latency_ms=128,
|
||
input_summary={"mode": "plan"},
|
||
output_summary={"question": "这件事打算谁来负责？"},
|
||
tool_trace=[{"tool_name": "knowledge_graph.search", "status": "completed"}],
|
||
)
|
||
policy = MountedToolCapabilityPolicy(
|
||
tenant_id=1,
|
||
tool_name="project.create",
|
||
risk_level="high",
|
||
confirm_required=True,
|
||
auto_execute_allowed=False,
|
||
)
|
||
activation = MountedSourceActivation(
|
||
tenant_id=1,
|
||
source_system="feishu",
|
||
scope_type="chat",
|
||
scope_ref="oc_runtime_lab",
|
||
status="active",
|
||
cutover_mode="mounted_only",
|
||
auto_reply_enabled=True,
|
||
metadata_json={"allow_private": True},
|
||
)
|
||
failure_asset = QualityAsset(
|
||
tenant_id=1,
|
||
project_id=None,
|
||
asset_type="failure",
|
||
status="approved",
|
||
prompt_key="mounted.runtime.reply",
|
||
workflow_type="mounted.plan",
|
||
asked_via="feishu",
|
||
question_text="帮我判断下这个需求应该谁来接。",
|
||
answer_text="目前还缺负责人信息。",
|
||
corrected_answer="建议先给后端负责人 A，再确认业务 owner。",
|
||
source_snapshot={"run_id": 1},
|
||
tags=["mounted", "failure"],
|
||
)
|
||
golden_asset = QualityAsset(
|
||
tenant_id=1,
|
||
project_id=None,
|
||
asset_type="golden",
|
||
status="approved",
|
||
prompt_key="mounted.runtime.reply",
|
||
workflow_type="mounted.plan",
|
||
asked_via="feishu",
|
||
question_text="这个需求先别立项，先帮我收敛。",
|
||
answer_text="我先帮你收敛目标和范围，再决定是否出草稿。",
|
||
source_snapshot={"run_id": 2},
|
||
tags=["mounted", "golden"],
|
||
)
|
||
eval_dataset = TrainingDataset(
|
||
tenant_id=1,
|
||
dataset_type="eval",
|
||
version="teacher-frozen-v1",
|
||
status="ready",
|
||
sample_count=18,
|
||
selection_rules={"suite_kind": "teacher_eval_frozen_set"},
|
||
)
|
||
db_session.add_all([trace, activation, policy, failure_asset, golden_asset, eval_dataset])
|
||
await db_session.flush()
|
||
|
||
db_session.add(
|
||
QualityAssetPair(
|
||
tenant_id=1,
|
||
pair_type="preference",
|
||
status="approved",
|
||
chosen_asset_id=golden_asset.id,
|
||
rejected_asset_id=failure_asset.id,
|
||
reason_text="golden answer 更符合 mounted runtime 的自然回复目标",
|
||
)
|
||
)
|
||
|
||
eval_job = TrainingJob(
|
||
tenant_id=1,
|
||
dataset_id=eval_dataset.id,
|
||
job_type="sft",
|
||
base_model_family="gpt-5.4",
|
||
target_slot="high",
|
||
status="queued",
|
||
provider_backend="openai",
|
||
)
|
||
db_session.add(eval_job)
|
||
await db_session.flush()
|
||
artifact = TrainedModelArtifact(
|
||
tenant_id=1,
|
||
training_job_id=eval_job.id,
|
||
artifact_ref="artifact:eval:1",
|
||
base_model_family="gpt-5.4",
|
||
target_slot="high",
|
||
status="pending_eval",
|
||
metrics_json={"offline_judge": 91.5},
|
||
)
|
||
db_session.add(artifact)
|
||
await db_session.flush()
|
||
db_session.add(
|
||
TrainedModelDeployment(
|
||
tenant_id=1,
|
||
artifact_id=artifact.id,
|
||
deployment_status="candidate",
|
||
bound_tier="high",
|
||
effective_scope={"sources": ["feishu"]},
|
||
decision="review_pending",
|
||
)
|
||
)
|
||
|
||
db_session.add(
|
||
TrainingEvaluation(
|
||
tenant_id=1,
|
||
training_job_id=eval_job.id,
|
||
evaluation_type="offline_judge",
|
||
status="completed",
|
||
score=91.5,
|
||
evaluation_summary={"pass": True},
|
||
)
|
||
)
|
||
db_session.add(
|
||
JudgeRun(
|
||
tenant_id=1,
|
||
asset_id=failure_asset.id,
|
||
prompt_key="mounted.runtime.reply",
|
||
workflow_type="mounted.plan",
|
||
asked_via="feishu",
|
||
judge_status="completed",
|
||
bucket_key="mounted_runtime.topic_binding",
|
||
rule_score=78.0,
|
||
llm_score=82.0,
|
||
overall_score=80.0,
|
||
findings=[{"code": "topic_mismatch"}],
|
||
metadata_json={"run_id": 1},
|
||
)
|
||
)
|
||
await db_session.commit()
|
||
|
||
app = _build_app(db_session)
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
sources_resp = await client.get("/ai-steward/mounted-ai/sources")
|
||
assert sources_resp.status_code == 200
|
||
assert sources_resp.json()["data"]["items"][0]["source_system"] == "feishu"
|
||
assert sources_resp.json()["data"]["items"][0]["topic_count"] == 1
|
||
|
||
topics_resp = await client.get("/ai-steward/mounted-ai/topics")
|
||
assert topics_resp.status_code == 200
|
||
assert topics_resp.json()["data"]["items"][0]["topic_key"] == "feishu:chat_1:topic_1"
|
||
assert topics_resp.json()["data"]["items"][0]["draft_state"] == "collecting"
|
||
|
||
runs_resp = await client.get("/ai-steward/mounted-ai/runs")
|
||
assert runs_resp.status_code == 200
|
||
assert runs_resp.json()["data"]["items"][0]["active_agent"] == "Clarifier"
|
||
|
||
detail_resp = await client.get(f"/ai-steward/mounted-ai/runs/{run.id}")
|
||
assert detail_resp.status_code == 200
|
||
assert detail_resp.json()["data"]["agent_traces"][0]["agent_name"] == "Clarifier"
|
||
assert detail_resp.json()["data"]["agent_traces"][0]["input_summary"]["mode"] == "plan"
|
||
assert detail_resp.json()["data"]["agent_traces"][0]["output_summary"]["question"] == "这件事打算谁来负责？"
|
||
assert detail_resp.json()["data"]["agent_traces"][0]["tool_trace"][0]["tool_name"] == "knowledge_graph.search"
|
||
|
||
cutover_resp = await client.get("/ai-steward/mounted-ai/cutover-status")
|
||
assert cutover_resp.status_code == 200
|
||
assert cutover_resp.json()["data"]["mounted_runtime_ready"] is True
|
||
assert cutover_resp.json()["data"]["tool_policy_count"] == 1
|
||
assert cutover_resp.json()["data"]["legacy_cutover_state"] == "retired"
|
||
assert cutover_resp.json()["data"]["retired_item_count"] >= 4
|
||
assert cutover_resp.json()["data"]["active_item_count"] == 0
|
||
assert {
|
||
item["key"]: item["status"]
|
||
for item in cutover_resp.json()["data"]["legacy_cutover_items"]
|
||
} == {
|
||
"ai_pmo_ingress_consultant_capture": "retired",
|
||
"feishu_mention_consultant_reply": "retired",
|
||
"signal_capture_legacy_entry": "retired",
|
||
"legacy_feishu_queue_followup": "retired",
|
||
}
|
||
|
||
eval_suites_resp = await client.get("/ai-steward/mounted-ai/eval-suites")
|
||
assert eval_suites_resp.status_code == 200
|
||
assert eval_suites_resp.json()["data"]["items"][0]["suite_kind"] == "teacher_eval_frozen_set"
|
||
assert eval_suites_resp.json()["data"]["items"][0]["latest_score"] == 91.5
|
||
|
||
eval_suite_detail_resp = await client.get(f"/ai-steward/mounted-ai/eval-suites/{eval_dataset.id}")
|
||
assert eval_suite_detail_resp.status_code == 200
|
||
assert eval_suite_detail_resp.json()["data"]["item"]["sample_count"] == 18
|
||
assert eval_suite_detail_resp.json()["data"]["evaluations"][0]["evaluation_type"] == "offline_judge"
|
||
|
||
activations_resp = await client.get("/ai-steward/mounted-ai/source-activations")
|
||
assert activations_resp.status_code == 200
|
||
assert activations_resp.json()["data"]["items"][0]["cutover_mode"] == "mounted_only"
|
||
assert activations_resp.json()["data"]["items"][0]["auto_reply_enabled"] is True
|
||
|
||
tool_policies_resp = await client.get("/ai-steward/mounted-ai/tool-policies")
|
||
assert tool_policies_resp.status_code == 200
|
||
assert tool_policies_resp.json()["data"]["items"][0]["tool_name"] == "project.create"
|
||
assert tool_policies_resp.json()["data"]["items"][0]["confirm_required"] is True
|
||
assert tool_policies_resp.json()["data"]["items"][0]["auto_execute_allowed"] is False
|
||
|
||
activation_upsert_resp = await client.post(
|
||
"/ai-steward/mounted-ai/admin/source-activation",
|
||
json={
|
||
"source_system": "feishu",
|
||
"scope_type": "chat",
|
||
"scope_ref": "oc_runtime_lab",
|
||
"status": "active",
|
||
"cutover_mode": "shadow",
|
||
"auto_reply_enabled": False,
|
||
"notes": "切回 shadow 做回放验证",
|
||
},
|
||
)
|
||
assert activation_upsert_resp.status_code == 200
|
||
assert activation_upsert_resp.json()["data"]["item"]["cutover_mode"] == "shadow"
|
||
assert activation_upsert_resp.json()["data"]["item"]["auto_reply_enabled"] is False
|
||
|
||
tool_policy_upsert_resp = await client.post(
|
||
"/ai-steward/mounted-ai/admin/tool-policy",
|
||
json={
|
||
"tool_name": "project.create",
|
||
"risk_level": "critical",
|
||
"confirm_required": True,
|
||
"auto_execute_allowed": False,
|
||
"is_active": True,
|
||
"source_scope": {"sources": ["feishu"]},
|
||
"tenant_scope": {"roles": ["manager"]},
|
||
"policy_metadata": {"reviewers": ["pm_owner"]},
|
||
"notes": "高风险工具必须保持人工确认",
|
||
},
|
||
)
|
||
assert tool_policy_upsert_resp.status_code == 200
|
||
assert tool_policy_upsert_resp.json()["data"]["item"]["risk_level"] == "critical"
|
||
assert tool_policy_upsert_resp.json()["data"]["item"]["tenant_scope"]["roles"] == ["manager"]
|
||
|
||
training_ops_resp = await client.get("/ai-steward/mounted-ai/training-ops")
|
||
assert training_ops_resp.status_code == 200
|
||
assert training_ops_resp.json()["data"]["dataset_count"] == 1
|
||
assert training_ops_resp.json()["data"]["candidate_deployment_count"] == 1
|
||
assert training_ops_resp.json()["data"]["approved_golden_asset_count"] == 1
|
||
assert training_ops_resp.json()["data"]["regression_buckets"][0]["bucket_key"] == "mounted_runtime.topic_binding"
|
||
|
||
hard_cases_resp = await client.get("/ai-steward/mounted-ai/hard-cases")
|
||
assert hard_cases_resp.status_code == 200
|
||
assert hard_cases_resp.json()["data"]["items"][0]["bucket_key"] == "mounted_runtime.topic_binding"
|
||
assert hard_cases_resp.json()["data"]["items"][0]["question_text"] == "帮我判断下这个需求应该谁来接。"
|
||
|
||
replay_resp = await client.post(
|
||
"/ai-steward/mounted-ai/admin/replay-run",
|
||
json={
|
||
"run_id": run.id,
|
||
"bucket_key": "mounted_runtime.replay.manual_review",
|
||
"asset_type": "failure",
|
||
"corrected_answer": "建议先澄清负责人，再决定是否出草稿。",
|
||
"notes": "人工回放后标记为 hard case",
|
||
},
|
||
)
|
||
assert replay_resp.status_code == 200
|
||
assert replay_resp.json()["data"]["run_id"] == run.id
|
||
assert replay_resp.json()["data"]["asset"]["asset_type"] == "failure"
|
||
assert replay_resp.json()["data"]["judge_run"]["bucket_key"] == "mounted_runtime.replay.manual_review"
|