389 lines
17 KiB
Python
389 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from fastapi import FastAPI
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
|
|
|
os.environ["DATABASE_URL"] = "postgresql+asyncpg://pmflow:pmflow_pass@127.0.0.1:5435/pmflow"
|
|
|
|
from app.api import prompt_assets_api
|
|
from app.core.database import get_db
|
|
from app.dependencies.permissions import require_platform_ai_ops
|
|
from app.dependencies.tenant import get_tenant_id
|
|
from app.models.base import Base
|
|
from app.models.ai_call_log import AICallLog
|
|
from app.models.reply_iteration import PromptRelease, PromptReleaseCheck
|
|
from app.services.prompt_release.service import PromptReleaseService
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def db_session():
|
|
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(
|
|
lambda sync_conn: Base.metadata.create_all(
|
|
sync_conn,
|
|
tables=[PromptRelease.__table__],
|
|
)
|
|
)
|
|
await conn.run_sync(
|
|
lambda sync_conn: Base.metadata.create_all(
|
|
sync_conn,
|
|
tables=[AICallLog.__table__],
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE TABLE prompt_release_checks (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
tenant_id BIGINT NOT NULL,
|
|
prompt_key VARCHAR(128) NOT NULL,
|
|
release_id INTEGER NOT NULL,
|
|
check_type VARCHAR(48) NOT NULL,
|
|
status VARCHAR(24) NOT NULL,
|
|
severity VARCHAR(16) NOT NULL DEFAULT 'info',
|
|
summary TEXT NOT NULL,
|
|
details_json JSON,
|
|
created_by BIGINT,
|
|
completed_at VARCHAR(64),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE TABLE prompt_golden_cases (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
tenant_id BIGINT NOT NULL,
|
|
prompt_key VARCHAR(128) NOT NULL,
|
|
case_name VARCHAR(255) NOT NULL,
|
|
status VARCHAR(24) NOT NULL DEFAULT 'active',
|
|
input_payload JSON,
|
|
expected_contract JSON,
|
|
expected_contains TEXT,
|
|
source VARCHAR(48) NOT NULL DEFAULT 'manual',
|
|
source_ai_call_log_id INTEGER,
|
|
case_type VARCHAR(32) NOT NULL DEFAULT 'regression',
|
|
priority VARCHAR(24) NOT NULL DEFAULT 'blocking',
|
|
scenario_key VARCHAR(128),
|
|
tags JSON,
|
|
last_run_status VARCHAR(24),
|
|
last_failed_at VARCHAR(64),
|
|
created_by BIGINT,
|
|
notes TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE TABLE prompt_replay_runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
tenant_id BIGINT NOT NULL,
|
|
prompt_key VARCHAR(128) NOT NULL,
|
|
release_id INTEGER NOT NULL,
|
|
baseline_release_id INTEGER,
|
|
run_status VARCHAR(24) NOT NULL DEFAULT 'pending',
|
|
run_mode VARCHAR(32) NOT NULL DEFAULT 'contract_smoke',
|
|
case_count INTEGER NOT NULL DEFAULT 0,
|
|
passed_count INTEGER NOT NULL DEFAULT 0,
|
|
failed_count INTEGER NOT NULL DEFAULT 0,
|
|
warning_count INTEGER NOT NULL DEFAULT 0,
|
|
score_delta FLOAT,
|
|
summary TEXT,
|
|
risk_flags_json JSON,
|
|
created_by BIGINT,
|
|
started_at VARCHAR(64),
|
|
completed_at VARCHAR(64),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
await conn.execute(
|
|
text(
|
|
"""
|
|
CREATE TABLE prompt_replay_case_results (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
tenant_id BIGINT NOT NULL,
|
|
run_id INTEGER NOT NULL,
|
|
golden_case_id INTEGER NOT NULL,
|
|
prompt_key VARCHAR(128) NOT NULL,
|
|
release_id INTEGER NOT NULL,
|
|
baseline_output TEXT,
|
|
candidate_output TEXT,
|
|
status VARCHAR(24) NOT NULL,
|
|
contract_result_json JSON,
|
|
diff_summary TEXT,
|
|
risk_flags_json JSON,
|
|
baseline_ai_call_log_id INTEGER,
|
|
candidate_ai_call_log_id INTEGER,
|
|
judge_score FLOAT,
|
|
judge_label VARCHAR(32),
|
|
judge_reason TEXT,
|
|
latency_ms INTEGER,
|
|
estimated_cost FLOAT,
|
|
error_summary TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
maker = async_sessionmaker(engine, expire_on_commit=False)
|
|
async with maker() as session:
|
|
yield session
|
|
await engine.dispose()
|
|
|
|
|
|
def _build_app(db_session) -> FastAPI:
|
|
app = FastAPI()
|
|
app.include_router(prompt_assets_api.router)
|
|
|
|
async def override_get_db():
|
|
yield db_session
|
|
|
|
app.dependency_overrides[get_db] = override_get_db
|
|
app.dependency_overrides[get_tenant_id] = lambda: 1
|
|
app.dependency_overrides[require_platform_ai_ops] = lambda: SimpleNamespace(id=1001, tenant_id=1, role="AI_OPS")
|
|
return app
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_assets_api_lists_and_reads_detail(db_session) -> None:
|
|
release_service = PromptReleaseService(db_session)
|
|
release = await release_service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="运营定制：{question}",
|
|
created_by=1001,
|
|
)
|
|
await release_service.activate_release(tenant_id=1, release_id=release["id"], actor_id=1001)
|
|
|
|
app = _build_app(db_session)
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
list_response = await client.get("/prompt-assets?keyword=PROMPT_FEISHU_DIRECT_QA")
|
|
detail_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA")
|
|
|
|
assert list_response.status_code == 200
|
|
payload = list_response.json()["data"]
|
|
assert payload["total"] == 1
|
|
assert payload["items"][0]["active_release"]["version"] == release["version"]
|
|
|
|
assert detail_response.status_code == 200
|
|
detail = detail_response.json()["data"]
|
|
assert detail["prompt_key"] == "PROMPT_FEISHU_DIRECT_QA"
|
|
assert detail["default_template_available"] is True
|
|
assert "default_template_body" in detail
|
|
assert detail["releases"][0]["body_snapshot"] == "运营定制：{question}"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_assets_api_manages_release_lifecycle(db_session) -> None:
|
|
app = _build_app(db_session)
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
first_response = await client.post(
|
|
"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases",
|
|
json={"body_snapshot": "第一版：{question}", "notes": "initial"},
|
|
)
|
|
second_response = await client.post(
|
|
"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases",
|
|
json={"body_snapshot": "第二版：{question}", "notes": "improve tone"},
|
|
)
|
|
|
|
assert first_response.status_code == 200
|
|
assert second_response.status_code == 200
|
|
first = first_response.json()["data"]
|
|
second = second_response.json()["data"]
|
|
assert first["version"] == 1
|
|
assert second["version"] == 2
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
activate_first = await client.post(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{first['id']}/activate")
|
|
activate_second = await client.post(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{second['id']}/activate")
|
|
rollback_first = await client.post(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{first['id']}/rollback")
|
|
detail_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA")
|
|
|
|
assert activate_first.status_code == 200
|
|
assert activate_second.status_code == 200
|
|
assert activate_second.json()["data"]["status"] == "active"
|
|
assert rollback_first.status_code == 200
|
|
assert rollback_first.json()["data"]["id"] == first["id"]
|
|
detail = detail_response.json()["data"]
|
|
assert detail["active_release"]["id"] == first["id"]
|
|
statuses = {item["id"]: item["status"] for item in detail["releases"]}
|
|
assert statuses[first["id"]] == "active"
|
|
assert statuses[second["id"]] == "archived"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_assets_api_runs_guarded_release_gate_and_blocks_activation(db_session) -> None:
|
|
app = _build_app(db_session)
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
create_response = await client.post(
|
|
"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases",
|
|
json={"body_snapshot": "门禁候选：{message}\\n{workflow_bundle}\\n{planner_message_intent}\\n请输出 JSON。", "notes": "guarded candidate"},
|
|
)
|
|
|
|
assert create_response.status_code == 200
|
|
release = create_response.json()["data"]
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
activate_before_check = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/activate")
|
|
checks_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/checks/run")
|
|
list_checks_response = await client.get(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/checks")
|
|
activate_after_check = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/activate")
|
|
|
|
assert activate_before_check.status_code == 400
|
|
assert "blocked by gate" in activate_before_check.json()["message"]
|
|
assert checks_response.status_code == 200
|
|
checks = checks_response.json()["data"]
|
|
assert checks["gate"]["status"] == "blocked"
|
|
assert {item["check_type"] for item in checks["items"]} == {"variable_contract", "schema_smoke", "golden_replay"}
|
|
assert any(item["check_type"] == "golden_replay" and item["status"] == "skipped" for item in checks["items"])
|
|
assert list_checks_response.status_code == 200
|
|
assert list_checks_response.json()["data"]["gate"]["status"] == "blocked"
|
|
assert activate_after_check.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_assets_api_creates_golden_case_from_call_and_quality_summary(db_session) -> None:
|
|
release_service = PromptReleaseService(db_session)
|
|
release = await release_service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="运营定制：{question}",
|
|
created_by=1001,
|
|
)
|
|
db_session.add(
|
|
AICallLog(
|
|
tenant_id=1,
|
|
function_name="adaptive_rag_answer_normal",
|
|
prompt="运营定制：这周项目风险是什么？",
|
|
response="[error] provider timeout",
|
|
status="failed",
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
prompt_release_id=release["id"],
|
|
prompt_source="active_release",
|
|
latency_ms=300,
|
|
cost=0.004,
|
|
)
|
|
)
|
|
await db_session.commit()
|
|
app = _build_app(db_session)
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
quality_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA/quality-summary?window=7d")
|
|
case_response = await client.post(
|
|
"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/golden-cases/from-call/1",
|
|
json={"case_name": "失败调用回归样本"},
|
|
)
|
|
list_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA/golden-cases")
|
|
watch_response = await client.get(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{release['id']}/watch-summary")
|
|
|
|
assert quality_response.status_code == 200
|
|
quality = quality_response.json()["data"]
|
|
assert quality["total_calls"] == 1
|
|
assert quality["failed_calls"] == 1
|
|
assert quality["recent_failures"][0]["id"] == 1
|
|
assert case_response.status_code == 200
|
|
assert case_response.json()["data"]["source"] == "ai_call_log"
|
|
assert case_response.json()["data"]["priority"] == "important"
|
|
assert list_response.status_code == 200
|
|
assert list_response.json()["data"]["active_count"] == 1
|
|
assert watch_response.status_code == 200
|
|
assert watch_response.json()["data"]["windows"]["24h"]["total_calls"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_assets_api_runs_replay_and_guarded_gate_allows_activation(db_session) -> None:
|
|
app = _build_app(db_session)
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
create_response = await client.post(
|
|
"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases",
|
|
json={"body_snapshot": "门禁候选：{message}\\n{workflow_bundle}\\n{planner_message_intent}\\n请输出 JSON。", "notes": "guarded candidate"},
|
|
)
|
|
|
|
assert create_response.status_code == 200
|
|
release = create_response.json()["data"]
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
case_response = await client.post(
|
|
"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/golden-cases",
|
|
json={
|
|
"case_name": "基础语义路由样本",
|
|
"input_payload": {
|
|
"message": "查一下项目风险",
|
|
"workflow_bundle": "项目上下文",
|
|
"planner_message_intent": "query",
|
|
},
|
|
"expected_contract": {
|
|
"required_input_keys": ["message"],
|
|
"required_prompt_contains": ["查一下项目风险"],
|
|
},
|
|
"priority": "blocking",
|
|
"case_type": "regression",
|
|
"tags": ["smoke"],
|
|
},
|
|
)
|
|
replay_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/replay-runs", json={"max_cases": 5})
|
|
runs_response = await client.get(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/replay-runs")
|
|
run_id = replay_response.json()["data"]["id"]
|
|
run_detail_response = await client.get(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/replay-runs/{run_id}")
|
|
checks_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/checks/run")
|
|
activate_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/activate")
|
|
|
|
assert case_response.status_code == 200
|
|
assert case_response.json()["data"]["priority"] == "blocking"
|
|
assert replay_response.status_code == 200
|
|
replay = replay_response.json()["data"]
|
|
assert replay["run_status"] == "completed"
|
|
assert replay["failed_count"] == 0
|
|
assert replay["results"][0]["status"] == "passed"
|
|
assert runs_response.status_code == 200
|
|
assert runs_response.json()["data"]["total"] == 1
|
|
assert run_detail_response.status_code == 200
|
|
assert run_detail_response.json()["data"]["id"] == run_id
|
|
assert checks_response.status_code == 200
|
|
assert checks_response.json()["data"]["gate"]["status"] == "passed"
|
|
assert activate_response.status_code == 200
|
|
assert activate_response.json()["data"]["status"] == "active"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_assets_api_rejects_migration_needed_and_unknown_variables(db_session) -> None:
|
|
app = _build_app(db_session)
|
|
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
|
migration_response = await client.post(
|
|
"/prompt-assets/PROMPT_SIGNAL_WEBPAGE_VALUE_ANALYSIS/releases",
|
|
json={"body_snapshot": "不能直接运营"},
|
|
)
|
|
unknown_var_response = await client.post(
|
|
"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases",
|
|
json={"body_snapshot": "未知变量：{unknown_signal}"},
|
|
)
|
|
|
|
assert migration_response.status_code == 400
|
|
assert "must be migrated" in migration_response.json()["message"]
|
|
assert unknown_var_response.status_code == 400
|
|
assert "unknown variables" in unknown_var_response.json()["message"]
|