from __future__ import annotations import os from types import SimpleNamespace import pytest import pytest_asyncio from fastapi import FastAPI from httpx import ASGITransport, AsyncClient from sqlalchemy import text from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine os.environ["DATABASE_URL"] = "postgresql+asyncpg://pmflow:pmflow_pass@127.0.0.1:5435/pmflow" from app.api import prompt_assets_api from app.core.database import get_db from app.dependencies.permissions import require_platform_ai_ops from app.dependencies.tenant import get_tenant_id from app.models.base import Base from app.models.ai_call_log import AICallLog from app.models.reply_iteration import PromptRelease, PromptReleaseCheck from app.services.prompt_release.service import PromptReleaseService @pytest_asyncio.fixture async def db_session(): engine = create_async_engine("sqlite+aiosqlite:///:memory:") async with engine.begin() as conn: await conn.run_sync( lambda sync_conn: Base.metadata.create_all( sync_conn, tables=[PromptRelease.__table__], ) ) await conn.run_sync( lambda sync_conn: Base.metadata.create_all( sync_conn, tables=[AICallLog.__table__], ) ) await conn.execute( text( """ CREATE TABLE prompt_release_checks ( id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id BIGINT NOT NULL, prompt_key VARCHAR(128) NOT NULL, release_id INTEGER NOT NULL, check_type VARCHAR(48) NOT NULL, status VARCHAR(24) NOT NULL, severity VARCHAR(16) NOT NULL DEFAULT 'info', summary TEXT NOT NULL, details_json JSON, created_by BIGINT, completed_at VARCHAR(64), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL ) """ ) ) await conn.execute( text( """ CREATE TABLE prompt_golden_cases ( id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id BIGINT NOT NULL, prompt_key VARCHAR(128) NOT NULL, case_name VARCHAR(255) NOT NULL, status VARCHAR(24) NOT NULL DEFAULT 'active', input_payload JSON, expected_contract JSON, expected_contains TEXT, source VARCHAR(48) NOT NULL DEFAULT 'manual', source_ai_call_log_id INTEGER, case_type VARCHAR(32) NOT NULL DEFAULT 'regression', priority VARCHAR(24) NOT NULL DEFAULT 'blocking', scenario_key VARCHAR(128), tags JSON, last_run_status VARCHAR(24), last_failed_at VARCHAR(64), created_by BIGINT, notes TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL ) """ ) ) await conn.execute( text( """ CREATE TABLE prompt_replay_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id BIGINT NOT NULL, prompt_key VARCHAR(128) NOT NULL, release_id INTEGER NOT NULL, baseline_release_id INTEGER, run_status VARCHAR(24) NOT NULL DEFAULT 'pending', run_mode VARCHAR(32) NOT NULL DEFAULT 'contract_smoke', case_count INTEGER NOT NULL DEFAULT 0, passed_count INTEGER NOT NULL DEFAULT 0, failed_count INTEGER NOT NULL DEFAULT 0, warning_count INTEGER NOT NULL DEFAULT 0, score_delta FLOAT, summary TEXT, risk_flags_json JSON, created_by BIGINT, started_at VARCHAR(64), completed_at VARCHAR(64), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL ) """ ) ) await conn.execute( text( """ CREATE TABLE prompt_replay_case_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id BIGINT NOT NULL, run_id INTEGER NOT NULL, golden_case_id INTEGER NOT NULL, prompt_key VARCHAR(128) NOT NULL, release_id INTEGER NOT NULL, baseline_output TEXT, candidate_output TEXT, status VARCHAR(24) NOT NULL, contract_result_json JSON, diff_summary TEXT, risk_flags_json JSON, baseline_ai_call_log_id INTEGER, candidate_ai_call_log_id INTEGER, judge_score FLOAT, judge_label VARCHAR(32), judge_reason TEXT, latency_ms INTEGER, estimated_cost FLOAT, error_summary TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL ) """ ) ) maker = async_sessionmaker(engine, expire_on_commit=False) async with maker() as session: yield session await engine.dispose() def _build_app(db_session) -> FastAPI: app = FastAPI() app.include_router(prompt_assets_api.router) async def override_get_db(): yield db_session app.dependency_overrides[get_db] = override_get_db app.dependency_overrides[get_tenant_id] = lambda: 1 app.dependency_overrides[require_platform_ai_ops] = lambda: SimpleNamespace(id=1001, tenant_id=1, role="AI_OPS") return app @pytest.mark.asyncio async def test_prompt_assets_api_lists_and_reads_detail(db_session) -> None: release_service = PromptReleaseService(db_session) release = await release_service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="运营定制：{question}", created_by=1001, ) await release_service.activate_release(tenant_id=1, release_id=release["id"], actor_id=1001) app = _build_app(db_session) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: list_response = await client.get("/prompt-assets?keyword=PROMPT_FEISHU_DIRECT_QA") detail_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA") assert list_response.status_code == 200 payload = list_response.json()["data"] assert payload["total"] == 1 assert payload["items"][0]["active_release"]["version"] == release["version"] assert detail_response.status_code == 200 detail = detail_response.json()["data"] assert detail["prompt_key"] == "PROMPT_FEISHU_DIRECT_QA" assert detail["default_template_available"] is True assert "default_template_body" in detail assert detail["releases"][0]["body_snapshot"] == "运营定制：{question}" @pytest.mark.asyncio async def test_prompt_assets_api_manages_release_lifecycle(db_session) -> None: app = _build_app(db_session) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: first_response = await client.post( "/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases", json={"body_snapshot": "第一版：{question}", "notes": "initial"}, ) second_response = await client.post( "/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases", json={"body_snapshot": "第二版：{question}", "notes": "improve tone"}, ) assert first_response.status_code == 200 assert second_response.status_code == 200 first = first_response.json()["data"] second = second_response.json()["data"] assert first["version"] == 1 assert second["version"] == 2 async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: activate_first = await client.post(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{first['id']}/activate") activate_second = await client.post(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{second['id']}/activate") rollback_first = await client.post(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{first['id']}/rollback") detail_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA") assert activate_first.status_code == 200 assert activate_second.status_code == 200 assert activate_second.json()["data"]["status"] == "active" assert rollback_first.status_code == 200 assert rollback_first.json()["data"]["id"] == first["id"] detail = detail_response.json()["data"] assert detail["active_release"]["id"] == first["id"] statuses = {item["id"]: item["status"] for item in detail["releases"]} assert statuses[first["id"]] == "active" assert statuses[second["id"]] == "archived" @pytest.mark.asyncio async def test_prompt_assets_api_runs_guarded_release_gate_and_blocks_activation(db_session) -> None: app = _build_app(db_session) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: create_response = await client.post( "/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases", json={"body_snapshot": "门禁候选：{message}\\n{workflow_bundle}\\n{planner_message_intent}\\n请输出 JSON。", "notes": "guarded candidate"}, ) assert create_response.status_code == 200 release = create_response.json()["data"] async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: activate_before_check = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/activate") checks_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/checks/run") list_checks_response = await client.get(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/checks") activate_after_check = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/activate") assert activate_before_check.status_code == 400 assert "blocked by gate" in activate_before_check.json()["message"] assert checks_response.status_code == 200 checks = checks_response.json()["data"] assert checks["gate"]["status"] == "blocked" assert {item["check_type"] for item in checks["items"]} == {"variable_contract", "schema_smoke", "golden_replay"} assert any(item["check_type"] == "golden_replay" and item["status"] == "skipped" for item in checks["items"]) assert list_checks_response.status_code == 200 assert list_checks_response.json()["data"]["gate"]["status"] == "blocked" assert activate_after_check.status_code == 400 @pytest.mark.asyncio async def test_prompt_assets_api_creates_golden_case_from_call_and_quality_summary(db_session) -> None: release_service = PromptReleaseService(db_session) release = await release_service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="运营定制：{question}", created_by=1001, ) db_session.add( AICallLog( tenant_id=1, function_name="adaptive_rag_answer_normal", prompt="运营定制：这周项目风险是什么？", response="[error] provider timeout", status="failed", prompt_key="PROMPT_FEISHU_DIRECT_QA", prompt_release_id=release["id"], prompt_source="active_release", latency_ms=300, cost=0.004, ) ) await db_session.commit() app = _build_app(db_session) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: quality_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA/quality-summary?window=7d") case_response = await client.post( "/prompt-assets/PROMPT_FEISHU_DIRECT_QA/golden-cases/from-call/1", json={"case_name": "失败调用回归样本"}, ) list_response = await client.get("/prompt-assets/PROMPT_FEISHU_DIRECT_QA/golden-cases") watch_response = await client.get(f"/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases/{release['id']}/watch-summary") assert quality_response.status_code == 200 quality = quality_response.json()["data"] assert quality["total_calls"] == 1 assert quality["failed_calls"] == 1 assert quality["recent_failures"][0]["id"] == 1 assert case_response.status_code == 200 assert case_response.json()["data"]["source"] == "ai_call_log" assert case_response.json()["data"]["priority"] == "important" assert list_response.status_code == 200 assert list_response.json()["data"]["active_count"] == 1 assert watch_response.status_code == 200 assert watch_response.json()["data"]["windows"]["24h"]["total_calls"] == 1 @pytest.mark.asyncio async def test_prompt_assets_api_runs_replay_and_guarded_gate_allows_activation(db_session) -> None: app = _build_app(db_session) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: create_response = await client.post( "/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases", json={"body_snapshot": "门禁候选：{message}\\n{workflow_bundle}\\n{planner_message_intent}\\n请输出 JSON。", "notes": "guarded candidate"}, ) assert create_response.status_code == 200 release = create_response.json()["data"] async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: case_response = await client.post( "/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/golden-cases", json={ "case_name": "基础语义路由样本", "input_payload": { "message": "查一下项目风险", "workflow_bundle": "项目上下文", "planner_message_intent": "query", }, "expected_contract": { "required_input_keys": ["message"], "required_prompt_contains": ["查一下项目风险"], }, "priority": "blocking", "case_type": "regression", "tags": ["smoke"], }, ) replay_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/replay-runs", json={"max_cases": 5}) runs_response = await client.get(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/replay-runs") run_id = replay_response.json()["data"]["id"] run_detail_response = await client.get(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/replay-runs/{run_id}") checks_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/checks/run") activate_response = await client.post(f"/prompt-assets/PROMPT_RAG_QUERY_SEMANTICS/releases/{release['id']}/activate") assert case_response.status_code == 200 assert case_response.json()["data"]["priority"] == "blocking" assert replay_response.status_code == 200 replay = replay_response.json()["data"] assert replay["run_status"] == "completed" assert replay["failed_count"] == 0 assert replay["results"][0]["status"] == "passed" assert runs_response.status_code == 200 assert runs_response.json()["data"]["total"] == 1 assert run_detail_response.status_code == 200 assert run_detail_response.json()["data"]["id"] == run_id assert checks_response.status_code == 200 assert checks_response.json()["data"]["gate"]["status"] == "passed" assert activate_response.status_code == 200 assert activate_response.json()["data"]["status"] == "active" @pytest.mark.asyncio async def test_prompt_assets_api_rejects_migration_needed_and_unknown_variables(db_session) -> None: app = _build_app(db_session) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: migration_response = await client.post( "/prompt-assets/PROMPT_SIGNAL_WEBPAGE_VALUE_ANALYSIS/releases", json={"body_snapshot": "不能直接运营"}, ) unknown_var_response = await client.post( "/prompt-assets/PROMPT_FEISHU_DIRECT_QA/releases", json={"body_snapshot": "未知变量：{unknown_signal}"}, ) assert migration_response.status_code == 400 assert "must be migrated" in migration_response.json()["message"] assert unknown_var_response.status_code == 400 assert "unknown variables" in unknown_var_response.json()["message"]