from __future__ import annotations import os import pytest_asyncio from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine import pytest os.environ["DATABASE_URL"] = "postgresql+asyncpg://pmflow:pmflow_pass@127.0.0.1:5435/pmflow" from sqlalchemy import JSON from app.models.base import Base from app.models.ai_call_log import AICallLog from app.models.reply_iteration import PromptGoldenCase, PromptRelease, PromptReleaseCheck, PromptReplayCaseResult, PromptReplayRun from app.services.prompt_assets_service import PromptAssetService from app.services.prompt_release.service import PromptReleaseService def _use_sqlite_json_types(*tables) -> None: for table in tables: for column in table.columns: if column.type.__class__.__name__ in {"JSONB", "ARRAY", "Vector"}: column.type = JSON() @pytest_asyncio.fixture async def db_session(): engine = create_async_engine("sqlite+aiosqlite:///:memory:") _use_sqlite_json_types( PromptRelease.__table__, PromptReleaseCheck.__table__, PromptGoldenCase.__table__, PromptReplayRun.__table__, PromptReplayCaseResult.__table__, AICallLog.__table__, ) async with engine.begin() as conn: await conn.run_sync( lambda sync_conn: Base.metadata.create_all( sync_conn, tables=[PromptRelease.__table__, PromptReleaseCheck.__table__], ) ) await conn.run_sync( lambda sync_conn: Base.metadata.create_all( sync_conn, tables=[PromptGoldenCase.__table__, PromptReplayRun.__table__, PromptReplayCaseResult.__table__, AICallLog.__table__], ) ) maker = async_sessionmaker(engine, expire_on_commit=False) async with maker() as session: yield session await engine.dispose() @pytest.mark.asyncio async def test_prompt_asset_service_lists_registry_with_release_state(db_session) -> None: release_service = PromptReleaseService(db_session) release = await release_service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="定制问答：{question}", created_by=1001, ) await release_service.activate_release(tenant_id=1, release_id=release["id"], actor_id=1001) payload = await PromptAssetService(db_session).list_assets(tenant_id=1, keyword="PROMPT_FEISHU_DIRECT_QA") assert payload["total"] == 1 item = payload["items"][0] assert item["prompt_key"] == "PROMPT_FEISHU_DIRECT_QA" assert item["default_template_available"] is True assert item["active_release"]["version"] == release["version"] assert "question" in item["template_variables"] @pytest.mark.asyncio async def test_prompt_asset_service_marks_inline_prompts_as_migration_needed(db_session) -> None: item = await PromptAssetService(db_session).get_asset( tenant_id=1, prompt_key="PROMPT_SIGNAL_WEBPAGE_VALUE_ANALYSIS", ) assert item["editable_policy"] == "needs_migration" assert item["runtime_source"] == "inline_discovered" assert item["default_template_available"] is False assert item["migration_status"] == "inline_prompt_needs_runtime_migration" @pytest.mark.asyncio async def test_prompt_asset_service_creates_activates_and_rolls_back_managed_release(db_session) -> None: service = PromptAssetService(db_session) first = await service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="第一版：{question}", created_by=1001, ) second = await service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="第二版：{question}", created_by=1001, ) await service.activate_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", release_id=first["id"], actor_id=1001, ) active_second = await service.activate_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", release_id=second["id"], actor_id=1001, ) rolled_back = await service.rollback_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", release_id=first["id"], actor_id=1001, ) assert first["version"] == 1 assert second["version"] == 2 assert active_second["status"] == "active" assert rolled_back["id"] == first["id"] detail = await service.get_asset(tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA") assert detail["active_release"]["id"] == first["id"] assert detail["active_release"]["body_snapshot"] == "第一版：{question}" @pytest.mark.asyncio async def test_prompt_asset_service_runs_guarded_release_gate_and_blocks_activation(db_session) -> None: service = PromptAssetService(db_session) release = await service.create_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}\n请输出 JSON。", created_by=1001, ) with pytest.raises(ValueError, match="blocked by gate"): await service.activate_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) checks = await service.run_release_checks( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) assert checks["gate"]["status"] == "blocked" assert any(item["check_type"] == "variable_contract" and item["status"] == "passed" for item in checks["items"]) assert any(item["check_type"] == "golden_replay" and item["status"] == "skipped" for item in checks["items"]) with pytest.raises(ValueError, match="golden"): await service.activate_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) @pytest.mark.asyncio async def test_prompt_asset_service_replay_unblocks_guarded_gate_and_quality_summary(db_session) -> None: service = PromptAssetService(db_session) release = await service.create_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}\n请输出 JSON。", created_by=1001, ) golden_case = await service.create_golden_case( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", case_name="基础语义路由样本", input_payload={ "message": "查一下项目风险", "workflow_bundle": "项目上下文", "planner_message_intent": "query", }, expected_contract={"required_input_keys": ["message"], "required_prompt_contains": ["查一下项目风险"]}, created_by=1001, ) db_session.add( AICallLog( tenant_id=1, function_name="rag_query_semantics", prompt="语义判断", response="[error] schema invalid", status="structured_invalid", prompt_key="PROMPT_RAG_QUERY_SEMANTICS", prompt_release_id=release["id"], prompt_source="active_release", routing_meta={ "prompt_key": "PROMPT_RAG_QUERY_SEMANTICS", "prompt_release_id": release["id"], "prompt_source": "active_release", }, latency_ms=120, cost=0.002, ) ) await db_session.commit() checks_before_replay = await service.run_release_checks( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) replay = await service.create_replay_run( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], created_by=1001, ) checks = await service.run_release_checks( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) quality = await service.get_quality_summary( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", window="7d", ) cases = await service.list_golden_cases(tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS") assert golden_case["case_name"] == "基础语义路由样本" assert checks_before_replay["gate"]["status"] == "blocked" assert replay["run_status"] == "completed" assert replay["failed_count"] == 0 assert replay["results"][0]["status"] == "passed" assert checks["gate"]["status"] == "passed" assert any(item["check_type"] == "golden_replay" and item["status"] == "passed" for item in checks["items"]) assert quality["total_calls"] == 1 assert quality["failed_calls"] == 1 assert quality["release_breakdown"][0]["prompt_release_id"] == release["id"] assert cases["active_count"] == 1 @pytest.mark.asyncio async def test_prompt_asset_service_blocks_guarded_activation_when_replay_failed(db_session) -> None: service = PromptAssetService(db_session) release = await service.create_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}", created_by=1001, ) await service.create_golden_case( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", case_name="缺失输入样本", input_payload={"message": "查一下项目风险"}, expected_contract={"required_input_keys": ["workflow_bundle"]}, created_by=1001, ) replay = await service.create_replay_run( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], created_by=1001, ) checks = await service.run_release_checks( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) assert replay["failed_count"] == 1 assert replay["results"][0]["status"] == "failed" assert checks["gate"]["status"] == "blocked" assert any(item["check_type"] == "golden_replay" and item["status"] == "failed" for item in checks["items"]) with pytest.raises(ValueError, match="blocked by gate"): await service.activate_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], actor_id=1001, ) @pytest.mark.asyncio async def test_prompt_asset_service_llm_replay_records_judge_and_call_links(db_session, monkeypatch) -> None: async def fake_llm_replay(self, *, prompt: str, function_name: str, **kwargs): return f"{function_name} 输出：{prompt}", 9001 if function_name.endswith("candidate") else 9000 async def fake_judge(self, **kwargs): return { "score": 0.91, "label": "better", "decision": "recommend_release", "reason": "候选输出更清晰且未违反边界。", "risk_flags": [], } monkeypatch.setattr(PromptAssetService, "_run_llm_replay_prompt", fake_llm_replay) monkeypatch.setattr(PromptAssetService, "_judge_replay_case", fake_judge) service = PromptAssetService(db_session) release = await service.create_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}\n请输出 JSON。", created_by=1001, ) await service.create_golden_case( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", case_name="LLM 回放样本", input_payload={ "message": "查一下项目风险", "workflow_bundle": "项目上下文", "planner_message_intent": "query", }, expected_contract={"required_input_keys": ["message"]}, priority="blocking", created_by=1001, ) replay = await service.create_replay_run( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], created_by=1001, include_llm=True, max_cases=5, ) assert replay["run_mode"] == "llm_compare" assert replay["failed_count"] == 0 assert replay["results"][0]["status"] == "passed" assert replay["results"][0]["candidate_ai_call_log_id"] == 9001 assert replay["results"][0]["judge_score"] == 0.91 assert replay["results"][0]["judge_label"] == "better" @pytest.mark.asyncio async def test_prompt_asset_service_non_blocking_case_failure_is_warning(db_session) -> None: service = PromptAssetService(db_session) release = await service.create_release( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}", created_by=1001, ) await service.create_golden_case( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", case_name="观察型失败样本", input_payload={"message": "查一下项目风险"}, expected_contract={"required_input_keys": ["workflow_bundle"]}, priority="important", created_by=1001, ) replay = await service.create_replay_run( tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS", release_id=release["id"], created_by=1001, ) assert replay["failed_count"] == 0 assert replay["warning_count"] == 1 assert replay["results"][0]["status"] == "warning" @pytest.mark.asyncio async def test_prompt_asset_service_release_watch_summary(db_session) -> None: service = PromptAssetService(db_session) release = await service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="运营定制：{question}", created_by=1001, ) db_session.add( AICallLog( tenant_id=1, function_name="adaptive_rag_answer_normal", prompt="运营定制：这周项目风险是什么？", response="[error] provider timeout", status="failed", prompt_key="PROMPT_FEISHU_DIRECT_QA", prompt_release_id=release["id"], prompt_source="active_release", latency_ms=300, cost=0.004, ) ) await db_session.commit() watch = await service.get_release_watch_summary( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", release_id=release["id"], ) assert watch["recommendation"] in {"needs_review", "rollback_suggested"} assert watch["windows"]["24h"]["total_calls"] == 1 @pytest.mark.asyncio async def test_prompt_asset_service_rejects_migration_needed_or_invalid_release(db_session) -> None: service = PromptAssetService(db_session) with pytest.raises(ValueError, match="migrated"): await service.create_release( tenant_id=1, prompt_key="PROMPT_SIGNAL_WEBPAGE_VALUE_ANALYSIS", body_snapshot="不能直接发布", created_by=1001, ) with pytest.raises(ValueError, match="unknown variables"): await service.create_release( tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA", body_snapshot="未知变量：{unknown_signal}", created_by=1001, )