437 lines
15 KiB
Python
437 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import pytest_asyncio
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
|
|
|
import pytest
|
|
|
|
os.environ["DATABASE_URL"] = "postgresql+asyncpg://pmflow:pmflow_pass@127.0.0.1:5435/pmflow"
|
|
|
|
from sqlalchemy import JSON
|
|
from app.models.base import Base
|
|
from app.models.ai_call_log import AICallLog
|
|
from app.models.reply_iteration import PromptGoldenCase, PromptRelease, PromptReleaseCheck, PromptReplayCaseResult, PromptReplayRun
|
|
from app.services.prompt_assets_service import PromptAssetService
|
|
from app.services.prompt_release.service import PromptReleaseService
|
|
|
|
|
|
def _use_sqlite_json_types(*tables) -> None:
|
|
for table in tables:
|
|
for column in table.columns:
|
|
if column.type.__class__.__name__ in {"JSONB", "ARRAY", "Vector"}:
|
|
column.type = JSON()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def db_session():
|
|
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
|
_use_sqlite_json_types(
|
|
PromptRelease.__table__,
|
|
PromptReleaseCheck.__table__,
|
|
PromptGoldenCase.__table__,
|
|
PromptReplayRun.__table__,
|
|
PromptReplayCaseResult.__table__,
|
|
AICallLog.__table__,
|
|
)
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(
|
|
lambda sync_conn: Base.metadata.create_all(
|
|
sync_conn,
|
|
tables=[PromptRelease.__table__, PromptReleaseCheck.__table__],
|
|
)
|
|
)
|
|
await conn.run_sync(
|
|
lambda sync_conn: Base.metadata.create_all(
|
|
sync_conn,
|
|
tables=[PromptGoldenCase.__table__, PromptReplayRun.__table__, PromptReplayCaseResult.__table__, AICallLog.__table__],
|
|
)
|
|
)
|
|
maker = async_sessionmaker(engine, expire_on_commit=False)
|
|
async with maker() as session:
|
|
yield session
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_lists_registry_with_release_state(db_session) -> None:
|
|
release_service = PromptReleaseService(db_session)
|
|
release = await release_service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="定制问答：{question}",
|
|
created_by=1001,
|
|
)
|
|
await release_service.activate_release(tenant_id=1, release_id=release["id"], actor_id=1001)
|
|
|
|
payload = await PromptAssetService(db_session).list_assets(tenant_id=1, keyword="PROMPT_FEISHU_DIRECT_QA")
|
|
|
|
assert payload["total"] == 1
|
|
item = payload["items"][0]
|
|
assert item["prompt_key"] == "PROMPT_FEISHU_DIRECT_QA"
|
|
assert item["default_template_available"] is True
|
|
assert item["active_release"]["version"] == release["version"]
|
|
assert "question" in item["template_variables"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_marks_inline_prompts_as_migration_needed(db_session) -> None:
|
|
item = await PromptAssetService(db_session).get_asset(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_SIGNAL_WEBPAGE_VALUE_ANALYSIS",
|
|
)
|
|
|
|
assert item["editable_policy"] == "needs_migration"
|
|
assert item["runtime_source"] == "inline_discovered"
|
|
assert item["default_template_available"] is False
|
|
assert item["migration_status"] == "inline_prompt_needs_runtime_migration"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_creates_activates_and_rolls_back_managed_release(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
|
|
first = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="第一版：{question}",
|
|
created_by=1001,
|
|
)
|
|
second = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="第二版：{question}",
|
|
created_by=1001,
|
|
)
|
|
await service.activate_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
release_id=first["id"],
|
|
actor_id=1001,
|
|
)
|
|
active_second = await service.activate_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
release_id=second["id"],
|
|
actor_id=1001,
|
|
)
|
|
rolled_back = await service.rollback_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
release_id=first["id"],
|
|
actor_id=1001,
|
|
)
|
|
|
|
assert first["version"] == 1
|
|
assert second["version"] == 2
|
|
assert active_second["status"] == "active"
|
|
assert rolled_back["id"] == first["id"]
|
|
detail = await service.get_asset(tenant_id=1, prompt_key="PROMPT_FEISHU_DIRECT_QA")
|
|
assert detail["active_release"]["id"] == first["id"]
|
|
assert detail["active_release"]["body_snapshot"] == "第一版：{question}"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_runs_guarded_release_gate_and_blocks_activation(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
|
|
release = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}\n请输出 JSON。",
|
|
created_by=1001,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="blocked by gate"):
|
|
await service.activate_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
|
|
checks = await service.run_release_checks(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
|
|
assert checks["gate"]["status"] == "blocked"
|
|
assert any(item["check_type"] == "variable_contract" and item["status"] == "passed" for item in checks["items"])
|
|
assert any(item["check_type"] == "golden_replay" and item["status"] == "skipped" for item in checks["items"])
|
|
|
|
with pytest.raises(ValueError, match="golden"):
|
|
await service.activate_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_replay_unblocks_guarded_gate_and_quality_summary(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
release = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}\n请输出 JSON。",
|
|
created_by=1001,
|
|
)
|
|
golden_case = await service.create_golden_case(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
case_name="基础语义路由样本",
|
|
input_payload={
|
|
"message": "查一下项目风险",
|
|
"workflow_bundle": "项目上下文",
|
|
"planner_message_intent": "query",
|
|
},
|
|
expected_contract={"required_input_keys": ["message"], "required_prompt_contains": ["查一下项目风险"]},
|
|
created_by=1001,
|
|
)
|
|
db_session.add(
|
|
AICallLog(
|
|
tenant_id=1,
|
|
function_name="rag_query_semantics",
|
|
prompt="语义判断",
|
|
response="[error] schema invalid",
|
|
status="structured_invalid",
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
prompt_release_id=release["id"],
|
|
prompt_source="active_release",
|
|
routing_meta={
|
|
"prompt_key": "PROMPT_RAG_QUERY_SEMANTICS",
|
|
"prompt_release_id": release["id"],
|
|
"prompt_source": "active_release",
|
|
},
|
|
latency_ms=120,
|
|
cost=0.002,
|
|
)
|
|
)
|
|
await db_session.commit()
|
|
|
|
checks_before_replay = await service.run_release_checks(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
replay = await service.create_replay_run(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
created_by=1001,
|
|
)
|
|
checks = await service.run_release_checks(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
quality = await service.get_quality_summary(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
window="7d",
|
|
)
|
|
cases = await service.list_golden_cases(tenant_id=1, prompt_key="PROMPT_RAG_QUERY_SEMANTICS")
|
|
|
|
assert golden_case["case_name"] == "基础语义路由样本"
|
|
assert checks_before_replay["gate"]["status"] == "blocked"
|
|
assert replay["run_status"] == "completed"
|
|
assert replay["failed_count"] == 0
|
|
assert replay["results"][0]["status"] == "passed"
|
|
assert checks["gate"]["status"] == "passed"
|
|
assert any(item["check_type"] == "golden_replay" and item["status"] == "passed" for item in checks["items"])
|
|
assert quality["total_calls"] == 1
|
|
assert quality["failed_calls"] == 1
|
|
assert quality["release_breakdown"][0]["prompt_release_id"] == release["id"]
|
|
assert cases["active_count"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_blocks_guarded_activation_when_replay_failed(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
release = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}",
|
|
created_by=1001,
|
|
)
|
|
await service.create_golden_case(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
case_name="缺失输入样本",
|
|
input_payload={"message": "查一下项目风险"},
|
|
expected_contract={"required_input_keys": ["workflow_bundle"]},
|
|
created_by=1001,
|
|
)
|
|
|
|
replay = await service.create_replay_run(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
created_by=1001,
|
|
)
|
|
checks = await service.run_release_checks(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
|
|
assert replay["failed_count"] == 1
|
|
assert replay["results"][0]["status"] == "failed"
|
|
assert checks["gate"]["status"] == "blocked"
|
|
assert any(item["check_type"] == "golden_replay" and item["status"] == "failed" for item in checks["items"])
|
|
with pytest.raises(ValueError, match="blocked by gate"):
|
|
await service.activate_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
actor_id=1001,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_llm_replay_records_judge_and_call_links(db_session, monkeypatch) -> None:
|
|
async def fake_llm_replay(self, *, prompt: str, function_name: str, **kwargs):
|
|
return f"{function_name} 输出：{prompt}", 9001 if function_name.endswith("candidate") else 9000
|
|
|
|
async def fake_judge(self, **kwargs):
|
|
return {
|
|
"score": 0.91,
|
|
"label": "better",
|
|
"decision": "recommend_release",
|
|
"reason": "候选输出更清晰且未违反边界。",
|
|
"risk_flags": [],
|
|
}
|
|
|
|
monkeypatch.setattr(PromptAssetService, "_run_llm_replay_prompt", fake_llm_replay)
|
|
monkeypatch.setattr(PromptAssetService, "_judge_replay_case", fake_judge)
|
|
service = PromptAssetService(db_session)
|
|
release = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}\n请输出 JSON。",
|
|
created_by=1001,
|
|
)
|
|
await service.create_golden_case(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
case_name="LLM 回放样本",
|
|
input_payload={
|
|
"message": "查一下项目风险",
|
|
"workflow_bundle": "项目上下文",
|
|
"planner_message_intent": "query",
|
|
},
|
|
expected_contract={"required_input_keys": ["message"]},
|
|
priority="blocking",
|
|
created_by=1001,
|
|
)
|
|
|
|
replay = await service.create_replay_run(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
created_by=1001,
|
|
include_llm=True,
|
|
max_cases=5,
|
|
)
|
|
|
|
assert replay["run_mode"] == "llm_compare"
|
|
assert replay["failed_count"] == 0
|
|
assert replay["results"][0]["status"] == "passed"
|
|
assert replay["results"][0]["candidate_ai_call_log_id"] == 9001
|
|
assert replay["results"][0]["judge_score"] == 0.91
|
|
assert replay["results"][0]["judge_label"] == "better"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_non_blocking_case_failure_is_warning(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
release = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
body_snapshot="候选版本：{message}\n{workflow_bundle}\n{planner_message_intent}",
|
|
created_by=1001,
|
|
)
|
|
await service.create_golden_case(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
case_name="观察型失败样本",
|
|
input_payload={"message": "查一下项目风险"},
|
|
expected_contract={"required_input_keys": ["workflow_bundle"]},
|
|
priority="important",
|
|
created_by=1001,
|
|
)
|
|
|
|
replay = await service.create_replay_run(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_RAG_QUERY_SEMANTICS",
|
|
release_id=release["id"],
|
|
created_by=1001,
|
|
)
|
|
|
|
assert replay["failed_count"] == 0
|
|
assert replay["warning_count"] == 1
|
|
assert replay["results"][0]["status"] == "warning"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_release_watch_summary(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
release = await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="运营定制：{question}",
|
|
created_by=1001,
|
|
)
|
|
db_session.add(
|
|
AICallLog(
|
|
tenant_id=1,
|
|
function_name="adaptive_rag_answer_normal",
|
|
prompt="运营定制：这周项目风险是什么？",
|
|
response="[error] provider timeout",
|
|
status="failed",
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
prompt_release_id=release["id"],
|
|
prompt_source="active_release",
|
|
latency_ms=300,
|
|
cost=0.004,
|
|
)
|
|
)
|
|
await db_session.commit()
|
|
|
|
watch = await service.get_release_watch_summary(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
release_id=release["id"],
|
|
)
|
|
|
|
assert watch["recommendation"] in {"needs_review", "rollback_suggested"}
|
|
assert watch["windows"]["24h"]["total_calls"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prompt_asset_service_rejects_migration_needed_or_invalid_release(db_session) -> None:
|
|
service = PromptAssetService(db_session)
|
|
|
|
with pytest.raises(ValueError, match="migrated"):
|
|
await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_SIGNAL_WEBPAGE_VALUE_ANALYSIS",
|
|
body_snapshot="不能直接发布",
|
|
created_by=1001,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="unknown variables"):
|
|
await service.create_release(
|
|
tenant_id=1,
|
|
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
|
body_snapshot="未知变量：{unknown_signal}",
|
|
created_by=1001,
|
|
)
|