258 lines
8.8 KiB
Python
258 lines
8.8 KiB
Python
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
import pytest
|
||
import pytest_asyncio
|
||
from sqlalchemy import JSON
|
||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||
|
||
os.environ["DATABASE_URL"] = "sqlite+aiosqlite:///:memory:"
|
||
|
||
from app.models.base import Base
|
||
from app.models.experiment import Experiment
|
||
from app.models.feedback_center import FeedbackCenterRecord
|
||
from app.models.qa_record import QARecord
|
||
from app.models.reply_iteration import (
|
||
JudgeRun,
|
||
OptimizerCandidate,
|
||
PromptRelease,
|
||
QualityAsset,
|
||
QualityAssetPair,
|
||
)
|
||
from app.services.prompt_release.service import PromptReleaseService
|
||
from app.services.reply_iteration.service import ReplyIterationService
|
||
|
||
|
||
def _use_sqlite_json_types() -> None:
|
||
for table in (
|
||
FeedbackCenterRecord.__table__,
|
||
QARecord.__table__,
|
||
QualityAsset.__table__,
|
||
QualityAssetPair.__table__,
|
||
JudgeRun.__table__,
|
||
OptimizerCandidate.__table__,
|
||
PromptRelease.__table__,
|
||
Experiment.__table__,
|
||
):
|
||
for column in table.columns:
|
||
if column.name in {
|
||
"before_value",
|
||
"after_value",
|
||
"ai_snapshot",
|
||
"reason_candidates",
|
||
"sources",
|
||
"domains_used",
|
||
"routing_snapshot",
|
||
"citations",
|
||
"sources_used",
|
||
"evidence_pack",
|
||
"tool_trace",
|
||
"retrieval_metrics",
|
||
"evidence",
|
||
"source_snapshot",
|
||
"tags",
|
||
"metadata_json",
|
||
"findings",
|
||
"llm_evidence",
|
||
"suggestion",
|
||
"scope_json",
|
||
"linked_context_nodes",
|
||
"result_summary",
|
||
}:
|
||
column.type = JSON()
|
||
|
||
|
||
@pytest_asyncio.fixture
|
||
async def db_session():
|
||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||
_use_sqlite_json_types()
|
||
async with engine.begin() as conn:
|
||
await conn.run_sync(
|
||
lambda sync_conn: Base.metadata.create_all(
|
||
sync_conn,
|
||
tables=[
|
||
FeedbackCenterRecord.__table__,
|
||
QARecord.__table__,
|
||
QualityAsset.__table__,
|
||
QualityAssetPair.__table__,
|
||
JudgeRun.__table__,
|
||
OptimizerCandidate.__table__,
|
||
PromptRelease.__table__,
|
||
Experiment.__table__,
|
||
],
|
||
)
|
||
)
|
||
maker = async_sessionmaker(engine, expire_on_commit=False)
|
||
async with maker() as session:
|
||
yield session
|
||
await engine.dispose()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_promote_feedback_record_builds_asset_and_preference_pair(db_session) -> None:
|
||
qa_record = QARecord(
|
||
tenant_id=1,
|
||
question="这个回复为什么不对",
|
||
answer="让我先看一下你的情况，再帮你判断。",
|
||
sources=[],
|
||
intent="question",
|
||
domains_used=[],
|
||
response_time_ms=120,
|
||
asked_by=None,
|
||
asked_via="feishu",
|
||
project_id=8,
|
||
turn_index=1,
|
||
routing_snapshot={
|
||
"planned_tier": "low",
|
||
"executed_tier": "high",
|
||
"route_source": "default",
|
||
"route_rule_name": "engine_high_answer",
|
||
},
|
||
)
|
||
db_session.add(qa_record)
|
||
await db_session.flush()
|
||
|
||
failure_record = FeedbackCenterRecord(
|
||
tenant_id=1,
|
||
project_id=8,
|
||
source_type="qa_feedback",
|
||
source_id=101,
|
||
object_type="qa_record",
|
||
object_id=qa_record.id,
|
||
field_name="qa_feedback",
|
||
source_title="飞书问答反馈",
|
||
object_title="问答修正",
|
||
before_value={"question": qa_record.question, "answer": qa_record.answer},
|
||
after_value={"rating": "negative", "correct_answer": "上一条回复走的是 high 路由。"},
|
||
ai_snapshot={"prompt_key": "PROMPT_FEISHU_DIRECT_QA"},
|
||
reason_code="answer_incorrect",
|
||
review_status="accepted",
|
||
)
|
||
golden_record = FeedbackCenterRecord(
|
||
tenant_id=1,
|
||
project_id=8,
|
||
source_type="qa_feedback",
|
||
source_id=102,
|
||
object_type="qa_record",
|
||
object_id=qa_record.id,
|
||
field_name="qa_feedback",
|
||
source_title="飞书问答反馈",
|
||
object_title="问答修正",
|
||
before_value={"question": qa_record.question, "answer": qa_record.answer},
|
||
after_value={"rating": "positive", "correct_answer": "这条回复实际走的是 high，来源于默认规则。"},
|
||
ai_snapshot={"prompt_key": "PROMPT_FEISHU_DIRECT_QA"},
|
||
reason_code="answer_incorrect",
|
||
review_status="accepted",
|
||
)
|
||
db_session.add_all([failure_record, golden_record])
|
||
await db_session.commit()
|
||
|
||
service = ReplyIterationService(db_session)
|
||
failure_result = await service.promote_feedback_record(
|
||
tenant_id=1,
|
||
record_id=failure_record.id,
|
||
actor_id=9001,
|
||
asset_type="failure",
|
||
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
||
workflow_type="query.read_only",
|
||
)
|
||
golden_result = await service.promote_feedback_record(
|
||
tenant_id=1,
|
||
record_id=golden_record.id,
|
||
actor_id=9001,
|
||
asset_type="golden",
|
||
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
||
workflow_type="query.read_only",
|
||
pair_with_asset_id=failure_result["asset"]["id"],
|
||
pair_role="chosen",
|
||
pair_reason="人工确认修正版明显优于原答复",
|
||
)
|
||
|
||
assert failure_result["asset"]["asset_type"] == "failure"
|
||
assert failure_result["asset"]["qa_record_id"] == qa_record.id
|
||
assert failure_result["asset"]["planned_tier"] == "low"
|
||
assert failure_result["asset"]["executed_tier"] == "high"
|
||
assert failure_result["asset"]["route_rule_name"] == "engine_high_answer"
|
||
assert failure_result["pair"] is None
|
||
|
||
assert golden_result["asset"]["asset_type"] == "golden"
|
||
assert golden_result["pair"] is not None
|
||
assert golden_result["pair"]["chosen_asset_id"] == golden_result["asset"]["id"]
|
||
assert golden_result["pair"]["rejected_asset_id"] == failure_result["asset"]["id"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_judge_run_and_optimizer_candidate_follow_bucket_rules(db_session) -> None:
|
||
asset = QualityAsset(
|
||
tenant_id=1,
|
||
project_id=3,
|
||
asset_type="failure",
|
||
status="approved",
|
||
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
||
workflow_type="query.read_only",
|
||
asked_via="feishu",
|
||
planned_tier="low",
|
||
executed_tier="low",
|
||
route_rule_name="engine_low_fallback",
|
||
question_text="这个回复为什么不对",
|
||
answer_text="让我先看一下，再通过 API 帮你创建。",
|
||
corrected_answer="直接说明这条回复走的是 low，并给出路由来源。",
|
||
source_snapshot={"reason_code": "answer_incorrect"},
|
||
)
|
||
db_session.add(asset)
|
||
await db_session.commit()
|
||
|
||
service = ReplyIterationService(db_session)
|
||
judge = await service.create_judge_run(
|
||
tenant_id=1,
|
||
asset_id=asset.id,
|
||
actor_id=9001,
|
||
)
|
||
candidate = await service.create_optimizer_candidate(
|
||
tenant_id=1,
|
||
judge_run_id=judge["id"],
|
||
actor_id=9001,
|
||
)
|
||
|
||
assert judge["bucket_key"] == "PROMPT_FEISHU_DIRECT_QA|query.read_only|feishu|low|low"
|
||
finding_codes = {item["code"] for item in judge["findings"]}
|
||
assert "process_talk_prefix" in finding_codes
|
||
assert "execution_claim_prefix" in finding_codes
|
||
assert candidate["target_type"] == "renderer_policy"
|
||
assert candidate["status"] == "draft"
|
||
assert candidate["target_ref"] == "reply_renderer"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_prompt_release_service_prefers_active_release_and_fails_closed_on_format_error(db_session) -> None:
|
||
service = PromptReleaseService(db_session)
|
||
release = await service.create_release(
|
||
tenant_id=1,
|
||
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
||
body_snapshot="【定制版】问题：{question}",
|
||
created_by=9001,
|
||
notes="定制回复语气",
|
||
)
|
||
await service.activate_release(
|
||
tenant_id=1,
|
||
release_id=release["id"],
|
||
actor_id=9001,
|
||
)
|
||
|
||
rendered = await service.render_prompt(
|
||
tenant_id=1,
|
||
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
||
fallback_template="默认：{question}",
|
||
question="系统现在走哪个 tier",
|
||
)
|
||
assert rendered == "【定制版】问题：系统现在走哪个 tier"
|
||
with pytest.raises(RuntimeError, match="active prompt release render failed"):
|
||
await service.render_prompt(
|
||
tenant_id=1,
|
||
prompt_key="PROMPT_FEISHU_DIRECT_QA",
|
||
fallback_template="默认：{question}",
|
||
question="系统现在走哪个 tier",
|
||
_override_template="坏模板：{missing_key}",
|
||
)
|