961 lines
43 KiB
Python
961 lines
43 KiB
Python
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
|
||
from app.services.external_query_service.planner import ExternalQueryPlanner
|
||
from app.services.external_query_service.presenter import ExternalAnswerPresenter
|
||
|
||
|
||
def test_fallback_plan_stays_generic_for_fresh_weather_semantics() -> None:
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=object(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
plan = planner._build_fallback_plan(
|
||
question="帮我看看未来5天的广州天气如何",
|
||
query_semantics={
|
||
"query_type": "status",
|
||
"object_mentions": ["广州"],
|
||
"query_profile": {
|
||
"needs_freshness": True,
|
||
"referenced_objects": ["weather"],
|
||
},
|
||
},
|
||
)
|
||
|
||
assert plan.query_kind == "generic_web"
|
||
assert plan.answer_contract == "summary_with_sources"
|
||
assert plan.location_hint is None
|
||
assert plan.result_limit == 3
|
||
|
||
|
||
def test_fallback_plan_uses_compare_contract_without_object_kind_mapping() -> None:
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=object(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
plan = planner._build_fallback_plan(
|
||
question="对比 OpenAI 和 Anthropic 最新模型能力差异",
|
||
query_semantics={
|
||
"query_type": "compare",
|
||
"object_mentions": ["OpenAI", "Anthropic"],
|
||
"query_profile": {
|
||
"needs_freshness": True,
|
||
"referenced_objects": ["public_model"],
|
||
},
|
||
},
|
||
)
|
||
|
||
assert plan.query_kind == "generic_web"
|
||
assert plan.answer_contract == "compare_with_sources"
|
||
assert plan.requires_multi_source is True
|
||
assert plan.result_limit == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_build_plan_threads_fit_brief_guidance_into_prompt_and_plan() -> None:
|
||
class _FakeLLM:
|
||
def __init__(self) -> None:
|
||
self.prompt = ""
|
||
|
||
async def generate_structured_object(self, **kwargs):
|
||
self.prompt = str(kwargs.get("prompt") or "")
|
||
return SimpleNamespace(
|
||
payload={
|
||
"topic": "Responses API 适配 PMFlow",
|
||
"search_queries": ["OpenAI Responses API project management workflow"],
|
||
}
|
||
)
|
||
|
||
fake_llm = _FakeLLM()
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=fake_llm,
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
fit_brief = {
|
||
"project_summary": "当前项目：PMFlow AI 管家，聚焦项目内问答与需求流转。",
|
||
"repo_summary": "当前仓库：FastAPI 后端 + React 前端。",
|
||
"current_capabilities": ["AI 管家", "智能配置"],
|
||
"constraints": ["只允许基于外部公开证据给出推荐。"],
|
||
"recommendation_focus": ["判断 Responses API 是否适合当前项目接入"],
|
||
}
|
||
plan, meta = await planner.build_plan(
|
||
question="这个项目适不适合接 OpenAI Responses API？",
|
||
query_semantics={
|
||
"query_profile": {"needs_freshness": True},
|
||
"object_mentions": ["OpenAI Responses API"],
|
||
},
|
||
fit_brief=fit_brief,
|
||
)
|
||
|
||
assert meta["planner"] == "llm_structured"
|
||
assert "PMFlow AI 管家" in fake_llm.prompt
|
||
assert "智能配置" in fake_llm.prompt
|
||
assert plan.fit_brief_summary
|
||
assert plan.project_scope_hint == "current_project_and_repo"
|
||
assert "适合当前项目接入" in str(plan.recommendation_goal)
|
||
assert "AI 管家" in plan.project_object_anchors
|
||
|
||
|
||
def test_fallback_plan_builds_concrete_signal_source_recommendation_queries() -> None:
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=object(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
plan = planner._build_fallback_plan(
|
||
question="测试产能警告这个项目，如果只考虑外部信号源，你会优先推荐哪 3 类？直接给推荐结果和理由。",
|
||
query_semantics={
|
||
"query_profile": {"needs_freshness": True},
|
||
},
|
||
fit_brief={
|
||
"project_summary": "当前项目：#81 测试产能警告 (status=active)",
|
||
"recommendation_focus": ["给测试产能警告推荐最值得先看的外部信号源。"],
|
||
"signal_source_objectives": [
|
||
"设计产能监控逻辑与数据采集：编写脚本收集测试队列与资源使用数据",
|
||
"实现产能警告触发服务：实时比对数据与阈值",
|
||
],
|
||
},
|
||
)
|
||
|
||
joined_queries = " ".join(plan.search_queries[:2])
|
||
assert plan.search_queries[0].startswith("测试产能警告 外部信号源")
|
||
assert "测试队列与资源使用数据" in joined_queries
|
||
assert "实时比对数据与阈值" in joined_queries
|
||
assert "发布变更" not in joined_queries
|
||
assert "工单" not in joined_queries
|
||
|
||
|
||
def test_fallback_plan_preserves_structured_interaction_level() -> None:
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=object(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
plan = planner._build_fallback_plan(
|
||
question="这个项目我想先补外部东西，你先帮我列几个最值得看的。",
|
||
query_semantics={
|
||
"query_profile": {
|
||
"needs_freshness": True,
|
||
"interaction_level": "recommendation",
|
||
},
|
||
"interaction_level": "recommendation",
|
||
},
|
||
fit_brief={
|
||
"project_summary": "当前项目：自动开票系统，目标是在收到开票请求后自动开票并发送发票图片。",
|
||
"recommendation_focus": ["给自动开票系统推荐最值得先看的外部信号源。"],
|
||
"signal_source_objectives": [
|
||
"处理开票请求：收到接口请求后自动开票",
|
||
"发送发票图片：把结果发给客户",
|
||
],
|
||
},
|
||
)
|
||
|
||
assert plan.interaction_level == "recommendation"
|
||
|
||
|
||
def test_fallback_plan_builds_store_inventory_recommendation_queries_from_project_objectives() -> None:
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=object(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
plan = planner._build_fallback_plan(
|
||
question="门店库存异常日报助手如果想更早发现异常，先看哪几类外部信号最值？",
|
||
query_semantics={
|
||
"query_profile": {
|
||
"needs_freshness": True,
|
||
"interaction_level": "recommendation",
|
||
},
|
||
"interaction_level": "recommendation",
|
||
},
|
||
fit_brief={
|
||
"project_summary": "当前项目：门店库存异常日报助手，目标是每日自动汇总门店库存异常并发给区域经理。",
|
||
"recommendation_focus": ["给门店库存异常日报助手推荐最值得先看的外部信号源。"],
|
||
"signal_source_objectives": [
|
||
"汇总门店库存异常：识别缺货、库存跳变和异常门店分布",
|
||
"推送区域经理日报：把高风险异常按区域维度发给负责人",
|
||
],
|
||
},
|
||
)
|
||
|
||
joined_queries = " ".join(plan.search_queries[:3])
|
||
assert plan.interaction_level == "recommendation"
|
||
assert plan.search_queries[0].startswith("门店库存异常日报助手 外部信号源")
|
||
assert "缺货" in joined_queries
|
||
assert "库存跳变" in joined_queries
|
||
assert "区域经理" in joined_queries
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_build_plan_signal_source_recommendation_keeps_concrete_seed_queries_ahead_of_generic_model_query() -> None:
|
||
class _FakeLLM:
|
||
async def generate_structured_object(self, **kwargs):
|
||
_ = kwargs
|
||
return SimpleNamespace(
|
||
payload={
|
||
"topic": "测试产能警告外部信号源推荐",
|
||
"search_queries": ["production capacity warning system external data sources industry best practices"],
|
||
}
|
||
)
|
||
|
||
planner = ExternalQueryPlanner(
|
||
llm_service=_FakeLLM(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
plan, meta = await planner.build_plan(
|
||
question="测试产能警告这个项目，如果只考虑外部信号源，你会优先推荐哪 3 类？直接给推荐结果和理由。",
|
||
query_semantics={
|
||
"query_profile": {"needs_freshness": True},
|
||
},
|
||
fit_brief={
|
||
"project_summary": "当前项目：#81 测试产能警告 (status=active)",
|
||
"recommendation_focus": ["给测试产能警告推荐最值得先看的外部信号源。"],
|
||
"signal_source_objectives": [
|
||
"设计产能监控逻辑与数据采集：编写脚本收集测试队列与资源使用数据",
|
||
"实现产能警告触发服务：实时比对数据与阈值",
|
||
],
|
||
},
|
||
)
|
||
|
||
assert meta["planner"] == "llm_structured"
|
||
joined_queries = " ".join(plan.search_queries[:3])
|
||
assert plan.search_queries[0].startswith("测试产能警告 外部信号源")
|
||
assert "测试队列与资源使用数据" in joined_queries
|
||
assert "实时比对数据与阈值" in joined_queries
|
||
assert "production capacity warning system external data sources industry best practices" in plan.search_queries
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize(
|
||
("retrieval_status", "expected_variant", "expected_phrase"),
|
||
[
|
||
("verifier_blocked", "external_verifier_blocked", "没有通过核验"),
|
||
("no_relevant_results", "external_no_relevant_results", "没有找到足够相关的外部公开证据"),
|
||
("retrieval_failed", "external_retrieval_failed", "外部检索这次执行失败"),
|
||
],
|
||
)
|
||
async def test_presenter_distinguishes_external_failure_states(
|
||
retrieval_status: str,
|
||
expected_variant: str,
|
||
expected_phrase: str,
|
||
) -> None:
|
||
class _FakeLLM:
|
||
def __init__(self) -> None:
|
||
self.calls = 0
|
||
|
||
async def generate(self, *args, **kwargs):
|
||
_ = (args, kwargs)
|
||
self.calls += 1
|
||
return "should not be used"
|
||
|
||
fake_llm = _FakeLLM()
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=fake_llm,
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="这个项目适不适合接某个外部能力？",
|
||
context_packet={
|
||
"question": "这个项目适不适合接某个外部能力？",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "外部能力适配判断",
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": retrieval_status,
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert result["prompt_variant_id"] == expected_variant
|
||
assert expected_phrase in result["answer_text"]
|
||
assert fake_llm.calls == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_distinguishes_indirect_only_without_overclaiming_direct_match() -> None:
|
||
class _FakeLLM:
|
||
def __init__(self) -> None:
|
||
self.calls = 0
|
||
|
||
async def generate(self, *args, **kwargs):
|
||
_ = (args, kwargs)
|
||
self.calls += 1
|
||
return "should not be used"
|
||
|
||
fake_llm = _FakeLLM()
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=fake_llm,
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="这个项目要不要跟进 OpenAI Responses API？",
|
||
context_packet={
|
||
"question": "这个项目要不要跟进 OpenAI Responses API？",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "Responses API 适配性",
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "success"}],
|
||
"verified_evidence": [
|
||
{
|
||
"source_type": "url",
|
||
"source_label": "OpenAI Docs",
|
||
"title": "Responses API guide",
|
||
"target_url": "https://platform.openai.com/docs/guides/responses",
|
||
"snippet": "Responses API can orchestrate tool calls and structured outputs.",
|
||
}
|
||
],
|
||
"verifier": {"ok": True},
|
||
"uncertainties": [],
|
||
"retrieval_status": "evidence_found",
|
||
"impact_status": "indirect_only",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert result["prompt_variant_id"] == "external_indirect_only"
|
||
assert "只能说明它和当前项目有背景相关性" in result["answer_text"]
|
||
assert "不能直接说明它已经命中当前项目对象" in result["answer_text"]
|
||
assert fake_llm.calls == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_prefers_recommendation_answer_for_signal_source_recommendation_question() -> None:
|
||
class _FakeLLM:
|
||
def __init__(self) -> None:
|
||
self.calls = 0
|
||
self.prompt = ""
|
||
self.structured_calls = 0
|
||
|
||
async def generate_structured_object(self, **kwargs):
|
||
self.structured_calls += 1
|
||
self.prompt = str(kwargs.get("prompt") or "")
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "测试执行失败和阻塞记录",
|
||
"reason": "它最早反映测试队列是不是已经开始堆积，适合先盯住产能吃紧的信号。",
|
||
},
|
||
{
|
||
"name": "构建或发布变更记录",
|
||
"reason": "它能帮你判断产能波动是不是由最近的版本改动带出来的。",
|
||
},
|
||
{
|
||
"name": "工单或异常反馈",
|
||
"reason": "它能看出问题有没有开始影响到实际交付，不会只停在内部指标波动。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = kwargs
|
||
self.calls += 1
|
||
self.prompt = prompt
|
||
return (
|
||
"如果只按当前项目任务来排优先级，我会先补齐“设计产能监控逻辑与数据采集”需要的外部输入，"
|
||
"再看“实现产能警告触发服务”依赖的实时变化信号。"
|
||
)
|
||
|
||
fake_llm = _FakeLLM()
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=fake_llm,
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="测试产能警告这个项目，如果只考虑外部信号源，你会优先推荐哪 3 类？直接给推荐结果和理由。",
|
||
context_packet={
|
||
"question": "测试产能警告这个项目，如果只考虑外部信号源，你会优先推荐哪 3 类？直接给推荐结果和理由。",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "测试产能警告外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：测试产能警告，聚焦识别产能异常和告警触发。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给测试产能警告推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["实现产能警告触发服务", "告警接收链路"],
|
||
"signal_source_objectives": [
|
||
"设计产能监控逻辑与数据采集：编写脚本收集测试队列与资源使用数据",
|
||
"实现产能警告触发服务：实时比对数据与阈值",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert result["prompt_variant_id"] == "external_signal_source_recommendation"
|
||
assert "1. 测试执行失败和阻塞记录" in result["answer_text"]
|
||
assert "2. 构建或发布变更记录" in result["answer_text"]
|
||
assert "3. 工单或异常反馈" in result["answer_text"]
|
||
assert "最早反映测试队列是不是已经开始堆积" in result["answer_text"]
|
||
assert "这些先算建议，不代表已经接入或验证完成。" in result["answer_text"]
|
||
assert "主链" not in result["answer_text"]
|
||
assert "这一层" not in result["answer_text"]
|
||
assert result["recommendation_packet"]["resource_role"] == "signal_source"
|
||
assert result["recommendation_packet"]["source_kind"] == "external"
|
||
assert result["recommendation_packet"]["evidence_mode"] == "advisory"
|
||
assert result["recommendation_packet"]["candidate_set_id"]
|
||
assert result["recommendation_packet"]["candidates"][0]["candidate_ref"]
|
||
assert result["recommendation_packet"]["candidates"][0]["display_name"] == "测试执行失败和阻塞记录"
|
||
assert (
|
||
result["recommendation_packet"]["candidates"][0]["materialization_payload"]["display_name"]
|
||
== "测试执行失败和阻塞记录"
|
||
)
|
||
assert fake_llm.structured_calls == 1
|
||
assert "给测试产能警告推荐最值得先看的外部信号源" in fake_llm.prompt
|
||
assert "测试队列与资源使用数据" in fake_llm.prompt
|
||
assert "实时比对数据与阈值" in fake_llm.prompt
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_signal_source_recommendation_varies_by_project_context() -> None:
|
||
class _FakeLLM:
|
||
def __init__(self) -> None:
|
||
self.structured_calls = 0
|
||
|
||
async def generate_structured_object(self, **kwargs):
|
||
self.structured_calls += 1
|
||
prompt = str(kwargs.get("prompt") or "")
|
||
if "自动开票系统" in prompt:
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "开票失败或重试回执",
|
||
"reason": "它最早告诉你票是不是卡在开票动作本身。",
|
||
},
|
||
{
|
||
"name": "税控或发票平台状态变化",
|
||
"reason": "它能解释失败到底是业务问题还是外部平台波动。",
|
||
},
|
||
{
|
||
"name": "发票发送失败反馈",
|
||
"reason": "它能看出问题有没有真正影响到客户收票。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "用户投诉和差评变化",
|
||
"reason": "它最容易最先暴露客服回复质量是不是已经开始出问题。",
|
||
},
|
||
{
|
||
"name": "社媒互动异常波动",
|
||
"reason": "它能帮你看出问题是不是已经从单点反馈扩到公开场域。",
|
||
},
|
||
{
|
||
"name": "竞品或行业FAQ变化",
|
||
"reason": "它能提示当前话术和知识点是不是已经落后。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = (prompt, kwargs)
|
||
return "如果只看外部信号源，我会先看最关键的外部输入。"
|
||
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=_FakeLLM(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
invoice_result = await presenter.present(
|
||
question="自动开票系统这个项目，除了系统里现有的数据，你觉得还应该盯哪些外部信号？直接说几个最有用的，不要给我出方案。",
|
||
context_packet={
|
||
"question": "自动开票系统这个项目，除了系统里现有的数据，你觉得还应该盯哪些外部信号？直接说几个最有用的，不要给我出方案。",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "自动开票系统外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：自动开票系统，目标是在收到开票请求后自动开票并发送发票图片。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给自动开票系统推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["开票请求处理", "发票发送结果"],
|
||
"signal_source_objectives": [
|
||
"处理开票请求：收到接口请求后自动开票",
|
||
"发送发票图片：把结果发给客户",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
ai_customer_result = await presenter.present(
|
||
question="AI客服这个项目，除了现在这些数据，你觉得最值得再补哪几类外部线索？直接推荐几个。",
|
||
context_packet={
|
||
"question": "AI客服这个项目，除了现在这些数据，你觉得最值得再补哪几类外部线索？直接推荐几个。",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "AI客服外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：AI客服，核心痛点是内容本地化，当前能力围绕社媒运营和用户洞察。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给 AI 客服推荐最值得先看的外部线索。",
|
||
"project_object_anchors": ["用户洞察数据源", "社媒运营数据源"],
|
||
"signal_source_objectives": [
|
||
"用户洞察：更快发现用户反馈与投诉变化",
|
||
"社媒运营：跟踪公开互动和话题波动",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert "开票失败或重试回执" in invoice_result["answer_text"]
|
||
assert "发票发送失败反馈" in invoice_result["answer_text"]
|
||
assert "用户投诉和差评变化" not in invoice_result["answer_text"]
|
||
|
||
assert "用户投诉和差评变化" in ai_customer_result["answer_text"]
|
||
assert "社媒互动异常波动" in ai_customer_result["answer_text"]
|
||
assert "开票失败或重试回执" not in ai_customer_result["answer_text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_signal_source_recommendation_varies_for_cashflow_and_knowledge_base_projects() -> None:
|
||
class _FakeLLM:
|
||
async def generate_structured_object(self, *, prompt: str, **kwargs):
|
||
_ = kwargs
|
||
if "智能现金流管家" in prompt:
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "银行回单或到账异常反馈",
|
||
"reason": "它最早告诉你资金是不是已经在到账环节卡住了。",
|
||
},
|
||
{
|
||
"name": "税费或社保扣缴提醒变化",
|
||
"reason": "它能帮助区分现金流压力是不是已经开始影响刚性支出。",
|
||
},
|
||
{
|
||
"name": "上游平台结算延迟通知",
|
||
"reason": "它能提前暴露回款节奏是不是被外部平台拖慢了。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "外部知识源更新通知",
|
||
"reason": "它能让知识库问答先知道哪些权威资料已经变化。",
|
||
},
|
||
{
|
||
"name": "第三方模型服务可用性波动",
|
||
"reason": "它最容易直接影响问答响应质量和稳定性。",
|
||
},
|
||
{
|
||
"name": "企业 SSO 或目录服务异常公告",
|
||
"reason": "它能帮助判断问答不可用是不是卡在登录或权限链路。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = (prompt, kwargs)
|
||
return "关于外部信号，我现在还不能直接推荐。"
|
||
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=_FakeLLM(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
cashflow_result = await presenter.present(
|
||
question="智能现金流管家除了银行流水，还值得补哪些外部线索？",
|
||
context_packet={
|
||
"question": "智能现金流管家除了银行流水，还值得补哪些外部线索？",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "智能现金流管家外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：智能现金流管家，目标是更早识别回款异常和资金压力。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给智能现金流管家推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["回款节奏监控", "资金压力预警"],
|
||
"signal_source_objectives": [
|
||
"识别到账异常：更早发现回款延迟和结算波动",
|
||
"识别刚性支出压力：尽快发现税费或社保扣缴变化",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
knowledge_base_result = await presenter.present(
|
||
question="企业知识库问答AI项目如果只补一个外部信号，你觉得先补哪个最值？",
|
||
context_packet={
|
||
"question": "企业知识库问答AI项目如果只补一个外部信号，你觉得先补哪个最值？",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "企业知识库问答AI项目外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：企业知识库问答AI项目，目标是提升企业知识问答的准确性和可用性。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给企业知识库问答AI项目推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["知识更新同步", "问答服务稳定性"],
|
||
"signal_source_objectives": [
|
||
"识别知识更新：及时发现外部知识源变化",
|
||
"识别服务波动：尽快定位问答质量和可用性异常",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert "银行回单或到账异常反馈" in cashflow_result["answer_text"]
|
||
assert "税费或社保扣缴提醒变化" in cashflow_result["answer_text"]
|
||
assert "外部知识源更新通知" not in cashflow_result["answer_text"]
|
||
|
||
assert "外部知识源更新通知" in knowledge_base_result["answer_text"]
|
||
assert "第三方模型服务可用性波动" in knowledge_base_result["answer_text"]
|
||
assert "银行回单或到账异常反馈" not in knowledge_base_result["answer_text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_signal_source_recommendation_varies_for_medical_industry_research_project() -> None:
|
||
class _FakeLLM:
|
||
async def generate_structured_object(self, **kwargs):
|
||
_ = kwargs
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "医药政策与监管口径更新",
|
||
"reason": "它最容易先改变研究假设和布局判断。",
|
||
},
|
||
{
|
||
"name": "重点药企投融资和并购动态",
|
||
"reason": "它能更快暴露产业链方向是不是正在变化。",
|
||
},
|
||
{
|
||
"name": "临床试验与注册进展公告",
|
||
"reason": "它能帮助判断哪些赛道是真的在加速，哪些还停在预期里。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = (prompt, kwargs)
|
||
return "我现在还不能直接推荐。"
|
||
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=_FakeLLM(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="医药行业十五五规划布局研究现在最值得优先盯的外部变化有哪些？",
|
||
context_packet={
|
||
"question": "医药行业十五五规划布局研究现在最值得优先盯的外部变化有哪些？",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "医药行业十五五规划布局研究外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：医药行业十五五规划布局研究，目标是研判政策、产业和重点赛道变化。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给医药行业十五五规划布局研究推荐最值得优先看的外部变化。",
|
||
"project_object_anchors": ["政策口径变化", "重点赛道布局判断"],
|
||
"signal_source_objectives": [
|
||
"识别政策变化：尽快发现监管和支持方向的变化",
|
||
"识别产业变化：尽快发现资本和临床进展带来的赛道变化",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert "1. 医药政策与监管口径更新" in result["answer_text"]
|
||
assert "2. 重点药企投融资和并购动态" in result["answer_text"]
|
||
assert "3. 临床试验与注册进展公告" in result["answer_text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_signal_source_recommendation_accepts_natural_business_phrasing() -> None:
|
||
class _FakeLLM:
|
||
async def generate_structured_object(self, **kwargs):
|
||
_ = kwargs
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "门店缺货和补货异常反馈",
|
||
"reason": "它最早反映异常是不是已经在门店端出现。",
|
||
},
|
||
{
|
||
"name": "供应商到货延迟变化",
|
||
"reason": "它能帮助区分库存异常到底是销售波动还是供给变慢。",
|
||
},
|
||
{
|
||
"name": "竞品价格或促销波动",
|
||
"reason": "它能解释销量突然变化是不是被外部活动带出来的。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = (prompt, kwargs)
|
||
return "关于外部信号，我现在还不能直接推荐。"
|
||
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=_FakeLLM(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="门店库存异常日报助手这个项目，如果想更早发现异常，你会先多看哪几类外部信号？",
|
||
context_packet={
|
||
"question": "门店库存异常日报助手这个项目，如果想更早发现异常，你会先多看哪几类外部信号？",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "门店库存异常日报助手外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：门店库存异常日报助手，目标是更早发现缺货、滞销和补货异常。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给门店库存异常日报助手推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["门店异常日报", "缺货异常识别"],
|
||
"signal_source_objectives": [
|
||
"识别门店异常：更早发现缺货和补货异常",
|
||
"日报推送：让运营尽快看到高风险门店",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert "1. 门店缺货和补货异常反馈" in result["answer_text"]
|
||
assert "2. 供应商到货延迟变化" in result["answer_text"]
|
||
assert "3. 竞品价格或促销波动" in result["answer_text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_signal_source_recommendation_uses_plan_interaction_level_not_question_cues() -> None:
|
||
class _FakeLLM:
|
||
async def generate_structured_object(self, **kwargs):
|
||
_ = kwargs
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "开票失败或重试回执",
|
||
"reason": "它最早告诉你票是不是卡在开票动作本身。",
|
||
},
|
||
{
|
||
"name": "税控或发票平台状态变化",
|
||
"reason": "它能解释失败到底是业务问题还是外部平台波动。",
|
||
},
|
||
{
|
||
"name": "发票发送失败反馈",
|
||
"reason": "它能看出问题有没有真正影响到客户收票。",
|
||
},
|
||
],
|
||
"caveat": "这些先算建议，不代表已经接入或验证完成。",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = (prompt, kwargs)
|
||
return "不该走到这里。"
|
||
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=_FakeLLM(),
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="自动开票系统这个项目，我想先把票开不出去这块外部情况摸清楚。",
|
||
context_packet={
|
||
"question": "自动开票系统这个项目，我想先把票开不出去这块外部情况摸清楚。",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "自动开票系统外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：自动开票系统，目标是在收到开票请求后自动开票并发送发票图片。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给自动开票系统推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["开票请求处理", "发票发送结果"],
|
||
"signal_source_objectives": [
|
||
"处理开票请求：收到接口请求后自动开票",
|
||
"发送发票图片：把结果发给客户",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "empty"}],
|
||
"verified_evidence": [],
|
||
"verifier": {"ok": False, "reason_code": "external_evidence_off_topic"},
|
||
"uncertainties": ["external_evidence_off_topic"],
|
||
"retrieval_status": "verifier_blocked",
|
||
"impact_status": "no_direct_impact",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert result["prompt_variant_id"] == "external_signal_source_recommendation"
|
||
assert "1. 开票失败或重试回执" in result["answer_text"]
|
||
assert "2. 税控或发票平台状态变化" in result["answer_text"]
|
||
assert "3. 发票发送失败反馈" in result["answer_text"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_presenter_signal_source_recommendation_falls_back_when_llm_overclaims_unverified_details() -> None:
|
||
class _FakeLLM:
|
||
def __init__(self) -> None:
|
||
self.structured_calls = 0
|
||
|
||
async def generate_structured_object(self, **kwargs):
|
||
_ = kwargs
|
||
self.structured_calls += 1
|
||
return SimpleNamespace(
|
||
payload={
|
||
"candidates": [
|
||
{
|
||
"name": "工厂级设备运行日志",
|
||
"reason": "近一年案例显示其与突发性产能下滑相关性达73%。",
|
||
},
|
||
{
|
||
"name": "测试任务调度系统记录",
|
||
"reason": "当前已确认无需额外对接验证。",
|
||
},
|
||
],
|
||
"caveat": "",
|
||
}
|
||
)
|
||
|
||
async def generate(self, prompt: str, **kwargs):
|
||
_ = (prompt, kwargs)
|
||
return (
|
||
"如果只看外部信号源，我会先看工厂级设备运行日志、测试任务调度系统记录、产线温湿度与电力波动数据。"
|
||
)
|
||
|
||
fake_llm = _FakeLLM()
|
||
presenter = ExternalAnswerPresenter(
|
||
llm_service=fake_llm,
|
||
db=object(),
|
||
tenant_id=1,
|
||
)
|
||
|
||
result = await presenter.present(
|
||
question="测试产能警告这个项目，如果只考虑外部信号源，你会优先推荐哪 3 类？直接给推荐结果和理由。",
|
||
context_packet={
|
||
"question": "测试产能警告这个项目，如果只考虑外部信号源，你会优先推荐哪 3 类？直接给推荐结果和理由。",
|
||
"plan": {
|
||
"answer_contract": "summary_with_sources",
|
||
"topic": "测试产能警告外部信号源推荐",
|
||
"interaction_level": "recommendation",
|
||
"fit_brief_summary": "当前项目：测试产能警告，聚焦识别产能异常和告警触发。",
|
||
"project_scope_hint": "current_project_and_repo",
|
||
"recommendation_goal": "给测试产能警告推荐最值得先看的外部信号源。",
|
||
"project_object_anchors": ["实现产能警告触发服务", "告警接收链路"],
|
||
"signal_source_objectives": [
|
||
"设计产能监控逻辑与数据采集：编写脚本收集测试队列与资源使用数据",
|
||
"实现产能警告触发服务：实时比对数据与阈值",
|
||
],
|
||
},
|
||
"adapter_attempts": [{"adapter": "live_search", "status": "success"}],
|
||
"verified_evidence": [
|
||
{
|
||
"source_type": "url",
|
||
"source_label": "FineReport",
|
||
"title": "MES 实时监控",
|
||
"target_url": "https://example.com/mes",
|
||
"snippet": "MES 可用于实时监控设备与产线状态。",
|
||
}
|
||
],
|
||
"verifier": {"ok": True},
|
||
"uncertainties": [],
|
||
"retrieval_status": "evidence_found",
|
||
"impact_status": "indirect_only",
|
||
},
|
||
thinking_mode="normal",
|
||
)
|
||
|
||
assert result["prompt_variant_id"] == "external_signal_source_recommendation"
|
||
assert "设计产能监控逻辑与数据采集" in result["answer_text"]
|
||
assert "实现产能警告触发服务" in result["answer_text"]
|
||
assert "73%" not in result["answer_text"]
|
||
assert "无需额外对接验证" not in result["answer_text"]
|
||
assert "测试执行结果或失败告警" not in result["answer_text"]
|
||
assert fake_llm.structured_calls == 1
|