575 lines
19 KiB
Python
575 lines
19 KiB
Python
import pytest
|
|
from types import SimpleNamespace
|
|
|
|
from app.config.ai_prompts import PROMPT_AI_PMO_QA
|
|
from app.services.adaptive_rag_service.mixin_answer_entry import AdaptiveRAGAnswerEntryMixin
|
|
from app.services.adaptive_rag_service.mixin_answer_mainline import AdaptiveRAGAnswerMainlineMixin
|
|
from app.services.adaptive_rag_service.mixin_deep_guard_answer import AdaptiveRAGDeepGuardAnswerMixin
|
|
from app.services.adaptive_rag_service.mixin_deep_guard_rules import AdaptiveRAGDeepGuardRulesMixin
|
|
from app.services.adaptive_rag_service.mixin_deterministic_query_core import (
|
|
AdaptiveRAGDeterministicQueryCoreMixin,
|
|
)
|
|
from app.services.adaptive_rag_service.mixin_retrieval_vector import AdaptiveRAGRetrievalVectorMixin
|
|
from app.services.adaptive_rag_service.shared_part_02 import DEEP_RETRIEVAL_STEPS, DEEP_VECTOR_TOP_K
|
|
|
|
|
|
class _FakeAdaptiveRAGService(
|
|
AdaptiveRAGAnswerEntryMixin,
|
|
AdaptiveRAGAnswerMainlineMixin,
|
|
):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 9
|
|
self.ack_calls = 0
|
|
self.base_calls = 0
|
|
self.base_kwargs: dict[str, object] = {}
|
|
|
|
def _normalize_thinking_mode(self, thinking_mode: str) -> str:
|
|
return "deep" if str(thinking_mode or "").strip().lower() == "deep" else "normal"
|
|
|
|
async def _prepare_answer_runtime(self, **kwargs):
|
|
clean_q = str(kwargs.get("clean_q") or "").strip()
|
|
action_mode = str(kwargs.get("action_mode") or "").strip().lower()
|
|
return {
|
|
"session_id": 101,
|
|
"turn_index": 2,
|
|
"context_session_id": 101,
|
|
"conversation": [],
|
|
"state_row": None,
|
|
"resolved_question": clean_q,
|
|
"effective_q": clean_q,
|
|
"rewrite_meta": {"mode": "direct"},
|
|
"query_type": "explain",
|
|
"effective_action_mode": action_mode,
|
|
"anchor_resolution": None,
|
|
}
|
|
|
|
async def _handle_ack_first_requirement(self, **kwargs):
|
|
self.ack_calls += 1
|
|
_ = kwargs
|
|
return {"answer": "ack-required", "processing_stage": "ack_only"}
|
|
|
|
async def _execute_base_answer_mainline(self, **kwargs):
|
|
self.base_calls += 1
|
|
self.base_kwargs = dict(kwargs)
|
|
return {"answer": "base-answer", "processing_stage": "qa"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_answer_entry_read_mainline_skips_requirement_ack() -> None:
|
|
service = _FakeAdaptiveRAGService()
|
|
|
|
result = await service.answer_question(
|
|
question="现在有哪些阻塞？",
|
|
asked_by=1,
|
|
asked_via="pmflow_manual",
|
|
action_mode="read",
|
|
sync_requirement=True,
|
|
)
|
|
|
|
assert result["processing_stage"] == "qa"
|
|
assert service.ack_calls == 0
|
|
assert service.base_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_answer_entry_write_mode_uses_requirement_ack_gate() -> None:
|
|
service = _FakeAdaptiveRAGService()
|
|
|
|
result = await service.answer_question(
|
|
question="请同步写入一条风险",
|
|
asked_by=1,
|
|
asked_via="pmflow_manual",
|
|
action_mode="write",
|
|
sync_requirement=True,
|
|
)
|
|
|
|
assert result["processing_stage"] == "ack_only"
|
|
assert service.ack_calls == 1
|
|
assert service.base_calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_answer_entry_high_accuracy_channel_respects_requested_normal_mode() -> None:
|
|
service = _FakeAdaptiveRAGService()
|
|
|
|
result = await service.answer_question(
|
|
question="最近有哪些项目？",
|
|
asked_by=1,
|
|
asked_via="pmflow",
|
|
thinking_mode="normal",
|
|
sync_requirement=True,
|
|
)
|
|
|
|
assert result["processing_stage"] == "qa"
|
|
assert service.ack_calls == 0
|
|
assert service.base_calls == 1
|
|
assert service.base_kwargs["active_thinking_mode"] == "normal"
|
|
assert service.base_kwargs["deep_thinking"] is False
|
|
assert int(service.base_kwargs["deep_retrieval_steps"]) == DEEP_RETRIEVAL_STEPS
|
|
assert int(service.base_kwargs["deep_vector_top_k"]) == DEEP_VECTOR_TOP_K
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_answer_entry_forwards_consultant_mode_preference_to_mainline() -> None:
|
|
service = _FakeAdaptiveRAGService()
|
|
|
|
result = await service.answer_question(
|
|
question="这个方案怎么做更稳妥？",
|
|
asked_by=1,
|
|
asked_via="pmflow",
|
|
thinking_mode="normal",
|
|
consultant_mode_preference="advise",
|
|
)
|
|
|
|
assert result["processing_stage"] == "qa"
|
|
assert service.base_calls == 1
|
|
assert service.base_kwargs["consultant_mode_preference"] == "advise"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_answer_entry_mixed_request_skips_requirement_ack_gate() -> None:
|
|
service = _FakeAdaptiveRAGService()
|
|
|
|
result = await service.answer_question(
|
|
question="现在进展如何，并且新增任务确认流",
|
|
asked_by=1,
|
|
asked_via="pmflow_manual",
|
|
action_mode="read_write",
|
|
sync_requirement=True,
|
|
message_intent="mixed",
|
|
)
|
|
|
|
assert result["processing_stage"] == "qa"
|
|
assert service.ack_calls == 0
|
|
assert service.base_calls == 1
|
|
assert service.base_kwargs["active_thinking_mode"] == "normal"
|
|
assert service.base_kwargs["deep_thinking"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_answer_entry_feishu_write_mode_still_goes_base_mainline() -> None:
|
|
service = _FakeAdaptiveRAGService()
|
|
|
|
result = await service.answer_question(
|
|
question="请在飞书里补一条风险",
|
|
asked_by=1,
|
|
asked_via="feishu",
|
|
action_mode="write",
|
|
sync_requirement=True,
|
|
)
|
|
|
|
assert result["processing_stage"] == "qa"
|
|
assert service.ack_calls == 0
|
|
assert service.base_calls == 1
|
|
assert service.base_kwargs["active_thinking_mode"] == "normal"
|
|
assert service.base_kwargs["deep_thinking"] is False
|
|
assert int(service.base_kwargs["deep_retrieval_steps"]) == DEEP_RETRIEVAL_STEPS
|
|
assert int(service.base_kwargs["deep_vector_top_k"]) == DEEP_VECTOR_TOP_K
|
|
|
|
|
|
def test_query_runtime_planner_prefers_fast_path_for_list_question_even_if_requested_deep() -> None:
|
|
class _FakePlannerService(AdaptiveRAGAnswerMainlineMixin):
|
|
pass
|
|
|
|
service = _FakePlannerService()
|
|
decision = service._plan_query_runtime(
|
|
question="现在有哪些项目？",
|
|
effective_question="现在有哪些项目？",
|
|
query_type="list",
|
|
requested_thinking_mode="deep",
|
|
enforce_scene_prompt_only=False,
|
|
)
|
|
|
|
assert decision.answer_mode == "fast_path"
|
|
assert decision.thinking_mode == "normal"
|
|
assert decision.allow_fast_path is True
|
|
|
|
|
|
def test_query_runtime_planner_bypasses_fast_path_when_consultant_mode_preference_present() -> None:
|
|
class _FakePlannerService(AdaptiveRAGAnswerMainlineMixin):
|
|
pass
|
|
|
|
service = _FakePlannerService()
|
|
decision = service._plan_query_runtime(
|
|
question="现在有哪些项目？",
|
|
effective_question="现在有哪些项目？",
|
|
query_type="list",
|
|
requested_thinking_mode="normal",
|
|
enforce_scene_prompt_only=False,
|
|
consultant_mode_preference="advise",
|
|
)
|
|
|
|
assert decision.answer_mode == "retrieval_normal"
|
|
assert decision.thinking_mode == "normal"
|
|
assert decision.allow_fast_path is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deterministic_query_prefers_project_created_at_answer() -> None:
|
|
calls: list[str] = []
|
|
|
|
class _FakeDeterministicService(AdaptiveRAGDeterministicQueryCoreMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
|
|
async def _try_fast_project_created_at_answer(self, *, clean_q: str, project_id):
|
|
calls.append("created_at")
|
|
_ = (clean_q, project_id)
|
|
return {"answer": "创建于 2026-03-01 09:30。", "sources": [], "missing_fields": []}
|
|
|
|
async def _try_fast_project_progress_answer(self, *, clean_q: str, project_id):
|
|
calls.append("progress")
|
|
_ = (clean_q, project_id)
|
|
return {"answer": "进度回答", "sources": []}
|
|
|
|
async def _try_fast_project_existence_answer(self, *, clean_q: str, project_id):
|
|
calls.append("existence")
|
|
_ = (clean_q, project_id)
|
|
return {"answer": "存在性回答", "sources": []}
|
|
|
|
service = _FakeDeterministicService()
|
|
result = await service._run_deterministic_query(
|
|
question="他是什么时候创建的？",
|
|
project_id=7,
|
|
query_type="status",
|
|
)
|
|
|
|
assert result is not None
|
|
assert "创建于 2026-03-01 09:30" in str(result["answer"])
|
|
assert calls[0] == "created_at"
|
|
|
|
|
|
def test_query_type_classifies_colloquial_list_request_as_list() -> None:
|
|
service = AdaptiveRAGDeterministicQueryCoreMixin()
|
|
|
|
assert service._classify_query_type("我感觉很多重复的项目在系统，你能列一下吗？") == "list"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_vector_retrieval_skips_legacy_fallback_for_non_postgresql_session() -> None:
|
|
class _FakeLLM:
|
|
async def get_embedding_async(self, question: str, **kwargs):
|
|
_ = (question, kwargs)
|
|
return [0.1, 0.2, 0.3]
|
|
|
|
class _FakeService(AdaptiveRAGRetrievalVectorMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
self.llm = _FakeLLM()
|
|
self.legacy_called = False
|
|
|
|
def _map_domains_to_source_types(self, domains: list[str]) -> list[str]:
|
|
return list(domains)
|
|
|
|
def _is_postgresql_session(self) -> bool:
|
|
return False
|
|
|
|
async def _retrieve_vector_context_legacy(self, **kwargs):
|
|
_ = kwargs
|
|
self.legacy_called = True
|
|
return ["legacy-hit"]
|
|
|
|
service = _FakeService()
|
|
result = await service._retrieve_vector_context(
|
|
"查询项目文档",
|
|
domains=["doc"],
|
|
project_id=None,
|
|
source_chat_id=None,
|
|
)
|
|
|
|
assert result == []
|
|
assert service.legacy_called is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_vector_retrieval_skips_legacy_fallback_when_pg_query_errors() -> None:
|
|
class _FakeLLM:
|
|
async def get_embedding_async(self, question: str, **kwargs):
|
|
_ = (question, kwargs)
|
|
return [0.1, 0.2, 0.3]
|
|
|
|
class _FakeService(AdaptiveRAGRetrievalVectorMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
self.llm = _FakeLLM()
|
|
self.legacy_called = False
|
|
|
|
def _map_domains_to_source_types(self, domains: list[str]) -> list[str]:
|
|
return list(domains)
|
|
|
|
def _is_postgresql_session(self) -> bool:
|
|
return True
|
|
|
|
async def _retrieve_vector_context_pg(self, **kwargs):
|
|
_ = kwargs
|
|
return None
|
|
|
|
async def _retrieve_vector_context_legacy(self, **kwargs):
|
|
_ = kwargs
|
|
self.legacy_called = True
|
|
return ["legacy-hit"]
|
|
|
|
service = _FakeService()
|
|
result = await service._retrieve_vector_context(
|
|
"查询项目文档",
|
|
domains=["doc"],
|
|
project_id=None,
|
|
source_chat_id=None,
|
|
)
|
|
|
|
assert result == []
|
|
assert service.legacy_called is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deep_guard_answer_raises_on_empty_llm_response() -> None:
|
|
class _FakeLLM:
|
|
async def generate(self, prompt: str, **kwargs):
|
|
_ = (prompt, kwargs)
|
|
return ""
|
|
|
|
class _FakeService(AdaptiveRAGDeepGuardAnswerMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
self.llm = _FakeLLM()
|
|
|
|
def _normalize_thinking_mode(self, thinking_mode: str) -> str:
|
|
return "normal" if str(thinking_mode or "").strip().lower() != "deep" else "deep"
|
|
|
|
async def get_project_manifest(self, project_id):
|
|
_ = project_id
|
|
return {}
|
|
|
|
async def _select_scene_prompt(self, **kwargs):
|
|
_ = kwargs
|
|
return "external_trend", "请回答问题"
|
|
|
|
def _enforce_feature_evidence_in_answer(self, *, question: str, answer_text: str, retrieved, system_snapshot):
|
|
_ = (question, retrieved, system_snapshot)
|
|
return answer_text
|
|
|
|
service = _FakeService()
|
|
|
|
with pytest.raises(ValueError, match="adaptive_rag_answer_empty_response"):
|
|
await service._generate_answer(
|
|
question="最近AI行业有什么动态",
|
|
retrieved=[],
|
|
conversation=[],
|
|
project_id=None,
|
|
asked_via="feishu",
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deep_guard_answer_prefers_member_workload_direct_answer() -> None:
|
|
class _FakeResult:
|
|
def __init__(self, rows=None, scalar=None):
|
|
self._rows = rows or []
|
|
self._scalar = scalar
|
|
|
|
def all(self):
|
|
return self._rows
|
|
|
|
def first(self):
|
|
return self._rows[0] if self._rows else None
|
|
|
|
def scalar_one_or_none(self):
|
|
return self._scalar
|
|
|
|
class _FakeDB:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
async def execute(self, stmt, params=None): # noqa: ANN001
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return _FakeResult(rows=[SimpleNamespace(id=24, name="杨宇", username="yangyu")])
|
|
return _FakeResult(rows=[(265.56, 95.6, 60.0)])
|
|
|
|
class _FakeLLM:
|
|
async def generate(self, prompt: str, **kwargs): # pragma: no cover - should not be called
|
|
raise AssertionError("LLM should not be called for direct member workload answer")
|
|
|
|
class _FakeService(AdaptiveRAGDeepGuardAnswerMixin):
|
|
def __init__(self) -> None:
|
|
self.db = _FakeDB()
|
|
self.tenant_id = 1
|
|
self.llm = _FakeLLM()
|
|
|
|
def _normalize_thinking_mode(self, thinking_mode: str) -> str:
|
|
return "normal"
|
|
|
|
async def get_project_manifest(self, project_id):
|
|
_ = project_id
|
|
return {}
|
|
|
|
def _select_scene_prompt(self, **kwargs):
|
|
_ = kwargs
|
|
return "member_workload", "member workload prompt"
|
|
|
|
def _enforce_feature_evidence_in_answer(self, *, question: str, answer_text: str, retrieved, system_snapshot):
|
|
_ = (question, retrieved, system_snapshot)
|
|
return answer_text
|
|
|
|
service = _FakeService()
|
|
result = await service._generate_answer(
|
|
question="杨宇现在负载情况如何？",
|
|
retrieved=[],
|
|
conversation=[],
|
|
project_id=None,
|
|
asked_via="feishu",
|
|
)
|
|
|
|
assert "杨宇当前负载265.56%" in result
|
|
assert "严重超载" in result
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_scene_prompt_applies_consultant_mode_preference_wrapper() -> None:
|
|
class _FakeService(AdaptiveRAGDeepGuardAnswerMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
|
|
async def _render_scene_prompt(self, *, prompt_key: str, fallback_template: str, **kwargs):
|
|
_ = (fallback_template, kwargs)
|
|
return f"PROMPT::{prompt_key}"
|
|
|
|
service = _FakeService()
|
|
|
|
scene_key, prompt = await service._select_scene_prompt(
|
|
question="这个方案怎么做更稳妥？",
|
|
project_id=7,
|
|
asked_via="pmflow",
|
|
message_intent=None,
|
|
mixed_signal_part=None,
|
|
conversation_text="无",
|
|
context_blocks_text="无",
|
|
system_snapshot={},
|
|
user_snapshot={},
|
|
system_snapshot_text="{}",
|
|
user_snapshot_text="{}",
|
|
relevance_max=0.61,
|
|
query_type="explain",
|
|
consultant_mode_preference="advise",
|
|
)
|
|
|
|
assert scene_key == "default_consultant_advise"
|
|
assert "PROMPT::PROMPT_RAG_ANSWER" in prompt
|
|
assert "稳妥建议" in prompt
|
|
|
|
|
|
def test_ai_pmo_qa_prompt_includes_five_lenses_and_runtime_state_guard() -> None:
|
|
assert "事实源在哪" in PROMPT_AI_PMO_QA
|
|
assert "输出模板在哪" in PROMPT_AI_PMO_QA
|
|
assert "检索范围在哪" in PROMPT_AI_PMO_QA
|
|
assert "执行器在哪" in PROMPT_AI_PMO_QA
|
|
assert "反馈回流在哪" in PROMPT_AI_PMO_QA
|
|
assert "静态资料" in PROMPT_AI_PMO_QA
|
|
assert "运行态配置" in PROMPT_AI_PMO_QA
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_scene_prompt_routes_business_review_to_ai_pmo_prompt() -> None:
|
|
class _FakeService(AdaptiveRAGDeepGuardAnswerMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
|
|
async def _render_scene_prompt(self, *, prompt_key: str, fallback_template: str, **kwargs):
|
|
_ = fallback_template
|
|
return (
|
|
f"PROMPT::{prompt_key}|"
|
|
f"business_review={kwargs.get('business_review_mode')}|"
|
|
f"framework={kwargs.get('review_framework')}"
|
|
)
|
|
|
|
service = _FakeService()
|
|
|
|
scene_key, prompt = await service._select_scene_prompt(
|
|
question="我要做一个大模型 API 聚合平台，帮我分析下要怎么做",
|
|
project_id=None,
|
|
asked_via="feishu",
|
|
message_intent="business_review",
|
|
mixed_signal_part=None,
|
|
conversation_text="用户上一轮提到要做 API 聚合平台",
|
|
context_blocks_text="无",
|
|
system_snapshot={},
|
|
user_snapshot={},
|
|
system_snapshot_text="{}",
|
|
user_snapshot_text="{}",
|
|
relevance_max=0.51,
|
|
query_type="explain",
|
|
consultant_mode_preference="clarify",
|
|
)
|
|
|
|
assert scene_key == "ai_pmo_business_review_consultant_clarify"
|
|
assert "PROMPT::PROMPT_AI_PMO_QA" in prompt
|
|
assert "business_review=true" in prompt
|
|
assert "framework=five_lenses" in prompt
|
|
|
|
|
|
def test_runtime_state_query_guard_blocks_static_catalog_evidence() -> None:
|
|
class _FakeService(AdaptiveRAGDeepGuardRulesMixin):
|
|
pass
|
|
|
|
service = _FakeService()
|
|
blocked = service._should_block_answer_with_guard(
|
|
question="高级模型路由是否启用？",
|
|
effective_question="高级模型路由是否启用？",
|
|
sources=[
|
|
{"source": "feature_catalog", "label": "Feature Catalog", "metadata": {}},
|
|
{"source": "graph", "label": "Knowledge Graph", "metadata": {"path": "1->uses->2"}},
|
|
],
|
|
relevance_max=0.96,
|
|
)
|
|
|
|
assert blocked is True
|
|
assert "无法仅凭静态资料确认" in service._build_evidence_guard_answer("高级模型路由是否启用？")
|
|
|
|
|
|
def test_runtime_state_query_guard_allows_routing_config_evidence() -> None:
|
|
class _FakeService(AdaptiveRAGDeepGuardRulesMixin):
|
|
pass
|
|
|
|
service = _FakeService()
|
|
blocked = service._should_block_answer_with_guard(
|
|
question="高级模型路由是否启用？",
|
|
effective_question="高级模型路由是否启用？",
|
|
sources=[
|
|
{
|
|
"source": "doc",
|
|
"label": "Routing Config",
|
|
"metadata": {"source_id": "backend/.openclaw/pmflow-routing-rules.json"},
|
|
}
|
|
],
|
|
relevance_max=0.74,
|
|
)
|
|
|
|
assert blocked is False
|
|
|
|
|
|
def test_correction_escalation_query_forces_deep_reasoning_in_feishu_channel() -> None:
|
|
class _FakeService(AdaptiveRAGDeepGuardRulesMixin):
|
|
pass
|
|
|
|
service = _FakeService()
|
|
reason = service._forced_deep_reason(
|
|
question="不对，调用高级模型来回答我的问题",
|
|
effective_question="不对，调用高级模型来回答我的问题",
|
|
conversation=[{"question": "我要做一个大模型 API 聚合平台", "answer": "上一轮回答"}],
|
|
)
|
|
|
|
assert reason == "correction_escalation"
|
|
assert service._should_force_deep_for_channel(
|
|
asked_via="feishu",
|
|
question="不对，调用高级模型来回答我的问题",
|
|
effective_question="不对，调用高级模型来回答我的问题",
|
|
reason=reason,
|
|
) is True
|