449 lines
18 KiB
Python
449 lines
18 KiB
Python
from __future__ import annotations
|
||
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
import pytest_asyncio
|
||
from sqlalchemy import JSON
|
||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||
|
||
from app.models.base import Base
|
||
from app.models.integration import Integration
|
||
from app.services.adaptive_rag_service.mixin_answer_retrieval import AdaptiveRAGAnswerRetrievalMixin
|
||
from app.services.adaptive_rag_service.mixin_deep_guard_rules import AdaptiveRAGDeepGuardRulesMixin
|
||
from app.services.adaptive_rag_service.shared_part_02 import RetrievedItem
|
||
from app.services.external_query_service.adapters import MCPExternalAdapter
|
||
from app.services.external_query_service.gateway import ExternalQueryGateway
|
||
from app.services.external_query_service.registry import ExternalProviderRegistry
|
||
from app.services.external_query_service.schemas import ExternalAdapterResult, ExternalEvidenceItem, ExternalQueryPlan
|
||
from app.services.mcp_config import McpConfigService
|
||
from app.services.mcp_config.runtime import MicrosoftLearnMCPRuntime
|
||
|
||
|
||
def _use_sqlite_json_types(*tables) -> None:
|
||
for table in tables:
|
||
for column in table.columns:
|
||
if column.type.__class__.__name__ in {"JSONB", "ARRAY", "Vector"}:
|
||
column.type = JSON()
|
||
|
||
|
||
@pytest_asyncio.fixture
|
||
async def db_session():
|
||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||
tables = [Integration.__table__]
|
||
_use_sqlite_json_types(*tables)
|
||
async with engine.begin() as conn:
|
||
await conn.run_sync(lambda sync_conn: Base.metadata.create_all(sync_conn, tables=tables))
|
||
maker = async_sessionmaker(engine, expire_on_commit=False)
|
||
async with maker() as session:
|
||
yield session
|
||
await engine.dispose()
|
||
|
||
|
||
async def _seed_backend_direct_server(db_session) -> None:
|
||
service = McpConfigService(db_session, tenant_id=1)
|
||
await service.create_server(
|
||
name="microsoft-learn",
|
||
display_name="Microsoft Learn MCP",
|
||
template_id="microsoft_learn",
|
||
params={},
|
||
)
|
||
await db_session.commit()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mcp_config_service_search_external_uses_backend_direct_server(db_session, monkeypatch) -> None:
|
||
await _seed_backend_direct_server(db_session)
|
||
|
||
async def _fake_search(self, *, query: str, result_limit: int = 3): # noqa: ANN001
|
||
_ = (self, query, result_limit)
|
||
return [
|
||
ExternalEvidenceItem(
|
||
source_type="mcp_search",
|
||
source_label="Microsoft Learn MCP",
|
||
title="Azure OpenAI quickstart",
|
||
target_url="https://learn.microsoft.com/azure/ai-foundry/openai/chatgpt-quickstart",
|
||
snippet="Official Microsoft quickstart.",
|
||
retrieved_at="2026-04-15T12:30:00+08:00",
|
||
freshness_score=0.9,
|
||
authority_score=0.98,
|
||
grounded_fields=["microsoft_docs_search"],
|
||
raw_metadata={"provider": "microsoft_learn_mcp"},
|
||
)
|
||
]
|
||
|
||
monkeypatch.setattr(MicrosoftLearnMCPRuntime, "search", _fake_search)
|
||
|
||
evidence = await McpConfigService(db_session, tenant_id=1).search_external(
|
||
query="azure openai quickstart",
|
||
result_limit=3,
|
||
)
|
||
|
||
assert len(evidence) == 1
|
||
assert evidence[0].source_type == "mcp_search"
|
||
assert evidence[0].target_url == "https://learn.microsoft.com/azure/ai-foundry/openai/chatgpt-quickstart"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_external_query_gateway_prefers_backend_direct_mcp_when_available(db_session, monkeypatch) -> None:
|
||
await _seed_backend_direct_server(db_session)
|
||
|
||
async def _fake_search_external(self, *, query: str, result_limit: int = 3): # noqa: ANN001
|
||
_ = (self, query, result_limit)
|
||
return [
|
||
ExternalEvidenceItem(
|
||
source_type="mcp_search",
|
||
source_label="Microsoft Learn MCP",
|
||
title="Azure OpenAI quickstart",
|
||
target_url="https://learn.microsoft.com/azure/ai-foundry/openai/chatgpt-quickstart",
|
||
snippet="Official Microsoft quickstart.",
|
||
retrieved_at="2026-04-15T12:30:00+08:00",
|
||
freshness_score=0.9,
|
||
authority_score=0.98,
|
||
grounded_fields=["microsoft_docs_search"],
|
||
raw_metadata={"provider": "microsoft_learn_mcp"},
|
||
)
|
||
]
|
||
|
||
monkeypatch.setattr(McpConfigService, "search_external", _fake_search_external)
|
||
|
||
class _FakeService(AdaptiveRAGAnswerRetrievalMixin, AdaptiveRAGDeepGuardRulesMixin):
|
||
def __init__(self, db):
|
||
self.db = db
|
||
self.tenant_id = 1
|
||
self.llm = None
|
||
|
||
service = _FakeService(db_session)
|
||
|
||
rows, meta = await service._execute_external_query_gateway(
|
||
question="Azure OpenAI quickstart",
|
||
query_semantics={},
|
||
result_limit=3,
|
||
max_pages=1,
|
||
)
|
||
|
||
assert len(rows) == 1
|
||
assert isinstance(rows[0], RetrievedItem)
|
||
assert rows[0].source == "mcp_search"
|
||
assert rows[0].metadata["external_adapter"] == "mcp"
|
||
assert meta["adapter"] == "mcp"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_external_query_gateway_falls_back_when_mcp_evidence_is_off_topic() -> None:
|
||
plan = ExternalQueryPlan(
|
||
plan_version="v1",
|
||
query_kind="weather",
|
||
answer_contract="summary_with_sources",
|
||
freshness_required=True,
|
||
topic="广州未来5天天气",
|
||
subject_entities=["广州"],
|
||
location_hint="广州",
|
||
time_range_hint="未来5天",
|
||
preferred_domains=[],
|
||
search_queries=["广州 未来5天天气"],
|
||
result_limit=3,
|
||
requires_multi_source=False,
|
||
clarify_on_ambiguity=False,
|
||
)
|
||
|
||
class _Planner:
|
||
async def build_plan(self, *, question: str, query_semantics: dict | None = None):
|
||
_ = (question, query_semantics)
|
||
return plan, {"planner": "test"}
|
||
|
||
class _MCPAdapter:
|
||
adapter_name = "mcp"
|
||
|
||
async def execute(self, current_plan: ExternalQueryPlan) -> ExternalAdapterResult:
|
||
_ = current_plan
|
||
return ExternalAdapterResult(
|
||
adapter_name="mcp",
|
||
status="success",
|
||
evidence=[
|
||
ExternalEvidenceItem(
|
||
source_type="mcp_search",
|
||
source_label="Microsoft Learn MCP",
|
||
title="Azure Maps Weather services",
|
||
target_url="https://learn.microsoft.com/azure/azure-maps/weather-services-concepts",
|
||
snippet="Learn how Azure Maps Weather APIs expose forecast data models and request options.",
|
||
retrieved_at="2026-04-15T12:30:00+08:00",
|
||
freshness_score=0.82,
|
||
authority_score=0.98,
|
||
grounded_fields=["microsoft_docs_search"],
|
||
raw_metadata={"provider": "microsoft_learn_mcp"},
|
||
)
|
||
],
|
||
meta={"provider": "mcp"},
|
||
)
|
||
|
||
class _LiveSearchAdapter:
|
||
adapter_name = "live_search"
|
||
|
||
async def execute(self, current_plan: ExternalQueryPlan) -> ExternalAdapterResult:
|
||
_ = current_plan
|
||
return ExternalAdapterResult(
|
||
adapter_name="live_search",
|
||
status="success",
|
||
evidence=[
|
||
ExternalEvidenceItem(
|
||
source_type="url",
|
||
source_label="Weather Example",
|
||
title="广州未来5天天气预报",
|
||
target_url="https://weather.example/guangzhou-5day",
|
||
snippet="广州未来5天有阵雨转多云，最高温31度，最低温24度。",
|
||
retrieved_at="2026-04-15T12:35:00+08:00",
|
||
freshness_score=0.91,
|
||
authority_score=0.76,
|
||
grounded_fields=["forecast"],
|
||
raw_metadata={"provider": "live_search"},
|
||
)
|
||
],
|
||
meta={"provider": "live_search"},
|
||
)
|
||
|
||
execution = await ExternalQueryGateway(
|
||
planner=_Planner(),
|
||
registry=ExternalProviderRegistry(adapters=[_MCPAdapter(), _LiveSearchAdapter()]),
|
||
).execute(
|
||
question="帮我看看未来5天的广州天气如何",
|
||
query_semantics={"query_profile": {"needs_freshness": True}},
|
||
)
|
||
|
||
assert execution.adapter_name == "live_search"
|
||
assert execution.meta["fallback_used"] is True
|
||
assert execution.verifier["ok"] is True
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_mcp_adapter_fetches_target_page_content_after_search_result() -> None:
|
||
plan = ExternalQueryPlan(
|
||
plan_version="v1",
|
||
query_kind="weather",
|
||
answer_contract="summary_with_sources",
|
||
freshness_required=True,
|
||
topic="湛江市未来五天天气",
|
||
subject_entities=["湛江市"],
|
||
location_hint="湛江市",
|
||
time_range_hint="未来5天",
|
||
preferred_domains=["weather.example"],
|
||
search_queries=["湛江市未来五天天气预报"],
|
||
result_limit=3,
|
||
requires_multi_source=False,
|
||
clarify_on_ambiguity=False,
|
||
)
|
||
|
||
class _Planner:
|
||
async def build_plan(self, *, question: str, query_semantics: dict | None = None):
|
||
_ = (question, query_semantics)
|
||
return plan, {"planner": "test"}
|
||
|
||
async def _search_callable(current_plan: ExternalQueryPlan) -> ExternalAdapterResult:
|
||
_ = current_plan
|
||
return ExternalAdapterResult(
|
||
adapter_name="mcp",
|
||
status="success",
|
||
evidence=[
|
||
ExternalEvidenceItem(
|
||
source_type="mcp_search",
|
||
source_label="Weather MCP",
|
||
title="Forecast detail page",
|
||
target_url="https://weather.example/forecast/zhanjiang-five-day",
|
||
snippet="Landing page",
|
||
retrieved_at="2026-04-16T01:05:00+08:00",
|
||
freshness_score=0.6,
|
||
authority_score=0.86,
|
||
grounded_fields=["mcp_search"],
|
||
raw_metadata={"provider": "weather_mcp"},
|
||
)
|
||
],
|
||
meta={"provider": "weather_mcp"},
|
||
)
|
||
|
||
async def _fetch_text(url: str) -> str:
|
||
assert url == "https://weather.example/forecast/zhanjiang-five-day"
|
||
return "页面说明。湛江未来5天有阵雨转多云，最高温33度，最低温26度。末尾。"
|
||
|
||
def _chunk_text(text: str, *, chunk_size: int = 900, overlap: int = 120) -> list[str]:
|
||
_ = (text, chunk_size, overlap)
|
||
return ["页面说明。", "湛江未来5天有阵雨转多云，最高温33度，最低温26度。"]
|
||
|
||
def _score_chunk(*, question: str, title: str, content: str, url: str) -> float:
|
||
_ = (question, title, url)
|
||
return 0.94 if "湛江未来5天" in content else 0.08
|
||
|
||
execution = await ExternalQueryGateway(
|
||
planner=_Planner(),
|
||
registry=ExternalProviderRegistry(
|
||
adapters=[
|
||
MCPExternalAdapter(
|
||
search_callable=_search_callable,
|
||
fetch_text_callable=_fetch_text,
|
||
chunk_text_callable=_chunk_text,
|
||
score_chunk_callable=_score_chunk,
|
||
max_pages=1,
|
||
)
|
||
]
|
||
),
|
||
).execute(
|
||
question="湛江市未来五天的天气",
|
||
query_semantics={"query_profile": {"needs_freshness": True}},
|
||
)
|
||
|
||
assert execution.adapter_name == "mcp"
|
||
assert execution.verifier["ok"] is True
|
||
assert execution.evidence[0].snippet == "湛江未来5天有阵雨转多云，最高温33度，最低温26度。"
|
||
assert execution.evidence[0].raw_metadata["search_snippet"] == "Landing page"
|
||
assert "page_content_snippet" in execution.evidence[0].grounded_fields
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_external_query_gateway_accepts_bundle_with_normalized_anchor_match() -> None:
|
||
plan = ExternalQueryPlan(
|
||
plan_version="v1",
|
||
query_kind="weather",
|
||
answer_contract="summary_with_sources",
|
||
freshness_required=True,
|
||
topic="湛江市未来五天天气",
|
||
subject_entities=["湛江市"],
|
||
location_hint="湛江市",
|
||
time_range_hint="未来5天",
|
||
preferred_domains=["weather.com.cn", "nmc.cn"],
|
||
search_queries=["湛江市未来五天天气预报"],
|
||
result_limit=3,
|
||
requires_multi_source=False,
|
||
clarify_on_ambiguity=False,
|
||
)
|
||
|
||
class _Planner:
|
||
async def build_plan(self, *, question: str, query_semantics: dict | None = None):
|
||
_ = (question, query_semantics)
|
||
return plan, {"planner": "test"}
|
||
|
||
class _LiveSearchAdapter:
|
||
adapter_name = "live_search"
|
||
|
||
async def execute(self, current_plan: ExternalQueryPlan) -> ExternalAdapterResult:
|
||
_ = current_plan
|
||
return ExternalAdapterResult(
|
||
adapter_name="live_search",
|
||
status="success",
|
||
evidence=[
|
||
ExternalEvidenceItem(
|
||
source_type="url",
|
||
source_label="中国天气网",
|
||
title="湛江天气预报,湛江7天天气预报,湛江15天天气预报,湛江天气查询",
|
||
target_url="https://www.weather.com.cn/weather/101281001.shtml",
|
||
snippet="湛江未来5天有阵雨转多云，最高温33度，最低温26度。",
|
||
retrieved_at="2026-04-16T00:30:00+08:00",
|
||
freshness_score=0.92,
|
||
authority_score=0.95,
|
||
grounded_fields=["forecast"],
|
||
raw_metadata={"provider": "live_search"},
|
||
)
|
||
],
|
||
meta={"provider": "live_search"},
|
||
)
|
||
|
||
execution = await ExternalQueryGateway(
|
||
planner=_Planner(),
|
||
registry=ExternalProviderRegistry(adapters=[_LiveSearchAdapter()]),
|
||
).execute(
|
||
question="湛江市未来五天的天气",
|
||
query_semantics={"query_profile": {"needs_freshness": True}},
|
||
)
|
||
|
||
assert execution.adapter_name == "live_search"
|
||
assert execution.verifier["ok"] is True
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_external_query_gateway_prefers_weather_capable_adapter_before_doc_mcp() -> None:
|
||
plan = ExternalQueryPlan(
|
||
plan_version="v1",
|
||
query_kind="weather",
|
||
answer_contract="summary_with_sources",
|
||
freshness_required=True,
|
||
topic="湛江市未来五天天气",
|
||
subject_entities=["湛江市"],
|
||
location_hint="湛江市",
|
||
time_range_hint="未来5天",
|
||
preferred_domains=["weather.com.cn", "nmc.cn"],
|
||
search_queries=["湛江市未来五天天气预报"],
|
||
result_limit=3,
|
||
requires_multi_source=False,
|
||
clarify_on_ambiguity=False,
|
||
)
|
||
call_order: list[str] = []
|
||
|
||
class _Planner:
|
||
async def build_plan(self, *, question: str, query_semantics: dict | None = None):
|
||
_ = (question, query_semantics)
|
||
return plan, {"planner": "test"}
|
||
|
||
class _MCPAdapter:
|
||
adapter_name = "mcp"
|
||
supported_query_kinds = {"model", "company", "generic_web", "unknown"}
|
||
|
||
async def execute(self, current_plan: ExternalQueryPlan) -> ExternalAdapterResult:
|
||
_ = current_plan
|
||
call_order.append("mcp")
|
||
return ExternalAdapterResult(
|
||
adapter_name="mcp",
|
||
status="success",
|
||
evidence=[
|
||
ExternalEvidenceItem(
|
||
source_type="mcp_search",
|
||
source_label="Microsoft Learn MCP",
|
||
title="Azure Maps Weather service coverage",
|
||
target_url="https://learn.microsoft.com/azure/azure-maps/weather-coverage",
|
||
snippet="Weather service coverage.",
|
||
retrieved_at="2026-04-16T00:30:00+08:00",
|
||
freshness_score=0.82,
|
||
authority_score=0.98,
|
||
grounded_fields=["microsoft_docs_search"],
|
||
raw_metadata={"provider": "mcp"},
|
||
)
|
||
],
|
||
meta={"provider": "mcp"},
|
||
)
|
||
|
||
class _LiveSearchAdapter:
|
||
adapter_name = "live_search"
|
||
supported_query_kinds = {"weather", "news", "market", "generic_web", "unknown"}
|
||
|
||
async def execute(self, current_plan: ExternalQueryPlan) -> ExternalAdapterResult:
|
||
_ = current_plan
|
||
call_order.append("live_search")
|
||
return ExternalAdapterResult(
|
||
adapter_name="live_search",
|
||
status="success",
|
||
evidence=[
|
||
ExternalEvidenceItem(
|
||
source_type="url",
|
||
source_label="中国天气网",
|
||
title="湛江天气预报,湛江7天天气预报,湛江15天天气预报,湛江天气查询",
|
||
target_url="https://www.weather.com.cn/weather/101281001.shtml",
|
||
snippet="湛江未来5天有阵雨转多云，最高温33度，最低温26度。",
|
||
retrieved_at="2026-04-16T00:30:00+08:00",
|
||
freshness_score=0.92,
|
||
authority_score=0.95,
|
||
grounded_fields=["forecast"],
|
||
raw_metadata={"provider": "live_search"},
|
||
)
|
||
],
|
||
meta={"provider": "live_search"},
|
||
)
|
||
|
||
execution = await ExternalQueryGateway(
|
||
planner=_Planner(),
|
||
registry=ExternalProviderRegistry(adapters=[_MCPAdapter(), _LiveSearchAdapter()]),
|
||
).execute(
|
||
question="湛江市未来五天的天气",
|
||
query_semantics={"query_profile": {"needs_freshness": True}},
|
||
)
|
||
|
||
assert call_order == ["live_search"]
|
||
assert execution.adapter_name == "live_search"
|
||
assert execution.verifier["ok"] is True
|