1022 lines
37 KiB
Python
1022 lines
37 KiB
Python
from datetime import datetime, timedelta
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
|
|
from app.api.ai_pmo_api import routes_core
|
|
from app.api.ai_pmo_api.schemas import PMOProcessTextRequest
|
|
from app.services.ai_pmo.ai_pmo_kernel.mixins.intent.ai_pmo_kernel_mixins_intent_rules import (
|
|
KernelIntentRulesMixin,
|
|
)
|
|
from app.services.ai_pmo.ai_pmo_kernel.ai_agent_kernel import gate_execution
|
|
from app.services.ai_pmo.ai_pmo_kernel.ai_pmo_reply_renderer import ReplyRenderer
|
|
from app.services.ai_pmo.ai_pmo_kernel_ingress_flow import mixin_entry
|
|
from app.services.ai_pmo.ai_pmo_kernel_ingress_flow import mixin_prepare_manual
|
|
from app.services.ai_pmo.ai_pmo_typed_executor_execute import mixin_execute_query_project
|
|
from app.services.ai_pmo.ai_pmo_intent import CanonicalIntent
|
|
from app.services.llm_service import ChatCompletionResult, StructuredExtractionResult
|
|
from app.services.llm_service import mixin_tier_escalation
|
|
from app.services.llm_service.mixin_tier_escalation import LLMTierEscalationMixin
|
|
from app.utils.thread_context import build_feishu_thread_key
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_process_text_route_uses_pmflow_manual_ingress(monkeypatch) -> None:
|
|
captured: dict[str, object] = {}
|
|
|
|
class FakeKernel:
|
|
def __init__(self, db, tenant_id: int):
|
|
captured["init_db"] = db
|
|
captured["init_tenant_id"] = tenant_id
|
|
|
|
async def process_ingress(self, **kwargs):
|
|
captured["ingress_kwargs"] = kwargs
|
|
return {
|
|
"handled": True,
|
|
"status": "answered",
|
|
"classification": "question",
|
|
"direct_reply_required": True,
|
|
"direct_reply_text": "当前主链正常。",
|
|
}
|
|
|
|
monkeypatch.setattr(routes_core, "AIPMOKernelService", FakeKernel)
|
|
|
|
request = PMOProcessTextRequest(
|
|
message_text="现在进展怎么样？",
|
|
sender="产品经理A",
|
|
source_message_id="manual-1",
|
|
project_id=11,
|
|
)
|
|
current_user = SimpleNamespace(name="ignored", username="ignored")
|
|
|
|
response = await routes_core.process_text(
|
|
request=request,
|
|
current_user=current_user,
|
|
db=object(),
|
|
tenant_id=5,
|
|
)
|
|
|
|
assert response["code"] == 0
|
|
data = response["data"]
|
|
assert data["status"] == "answered"
|
|
kwargs = captured["ingress_kwargs"]
|
|
assert isinstance(kwargs, dict)
|
|
assert kwargs["source_system"] == "pmflow_manual"
|
|
assert kwargs["source_message_id"] == "manual-1"
|
|
assert kwargs["message_text"] == "现在进展怎么样？"
|
|
assert kwargs["sender"] == "产品经理A"
|
|
assert kwargs["channel_ref"] == "pmflow_manual"
|
|
assert kwargs["chat_type"] == "manual"
|
|
assert kwargs["project_hint_id"] == 11
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_process_text_route_rejects_unhandled_result(monkeypatch) -> None:
|
|
class FakeKernel:
|
|
def __init__(self, db, tenant_id: int):
|
|
_ = (db, tenant_id)
|
|
|
|
async def process_ingress(self, **kwargs):
|
|
_ = kwargs
|
|
return {"handled": False, "reason": "kernel_not_handled"}
|
|
|
|
monkeypatch.setattr(routes_core, "AIPMOKernelService", FakeKernel)
|
|
request = PMOProcessTextRequest(message_text="hello")
|
|
current_user = SimpleNamespace(name="u", username="u")
|
|
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await routes_core.process_text(
|
|
request=request,
|
|
current_user=current_user,
|
|
db=object(),
|
|
tenant_id=1,
|
|
)
|
|
assert exc_info.value.status_code == 400
|
|
assert str(exc_info.value.detail) == "kernel_not_handled"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_mixed_sidecar_context_passes_mixed_contract(monkeypatch) -> None:
|
|
captured: dict[str, object] = {}
|
|
|
|
class FakeAdaptiveRAGService:
|
|
def __init__(self, db, tenant_id: int):
|
|
captured["tenant_id"] = tenant_id
|
|
_ = db
|
|
|
|
async def classify_intent(self, text: str):
|
|
captured["classified_text"] = text
|
|
return {
|
|
"intent": "mixed",
|
|
"question_part": "现在进展如何",
|
|
"statement_part": "新增任务确认流",
|
|
}
|
|
|
|
async def answer_question(self, **kwargs):
|
|
captured["answer_kwargs"] = kwargs
|
|
return {"answer": "[回答] 进展稳定\n[已收到需求] 新增任务确认流"}
|
|
|
|
class FakeKernel(mixin_entry.KernelIngressFlowEntryMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 9
|
|
|
|
monkeypatch.setattr(mixin_entry, "AdaptiveRAGService", FakeAdaptiveRAGService)
|
|
kernel = FakeKernel()
|
|
clean_text, mixed_meta = await kernel._prepare_mixed_sidecar_context(
|
|
source_system="pmflow_manual",
|
|
source_message_id="manual-msg-2",
|
|
channel_ref="pmflow-chat",
|
|
message_text="顺便问下现在进展如何，并且新增任务确认流",
|
|
project_hint_id=22,
|
|
)
|
|
|
|
assert clean_text == "新增任务确认流"
|
|
assert mixed_meta["mixed_intent"] is True
|
|
assert mixed_meta["mixed_question_part"] == "现在进展如何"
|
|
assert mixed_meta["mixed_signal_part"] == "新增任务确认流"
|
|
assert "[回答] 进展稳定" in str(mixed_meta.get("mixed_sidecar_reply") or "")
|
|
answer_kwargs = captured["answer_kwargs"]
|
|
assert isinstance(answer_kwargs, dict)
|
|
assert answer_kwargs["asked_via"] == "pmflow_manual"
|
|
assert answer_kwargs["project_id"] is None
|
|
assert answer_kwargs["message_intent"] == "mixed"
|
|
assert answer_kwargs["mixed_signal_part"] == "新增任务确认流"
|
|
assert answer_kwargs["conversation_key"] == "manual-msg-2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_process_ingress_prioritizes_draft_pending_confirm(monkeypatch) -> None:
|
|
class FakeKernel(mixin_entry.KernelIngressFlowEntryMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 1
|
|
self.draft_called = False
|
|
|
|
async def _prepare_ingress_envelope(self, **kwargs):
|
|
_ = kwargs
|
|
return {
|
|
"early_result": None,
|
|
"clean_text": "确认",
|
|
"synthetic_message_id": "s-msg-1",
|
|
"closure_payload": {},
|
|
"ingest_event": SimpleNamespace(id=100),
|
|
"operator": SimpleNamespace(id=8),
|
|
"work_item": SimpleNamespace(id=55),
|
|
"session": SimpleNamespace(last_intent=mixin_entry.STATE_DRAFT_PENDING_CONFIRM),
|
|
}
|
|
|
|
async def _maybe_handle_draft_confirmation_ingress(self, **kwargs):
|
|
self.draft_called = True
|
|
_ = kwargs
|
|
return {
|
|
"handled": True,
|
|
"status": "answered",
|
|
"classification": "command",
|
|
"direct_reply_required": True,
|
|
"direct_reply_text": "已确认草稿并继续执行。",
|
|
}
|
|
|
|
def _is_short_confirmation_message(self, **kwargs):
|
|
_ = kwargs
|
|
return False
|
|
|
|
async def _prepare_mixed_sidecar_context(self, **kwargs):
|
|
_ = kwargs
|
|
raise AssertionError("draft_pending_confirm flow should bypass mixed sidecar precheck")
|
|
|
|
kernel = FakeKernel()
|
|
result = await kernel.process_ingress(
|
|
source_system="feishu",
|
|
source_message_id="feishu-1",
|
|
message_text="确认",
|
|
sender="tester",
|
|
channel_ref="chat-1",
|
|
source_sender_open_id="ou_xxx",
|
|
ingress_options={"thread_context": {"reply_to_message_id": "msg-root-1"}},
|
|
)
|
|
|
|
assert kernel.draft_called is True
|
|
assert result["status"] == "answered"
|
|
assert "草稿" in result["direct_reply_text"]
|
|
|
|
|
|
def test_should_route_to_draft_confirmation_requires_anchor_or_explicit_id() -> None:
|
|
session = SimpleNamespace(last_intent=mixin_entry.STATE_DRAFT_PENDING_CONFIRM)
|
|
|
|
assert mixin_entry.KernelIngressFlowEntryMixin._should_route_to_draft_confirmation(
|
|
session=session,
|
|
message_text="确认",
|
|
ingress_options=None,
|
|
) is False
|
|
|
|
assert mixin_entry.KernelIngressFlowEntryMixin._should_route_to_draft_confirmation(
|
|
session=session,
|
|
message_text="确认",
|
|
ingress_options={"thread_context": {"thread_id": "th_123"}},
|
|
) is False
|
|
|
|
assert mixin_entry.KernelIngressFlowEntryMixin._should_route_to_draft_confirmation(
|
|
session=session,
|
|
message_text="确认",
|
|
ingress_options={"thread_context": {"root_message_id": "root_123"}},
|
|
) is False
|
|
|
|
assert mixin_entry.KernelIngressFlowEntryMixin._should_route_to_draft_confirmation(
|
|
session=session,
|
|
message_text="创建时间是多少？",
|
|
ingress_options={"thread_context": {"reply_to_message_id": "msg-123"}},
|
|
) is False
|
|
|
|
assert mixin_entry.KernelIngressFlowEntryMixin._should_route_to_draft_confirmation(
|
|
session=session,
|
|
message_text="确认",
|
|
ingress_options={"thread_context": {"reply_to_message_id": "msg-123"}},
|
|
) is True
|
|
|
|
assert mixin_entry.KernelIngressFlowEntryMixin._should_route_to_draft_confirmation(
|
|
session=session,
|
|
message_text="工作项ID: 88 创建时间是多少？",
|
|
ingress_options=None,
|
|
) is True
|
|
|
|
|
|
def test_short_confirmation_requires_reply_anchor() -> None:
|
|
session = SimpleNamespace(
|
|
last_intent=mixin_entry.STATE_DRAFT_PENDING_CONFIRM,
|
|
last_active_at=datetime.now(),
|
|
)
|
|
sent_at = datetime.now() + timedelta(seconds=1)
|
|
|
|
assert (
|
|
KernelIntentRulesMixin._is_short_confirmation_message(
|
|
message_text="确认",
|
|
session=session,
|
|
message_sent_at=sent_at,
|
|
session_snapshot=None,
|
|
reply_anchor_present=False,
|
|
)
|
|
is False
|
|
)
|
|
assert (
|
|
KernelIntentRulesMixin._is_short_confirmation_message(
|
|
message_text="确认",
|
|
session=session,
|
|
message_sent_at=sent_at,
|
|
session_snapshot=None,
|
|
reply_anchor_present=True,
|
|
)
|
|
is True
|
|
)
|
|
|
|
|
|
def test_reply_renderer_formats_project_delete_pending_and_success() -> None:
|
|
renderer = ReplyRenderer(
|
|
state_waiting_manual="waiting_manual",
|
|
state_failed="failed",
|
|
receipt_phase_action="action",
|
|
receipt_phase_pending="pending",
|
|
receipt_phase_error="error",
|
|
)
|
|
intent = CanonicalIntent(
|
|
name="project.delete",
|
|
confidence=0.96,
|
|
project_id=None,
|
|
signal_title="",
|
|
signal_summary="",
|
|
metadata={"llm_parameters": {"project_id": 12, "project_title": "重复项目"}},
|
|
)
|
|
|
|
pending = renderer.render(
|
|
work_state="waiting_manual",
|
|
intent=intent,
|
|
decision_record={"approval_requirement": {"message": "检测到删除或归档类写操作，必须进入显式审批/人工确认。"}},
|
|
execution_result={
|
|
"status": "waiting_manual",
|
|
"action_contract": {
|
|
"operation_code": "project.delete",
|
|
"params": {"project_id": 12, "project_title": "重复项目"},
|
|
},
|
|
},
|
|
)
|
|
assert pending["template_key"] == "pending_manual"
|
|
assert "重复项目" in pending["text"]
|
|
assert "人工确认" in pending["text"]
|
|
|
|
success = renderer.render(
|
|
work_state="succeeded",
|
|
intent=intent,
|
|
decision_record={"approval_requirement": {}},
|
|
execution_result={"status": "succeeded", "operation_code": "project.delete", "project_id": 12},
|
|
)
|
|
assert success["template_key"] == "action_succeeded"
|
|
assert "已删除项目#12《重复项目》" in success["text"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_waiting_manual_passes_execution_result_to_renderer(monkeypatch) -> None:
|
|
captured: dict[str, object] = {}
|
|
|
|
class FakeReviewService:
|
|
async def review_work_item(self, **kwargs):
|
|
_ = kwargs
|
|
return {"verified": False, "issues": ["waiting_manual"], "next_step": "请确认"}
|
|
|
|
class FakeRenderer:
|
|
def render(self, **kwargs):
|
|
captured["render_kwargs"] = kwargs
|
|
return {
|
|
"receipt_phase": "pending",
|
|
"template_key": "pending_manual",
|
|
"payload": {"intent": kwargs["intent"].name},
|
|
"text": "ok",
|
|
}
|
|
|
|
class FakeKernel(mixin_prepare_manual.KernelIngressFlowPrepareManualMixin):
|
|
def __init__(self) -> None:
|
|
class _FakeDB:
|
|
async def commit(self):
|
|
return None
|
|
|
|
self.db = _FakeDB()
|
|
self.tenant_id = 7
|
|
self.review_service = FakeReviewService()
|
|
self.reply_renderer = FakeRenderer()
|
|
|
|
async def _maybe_emit_risk_cadence_snapshot(self, **kwargs):
|
|
_ = kwargs
|
|
|
|
async def _claim_reply_receipt(self, **kwargs):
|
|
_ = kwargs
|
|
return True
|
|
|
|
async def _write_processing_log(self, **kwargs):
|
|
_ = kwargs
|
|
|
|
async def _update_session(self, **kwargs):
|
|
_ = kwargs
|
|
|
|
async def _upsert_operation_execution_ledger(self, **kwargs):
|
|
_ = kwargs
|
|
return None
|
|
|
|
def _extract_execution_operation_code(self, execution_result):
|
|
if not isinstance(execution_result, dict):
|
|
return None
|
|
action_contract = execution_result.get("action_contract") if isinstance(execution_result.get("action_contract"), dict) else {}
|
|
if isinstance(action_contract, dict):
|
|
operation_code = str(action_contract.get("operation_code") or "").strip().lower()
|
|
if operation_code:
|
|
return operation_code
|
|
operation_code = str(execution_result.get("operation_code") or "").strip().lower()
|
|
return operation_code or None
|
|
|
|
def _normalize_execution_ledger_status(self, *, work_item_state, execution_status):
|
|
_ = execution_status
|
|
return "pending_manual" if str(work_item_state or "").strip().lower() == "waiting_manual" else "processing"
|
|
|
|
async def _project_waiting_manual(self, **kwargs):
|
|
captured["project_waiting_manual"] = kwargs
|
|
|
|
def _apply_execution_action_contract(self, **kwargs):
|
|
execution_result = dict(kwargs["execution_result"])
|
|
execution_result["action_contract"] = {
|
|
"operation_code": "project.delete",
|
|
"params": {"project_id": 12, "project_title": "重复项目"},
|
|
}
|
|
return execution_result
|
|
|
|
def _build_capture_result(self, *, work_item, intent, execution_result, render, direct_reply_required, status, classification):
|
|
_ = (work_item, intent, execution_result, render, direct_reply_required, classification)
|
|
return {"status": status, "render": render}
|
|
|
|
kernel = FakeKernel()
|
|
work_item = SimpleNamespace(
|
|
state=None,
|
|
action_bundle=None,
|
|
execution_result=None,
|
|
review_summary=None,
|
|
reply_template_key=None,
|
|
id=99,
|
|
project_id=12,
|
|
signal_id=None,
|
|
intent="project.delete",
|
|
source_message_id="msg-1",
|
|
draft_version=None,
|
|
draft_expires_at=None,
|
|
)
|
|
session = SimpleNamespace()
|
|
ingest_event = SimpleNamespace()
|
|
decision_payload = {
|
|
"policy": {"approval_requirement": {"message": "检测到删除或归档类写操作，必须进入显式审批/人工确认。"}},
|
|
"execution_plan": {"actions": []},
|
|
"simulation": {},
|
|
}
|
|
workflow_route = SimpleNamespace(key="project.delete", action="delete")
|
|
intent = CanonicalIntent(
|
|
name="project.delete",
|
|
confidence=0.94,
|
|
project_id=12,
|
|
signal_title="",
|
|
signal_summary="",
|
|
metadata={"llm_parameters": {"project_id": 12, "project_title": "重复项目"}},
|
|
)
|
|
|
|
result = await kernel._finalize_waiting_manual_ingress(
|
|
work_item=work_item,
|
|
session=session,
|
|
intent=intent,
|
|
world_model={},
|
|
decision_payload=decision_payload,
|
|
workflow_route=workflow_route,
|
|
decision_trace={"reason": "test"},
|
|
source_system="feishu",
|
|
source_message_id="msg-1",
|
|
message_text="删除重复项目",
|
|
sender="tester",
|
|
ingest_event=ingest_event,
|
|
started=0.0,
|
|
)
|
|
|
|
assert result["status"] == "captured"
|
|
render_kwargs = captured["render_kwargs"]
|
|
assert isinstance(render_kwargs, dict)
|
|
assert render_kwargs["execution_result"]["action_contract"]["operation_code"] == "project.delete"
|
|
assert render_kwargs["execution_result"]["action_contract"]["params"]["project_title"] == "重复项目"
|
|
|
|
|
|
def test_gate_execution_ignores_low_confidence_but_keeps_risk_review() -> None:
|
|
low_confidence = gate_execution(confidence=0.02, risk_level="medium")
|
|
assert low_confidence.action == "auto_execute"
|
|
assert low_confidence.auto_execute is True
|
|
assert low_confidence.gate_code == "auto_execute_allowed"
|
|
assert "置信度" not in low_confidence.reason
|
|
|
|
high_risk = gate_execution(confidence=0.02, risk_level="high")
|
|
assert high_risk.action == "auto_execute"
|
|
assert high_risk.auto_execute is True
|
|
assert high_risk.gate_code == "auto_execute_allowed"
|
|
assert "人工确认" not in high_risk.reason
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_query_intent_honors_rag_request_context(monkeypatch) -> None:
|
|
captured: dict[str, object] = {}
|
|
|
|
class FakeAdaptiveRAGService:
|
|
def __init__(self, db, tenant_id: int):
|
|
_ = db
|
|
captured["tenant_id"] = tenant_id
|
|
|
|
async def answer_question(self, **kwargs):
|
|
captured["answer_kwargs"] = kwargs
|
|
return {"answer": "来自统一 ingress 的回答", "session_id": 7}
|
|
|
|
class FakeExecutor(mixin_execute_query_project.ExecuteQueryProjectMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 3
|
|
|
|
monkeypatch.setattr(mixin_execute_query_project, "AdaptiveRAGService", FakeAdaptiveRAGService)
|
|
executor = FakeExecutor()
|
|
intent = SimpleNamespace(
|
|
project_id=12,
|
|
metadata={
|
|
"rag_request_context": {
|
|
"asked_by": 101,
|
|
"asked_via": "pmflow_manual",
|
|
"project_id": 19,
|
|
"source_chat_id": "chat-abc",
|
|
"session_id": 55,
|
|
"force_new_session": True,
|
|
"thinking_mode": "deep",
|
|
"action_mode": "read_write",
|
|
"sync_requirement": True,
|
|
"message_intent": "mixed",
|
|
"mixed_signal_part": "新增任务确认流",
|
|
}
|
|
},
|
|
)
|
|
result = await executor._execute_query_intent(
|
|
intent=intent,
|
|
message_text="请同步更新状态",
|
|
operator_user_id=999,
|
|
source_system="feishu",
|
|
source_message_id="msg-1",
|
|
)
|
|
|
|
assert result["status"] == "succeeded"
|
|
assert "统一 ingress" in result["answer"]
|
|
answer_kwargs = captured["answer_kwargs"]
|
|
assert isinstance(answer_kwargs, dict)
|
|
assert answer_kwargs["asked_by"] == 101
|
|
assert answer_kwargs["asked_via"] == "pmflow_manual"
|
|
assert answer_kwargs["project_id"] == 19
|
|
assert answer_kwargs["source_chat_id"] == "chat-abc"
|
|
assert answer_kwargs["session_id"] == 55
|
|
assert answer_kwargs["force_new_session"] is True
|
|
assert answer_kwargs["thinking_mode"] == "deep"
|
|
assert answer_kwargs["action_mode"] == "read_write"
|
|
assert answer_kwargs["sync_requirement"] is True
|
|
assert answer_kwargs["message_intent"] == "mixed"
|
|
assert answer_kwargs["mixed_signal_part"] == "新增任务确认流"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_query_intent_feishu_fallback_uses_thread_key(monkeypatch) -> None:
|
|
captured: dict[str, object] = {}
|
|
|
|
class FakeAdaptiveRAGService:
|
|
def __init__(self, db, tenant_id: int):
|
|
_ = db
|
|
captured["tenant_id"] = tenant_id
|
|
|
|
async def answer_question(self, **kwargs):
|
|
captured["answer_kwargs"] = kwargs
|
|
return {"answer": "fallback-answer", "session_id": 7}
|
|
|
|
class FakeExecutor(mixin_execute_query_project.ExecuteQueryProjectMixin):
|
|
def __init__(self) -> None:
|
|
self.db = object()
|
|
self.tenant_id = 3
|
|
|
|
monkeypatch.setattr(mixin_execute_query_project, "AdaptiveRAGService", FakeAdaptiveRAGService)
|
|
executor = FakeExecutor()
|
|
intent = SimpleNamespace(project_id=None, metadata={"thread_context": {"root_message_id": "root-1"}})
|
|
result = await executor._execute_query_intent(
|
|
intent=intent,
|
|
message_text="最近AI行业有什么动态",
|
|
operator_user_id=999,
|
|
source_system="feishu",
|
|
source_message_id="msg-feishu-1",
|
|
)
|
|
|
|
assert result["status"] == "succeeded"
|
|
answer_kwargs = captured["answer_kwargs"]
|
|
assert isinstance(answer_kwargs, dict)
|
|
assert answer_kwargs["asked_by"] is None
|
|
assert answer_kwargs["asked_via"] == "feishu"
|
|
assert answer_kwargs["source_chat_id"] is None
|
|
assert answer_kwargs["conversation_key"] == build_feishu_thread_key(
|
|
source_chat_id=None,
|
|
source_sender_open_id=None,
|
|
sender=None,
|
|
source_message_id="msg-feishu-1",
|
|
thread_context={"root_message_id": "root-1"},
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_low_tier_rag_failure_prefers_glm5_openclaw_failover(monkeypatch) -> None:
|
|
captured: dict[str, object] = {"logs": []}
|
|
|
|
monkeypatch.setattr(
|
|
mixin_tier_escalation,
|
|
"_runtime_log",
|
|
lambda scope, message: captured["logs"].append((scope, message)),
|
|
)
|
|
|
|
low_engine = {
|
|
"id": "low-1",
|
|
"provider": "glm",
|
|
"model": "qwen-plus",
|
|
"base_url": "https://open.bigmodel.cn/api/paas/v4",
|
|
"api_key": "low-key",
|
|
}
|
|
high_engine = {
|
|
"id": "high-1",
|
|
"provider": "openai",
|
|
"model": "GLM-5",
|
|
"base_url": "https://api.openai.com/v1",
|
|
"api_key": "high-key",
|
|
}
|
|
bundle = {
|
|
"settings": {},
|
|
"active_engine_id": "low-1",
|
|
"engines": [low_engine, high_engine],
|
|
}
|
|
openclaw_bundle = {
|
|
"rules": [
|
|
{
|
|
"name": "engine_high_reasoning_glm5",
|
|
"enabled": True,
|
|
"priority": 220,
|
|
"tier": "high",
|
|
"provider": "openai",
|
|
"model": "GLM-5",
|
|
"api_env": "OPENAI_API_KEY",
|
|
"match_any": ["adaptive_rag_answer"],
|
|
}
|
|
]
|
|
}
|
|
|
|
class FakeLLM(LLMTierEscalationMixin):
|
|
def _low_to_high_escalation_enabled(self) -> bool:
|
|
return True
|
|
|
|
def _is_low_tier_function(self, function_name: str) -> bool:
|
|
return function_name == "adaptive_rag_answer_normal"
|
|
|
|
@staticmethod
|
|
def _rule_matches_function(rule, function_name: str) -> bool:
|
|
fn = str(function_name or "").strip().lower()
|
|
if not fn:
|
|
return False
|
|
if bool(rule.get("match_all", False)):
|
|
return True
|
|
exact = str(rule.get("function_exact") or "").strip().lower()
|
|
if exact and fn == exact:
|
|
return True
|
|
prefix = str(rule.get("function_prefix") or "").strip().lower()
|
|
if prefix and fn.startswith(prefix):
|
|
return True
|
|
match_any = rule.get("match_any")
|
|
if isinstance(match_any, list):
|
|
for hint in match_any:
|
|
token = str(hint or "").strip().lower()
|
|
if token and token in fn:
|
|
return True
|
|
return False
|
|
|
|
async def _get_or_init_bundle(self, db, *, tenant_id: int):
|
|
_ = (db, tenant_id)
|
|
return SimpleNamespace(), bundle
|
|
|
|
def _resolve_active_engine_from_bundle(self, bundle_value):
|
|
_ = bundle_value
|
|
return dict(low_engine)
|
|
|
|
def _load_openclaw_routing_bundle(self):
|
|
return dict(openclaw_bundle)
|
|
|
|
def _resolve_engine_from_rule(self, *, rule, engines, active_engine):
|
|
_ = active_engine
|
|
for item in engines:
|
|
if (
|
|
str(item.get("provider") or "").strip().lower() == str(rule.get("provider") or "").strip().lower()
|
|
and str(item.get("model") or "").strip() == str(rule.get("model") or "").strip()
|
|
):
|
|
return dict(item)
|
|
return None
|
|
|
|
def _ensure_openclaw_control_plane_engine(self, engine):
|
|
return dict(engine)
|
|
|
|
def _normalize_engine_settings(self, raw):
|
|
_ = raw
|
|
return {}
|
|
|
|
def _ensure_engine_allowed_for_function(self, *, engine, function_name, settings, engine_label):
|
|
_ = (function_name, settings, engine_label)
|
|
return dict(engine)
|
|
|
|
def _is_invalid_structured_output(self, *, function_name, response_text):
|
|
_ = (function_name, response_text)
|
|
return False
|
|
|
|
async def _chat_completion(self, **kwargs):
|
|
captured["chat_kwargs"] = kwargs
|
|
return "GLM-5 answer"
|
|
|
|
llm = FakeLLM()
|
|
text, resolved_engine = await llm._retry_with_high_tier_after_low_model_failure(
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="adaptive_rag_answer_normal",
|
|
failed_engine=dict(low_engine),
|
|
prompt="请回答一个问题",
|
|
system_prompt="system prompt",
|
|
temperature=0.2,
|
|
max_tokens=512,
|
|
response_mode="complete",
|
|
error=ValueError("LLM 返回空内容"),
|
|
)
|
|
|
|
assert text == "GLM-5 answer"
|
|
assert isinstance(resolved_engine, dict)
|
|
assert resolved_engine["provider"] == "openai"
|
|
assert resolved_engine["model"] == "GLM-5"
|
|
assert captured["chat_kwargs"]["model"] == "GLM-5"
|
|
log_text = "\n".join(message for _, message in captured["logs"])
|
|
assert "OpenClaw高阶升级命中" in log_text
|
|
assert "高阶升级成功" in log_text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_low_tier_intent_failure_forces_glm5_retry_even_when_generic_high_tier_differs(monkeypatch) -> None:
|
|
captured: dict[str, object] = {"logs": []}
|
|
|
|
monkeypatch.setattr(
|
|
mixin_tier_escalation,
|
|
"_runtime_log",
|
|
lambda scope, message: captured["logs"].append((scope, message)),
|
|
)
|
|
|
|
low_engine = {
|
|
"id": "low-1",
|
|
"provider": "glm",
|
|
"model": "qwen-plus",
|
|
"base_url": "https://open.bigmodel.cn/api/paas/v4",
|
|
"api_key": "low-key",
|
|
}
|
|
mini_engine = {
|
|
"id": "high-mini-1",
|
|
"provider": "openai",
|
|
"model": "MiniMax-M2.5",
|
|
"base_url": "https://api.openai.com/v1",
|
|
"api_key": "mini-key",
|
|
}
|
|
glm5_engine = {
|
|
"id": "high-glm5-1",
|
|
"provider": "openai",
|
|
"model": "GLM-5",
|
|
"base_url": "https://api.openai.com/v1",
|
|
"api_key": "glm5-key",
|
|
}
|
|
bundle = {
|
|
"settings": {},
|
|
"active_engine_id": "low-1",
|
|
"engines": [low_engine, mini_engine, glm5_engine],
|
|
}
|
|
openclaw_bundle = {
|
|
"rules": [
|
|
{
|
|
"name": "engine_low_intent_kernel_exact",
|
|
"enabled": True,
|
|
"priority": 310,
|
|
"tier": "low",
|
|
"provider": "glm",
|
|
"model": "qwen-plus",
|
|
"api_env": "DASHSCOPE_API_KEY",
|
|
"function_exact": "classify_intent_ai_pmo_kernel",
|
|
}
|
|
]
|
|
}
|
|
|
|
class FakeLLM(LLMTierEscalationMixin):
|
|
def _low_to_high_escalation_enabled(self) -> bool:
|
|
return True
|
|
|
|
def _is_low_tier_function(self, function_name: str) -> bool:
|
|
return function_name == "classify_intent_ai_pmo_kernel"
|
|
|
|
@staticmethod
|
|
def _rule_matches_function(rule, function_name: str) -> bool:
|
|
fn = str(function_name or "").strip().lower()
|
|
if not fn:
|
|
return False
|
|
if bool(rule.get("match_all", False)):
|
|
return True
|
|
exact = str(rule.get("function_exact") or "").strip().lower()
|
|
if exact and fn == exact:
|
|
return True
|
|
prefix = str(rule.get("function_prefix") or "").strip().lower()
|
|
if prefix and fn.startswith(prefix):
|
|
return True
|
|
match_any = rule.get("match_any")
|
|
if isinstance(match_any, list):
|
|
for hint in match_any:
|
|
token = str(hint or "").strip().lower()
|
|
if token and token in fn:
|
|
return True
|
|
return False
|
|
|
|
async def _get_or_init_bundle(self, db, *, tenant_id: int):
|
|
_ = (db, tenant_id)
|
|
return SimpleNamespace(), bundle
|
|
|
|
def _resolve_active_engine_from_bundle(self, bundle_value):
|
|
_ = bundle_value
|
|
return dict(low_engine)
|
|
|
|
def _load_openclaw_routing_bundle(self):
|
|
return dict(openclaw_bundle)
|
|
|
|
def _resolve_engine_from_rule(self, *, rule, engines, active_engine):
|
|
_ = active_engine
|
|
for item in engines:
|
|
if (
|
|
str(item.get("provider") or "").strip().lower() == str(rule.get("provider") or "").strip().lower()
|
|
and str(item.get("model") or "").strip() == str(rule.get("model") or "").strip()
|
|
):
|
|
return dict(item)
|
|
return None
|
|
|
|
def _resolve_tier_engine(self, *, tier, engines, active_engine):
|
|
_ = (engines, active_engine)
|
|
if str(tier).strip().lower() == "high":
|
|
return dict(mini_engine)
|
|
return dict(low_engine)
|
|
|
|
def _ensure_openclaw_control_plane_engine(self, engine):
|
|
return dict(engine)
|
|
|
|
def _normalize_engine_settings(self, raw):
|
|
_ = raw
|
|
return {}
|
|
|
|
def _ensure_engine_allowed_for_function(self, *, engine, function_name, settings, engine_label):
|
|
_ = (function_name, settings, engine_label)
|
|
return dict(engine)
|
|
|
|
def _is_invalid_structured_output(self, *, function_name, response_text):
|
|
_ = (function_name, response_text)
|
|
return False
|
|
|
|
async def _chat_completion(self, **kwargs):
|
|
captured["chat_kwargs"] = kwargs
|
|
return "GLM-5 answer"
|
|
|
|
llm = FakeLLM()
|
|
text, resolved_engine = await llm._retry_with_high_tier_after_low_model_failure(
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="classify_intent_ai_pmo_kernel",
|
|
failed_engine=dict(low_engine),
|
|
prompt="请返回结构化意图",
|
|
system_prompt="system prompt",
|
|
temperature=0.2,
|
|
max_tokens=512,
|
|
response_mode="complete",
|
|
error=ValueError("LLM 返回空内容"),
|
|
)
|
|
|
|
assert text == "GLM-5 answer"
|
|
assert isinstance(resolved_engine, dict)
|
|
assert resolved_engine["model"] == "GLM-5"
|
|
assert captured["chat_kwargs"]["model"] == "GLM-5"
|
|
log_text = "\n".join(message for _, message in captured["logs"])
|
|
assert "GLM-5强制高阶升级命中" in log_text or "OpenClaw高阶升级命中" in log_text
|
|
assert "高阶升级成功" in log_text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_low_tier_structured_failure_forces_glm5_retry(monkeypatch) -> None:
|
|
captured: dict[str, object] = {"logs": []}
|
|
|
|
monkeypatch.setattr(
|
|
mixin_tier_escalation,
|
|
"_runtime_log",
|
|
lambda scope, message: captured["logs"].append((scope, message)),
|
|
)
|
|
|
|
low_engine = {
|
|
"id": "low-1",
|
|
"provider": "glm",
|
|
"model": "qwen-plus",
|
|
"base_url": "https://open.bigmodel.cn/api/paas/v4",
|
|
"api_key": "low-key",
|
|
}
|
|
mini_engine = {
|
|
"id": "high-mini-1",
|
|
"provider": "openai",
|
|
"model": "MiniMax-M2.5",
|
|
"base_url": "https://api.openai.com/v1",
|
|
"api_key": "mini-key",
|
|
}
|
|
glm5_engine = {
|
|
"id": "high-glm5-1",
|
|
"provider": "openai",
|
|
"model": "GLM-5",
|
|
"base_url": "https://api.openai.com/v1",
|
|
"api_key": "glm5-key",
|
|
}
|
|
bundle = {
|
|
"settings": {},
|
|
"active_engine_id": "low-1",
|
|
"engines": [low_engine, mini_engine, glm5_engine],
|
|
}
|
|
openclaw_bundle = {
|
|
"rules": [
|
|
{
|
|
"name": "engine_low_intent_kernel_exact",
|
|
"enabled": True,
|
|
"priority": 310,
|
|
"tier": "low",
|
|
"provider": "glm",
|
|
"model": "qwen-plus",
|
|
"api_env": "DASHSCOPE_API_KEY",
|
|
"function_exact": "classify_intent_ai_pmo_kernel",
|
|
}
|
|
]
|
|
}
|
|
|
|
class FakeLLM(LLMTierEscalationMixin):
|
|
def _low_to_high_escalation_enabled(self) -> bool:
|
|
return True
|
|
|
|
def _is_low_tier_function(self, function_name: str) -> bool:
|
|
return function_name == "classify_intent_ai_pmo_kernel"
|
|
|
|
@staticmethod
|
|
def _rule_matches_function(rule, function_name: str) -> bool:
|
|
fn = str(function_name or "").strip().lower()
|
|
if not fn:
|
|
return False
|
|
if bool(rule.get("match_all", False)):
|
|
return True
|
|
exact = str(rule.get("function_exact") or "").strip().lower()
|
|
if exact and fn == exact:
|
|
return True
|
|
prefix = str(rule.get("function_prefix") or "").strip().lower()
|
|
if prefix and fn.startswith(prefix):
|
|
return True
|
|
match_any = rule.get("match_any")
|
|
if isinstance(match_any, list):
|
|
for hint in match_any:
|
|
token = str(hint or "").strip().lower()
|
|
if token and token in fn:
|
|
return True
|
|
return False
|
|
|
|
async def _get_or_init_bundle(self, db, *, tenant_id: int):
|
|
_ = (db, tenant_id)
|
|
return SimpleNamespace(), bundle
|
|
|
|
def _resolve_active_engine_from_bundle(self, bundle_value):
|
|
_ = bundle_value
|
|
return dict(low_engine)
|
|
|
|
def _load_openclaw_routing_bundle(self):
|
|
return dict(openclaw_bundle)
|
|
|
|
def _resolve_engine_from_rule(self, *, rule, engines, active_engine):
|
|
_ = active_engine
|
|
for item in engines:
|
|
if (
|
|
str(item.get("provider") or "").strip().lower() == str(rule.get("provider") or "").strip().lower()
|
|
and str(item.get("model") or "").strip() == str(rule.get("model") or "").strip()
|
|
):
|
|
return dict(item)
|
|
return None
|
|
|
|
def _resolve_tier_engine(self, *, tier, engines, active_engine):
|
|
_ = (engines, active_engine)
|
|
if str(tier).strip().lower() == "high":
|
|
return dict(mini_engine)
|
|
return dict(low_engine)
|
|
|
|
def _ensure_openclaw_control_plane_engine(self, engine):
|
|
return dict(engine)
|
|
|
|
def _normalize_engine_settings(self, raw):
|
|
_ = raw
|
|
return {}
|
|
|
|
def _ensure_engine_allowed_for_function(self, *, engine, function_name, settings, engine_label):
|
|
_ = (function_name, settings, engine_label)
|
|
return dict(engine)
|
|
|
|
def _is_invalid_structured_output(self, *, function_name, response_text):
|
|
_ = (function_name, response_text)
|
|
return False
|
|
|
|
def _extract_structured_object_from_response(
|
|
self,
|
|
*,
|
|
function_name,
|
|
response,
|
|
fallback_text,
|
|
provider,
|
|
model,
|
|
):
|
|
_ = (function_name, response, fallback_text)
|
|
return StructuredExtractionResult(
|
|
payload={"confidence": 0.91, "reasoning": "ok", "intent": "query.read_only"},
|
|
source_field="generated_text",
|
|
provider=provider,
|
|
model=model,
|
|
raw_text='{"confidence": 0.91, "reasoning": "ok", "intent": "query.read_only"}',
|
|
validation="ok",
|
|
)
|
|
|
|
async def _chat_completion_raw(self, **kwargs):
|
|
captured["chat_kwargs"] = kwargs
|
|
raw_text = '{"confidence": 0.91, "reasoning": "ok", "intent": "query.read_only"}'
|
|
return ChatCompletionResult(text=raw_text, raw_response=raw_text)
|
|
|
|
llm = FakeLLM()
|
|
result, resolved_engine = await llm._retry_structured_with_high_tier_after_low_model_failure(
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="classify_intent_ai_pmo_kernel",
|
|
failed_engine=dict(low_engine),
|
|
prompt="请返回结构化意图",
|
|
system_prompt="system prompt",
|
|
temperature=0.2,
|
|
max_tokens=512,
|
|
response_mode="complete",
|
|
error=ValueError("LLM 返回空内容"),
|
|
)
|
|
|
|
assert getattr(result, "payload", None) is not None
|
|
assert result.payload["confidence"] == pytest.approx(0.91)
|
|
assert result.model == "GLM-5"
|
|
assert isinstance(resolved_engine, dict)
|
|
assert resolved_engine["model"] == "GLM-5"
|
|
assert captured["chat_kwargs"]["model"] == "GLM-5"
|
|
log_text = "\n".join(message for _, message in captured["logs"])
|
|
assert "GLM-5强制高阶升级命中" in log_text or "OpenClaw高阶升级命中" in log_text
|
|
assert "高阶结构化升级成功" in log_text
|