355 lines
13 KiB
Python
355 lines
13 KiB
Python
import os
|
|
import unittest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from app.services.llm import ChatCompletionResult, LLMService, StructuredExtractionResult
|
|
|
|
|
|
class LLMStructuredExtractionTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.service = LLMService()
|
|
|
|
def _extract(self, response: dict, *, function_name: str = "openclaw_command_dispatch") -> StructuredExtractionResult:
|
|
return self.service._extract_structured_object_from_response(
|
|
function_name=function_name,
|
|
response=response,
|
|
fallback_text="",
|
|
provider="glm",
|
|
model="GLM-4.7",
|
|
)
|
|
|
|
def test_extracts_object_from_plain_content(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": '{"summary":"ok","confidence":0.9,"operations":[]}',
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.validation, "ok")
|
|
self.assertEqual(result.payload["summary"], "ok")
|
|
|
|
def test_extracts_object_from_typed_chunk_content(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": [
|
|
{"type": "output_text", "text": '{"summary":"chunk","confidence":0.8,"operations":[]}'}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.validation, "ok")
|
|
self.assertEqual(result.payload["summary"], "chunk")
|
|
|
|
def test_extracts_object_from_tool_call_arguments(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"tool_calls": [
|
|
{
|
|
"function": {
|
|
"arguments": '{"summary":"args","confidence":0.7,"operations":[]}'
|
|
}
|
|
}
|
|
],
|
|
"content": "",
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.source_field, "tool_calls.function.arguments")
|
|
self.assertEqual(result.payload["summary"], "args")
|
|
|
|
def test_extracts_json_from_reasoning_content_with_prefix_text(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": "",
|
|
"reasoning_content": "先分析上下文，再给结果：\n{\"summary\":\"from_reasoning\",\"confidence\":0.66,\"operations\":[]}",
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.source_field, "message.reasoning_content")
|
|
self.assertEqual(result.payload["summary"], "from_reasoning")
|
|
|
|
def test_returns_json_not_found_when_reasoning_has_only_natural_language(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": "",
|
|
"reasoning_content": "我认为还需要更多上下文。",
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.error_code, "structured_json_not_found")
|
|
self.assertEqual(result.source_field, "message.reasoning_content")
|
|
|
|
def test_unwraps_nested_json_field_once(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": '{"json":"{\\"summary\\":\\"nested\\",\\"confidence\\":0.55,\\"operations\\":[]}"}',
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.validation, "ok")
|
|
self.assertEqual(result.payload["summary"], "nested")
|
|
|
|
def test_rejects_top_level_array_for_object_only_function(self) -> None:
|
|
result = self._extract(
|
|
{
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": '[{"summary":"bad"}]',
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.error_code, "structured_schema_invalid")
|
|
|
|
|
|
class LLMStructuredCacheTest(unittest.IsolatedAsyncioTestCase):
|
|
async def test_generate_structured_object_uses_cache_after_first_hit(self) -> None:
|
|
service = LLMService()
|
|
calls: list[str] = []
|
|
cache: dict[str, str] = {}
|
|
|
|
class _FakeRedis:
|
|
async def get(self, key: str):
|
|
return cache.get(key)
|
|
|
|
async def setex(self, key: str, _ttl: int, value: str):
|
|
cache[key] = value
|
|
|
|
service._get_redis = AsyncMock(return_value=_FakeRedis())
|
|
service._resolve_engine_for_call = AsyncMock(
|
|
return_value={
|
|
"provider": "glm",
|
|
"model": "GLM-4.7",
|
|
"api_key": "fake",
|
|
"base_url": "http://unit.test",
|
|
}
|
|
)
|
|
service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""})
|
|
service._log_ai_call = AsyncMock()
|
|
|
|
async def _fake_chat_completion_raw(**_: object) -> ChatCompletionResult:
|
|
calls.append("hit")
|
|
return ChatCompletionResult(
|
|
text='{"summary":"cached","confidence":0.9,"operations":[]}',
|
|
raw_response={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": '{"summary":"cached","confidence":0.9,"operations":[]}',
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
service._chat_completion_raw = AsyncMock(side_effect=_fake_chat_completion_raw)
|
|
|
|
first = await service.generate_structured_object(
|
|
"planner prompt",
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="openclaw_command_dispatch",
|
|
response_mode="fast",
|
|
)
|
|
second = await service.generate_structured_object(
|
|
"planner prompt",
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="openclaw_command_dispatch",
|
|
response_mode="fast",
|
|
)
|
|
|
|
self.assertEqual(first.payload["summary"], "cached")
|
|
self.assertEqual(second.payload["summary"], "cached")
|
|
self.assertTrue(second.from_cache)
|
|
self.assertEqual(len(calls), 1)
|
|
|
|
|
|
class LLMQueryFailureBehaviorTest(unittest.IsolatedAsyncioTestCase):
|
|
async def test_generate_strict_query_retries_high_tier_even_when_rule_not_low(self) -> None:
|
|
service = LLMService()
|
|
service._resolve_engine_for_call = AsyncMock(
|
|
return_value={
|
|
"provider": "glm",
|
|
"model": "GLM-4.7",
|
|
"api_key": "fake",
|
|
"base_url": "http://unit.test",
|
|
}
|
|
)
|
|
service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""})
|
|
service._is_low_tier_function = MagicMock(return_value=False)
|
|
service._resolve_high_tier_engine_for_escalation = AsyncMock(
|
|
return_value={
|
|
"provider": "openai",
|
|
"model": "GLM-5",
|
|
"api_key": "fake-openai",
|
|
"base_url": "http://unit.test/openai",
|
|
}
|
|
)
|
|
service._chat_completion = AsyncMock(
|
|
side_effect=[ValueError("provider_circuit_open:glm:GLM-4.7"), "高阶模型正常响应"]
|
|
)
|
|
service._try_safety_engine = AsyncMock(return_value=("", None))
|
|
service._log_ai_call = AsyncMock()
|
|
|
|
text = await service.generate(
|
|
"query prompt",
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="adaptive_rag_answer_normal",
|
|
use_prompt_profile=False,
|
|
)
|
|
|
|
self.assertEqual(text, "高阶模型正常响应")
|
|
self.assertEqual(service._chat_completion.await_count, 2)
|
|
|
|
async def test_generate_raises_runtime_error_for_query_answer_fail_fast(self) -> None:
|
|
service = LLMService()
|
|
service._resolve_engine_for_call = AsyncMock(
|
|
return_value={
|
|
"provider": "glm",
|
|
"model": "GLM-4.7",
|
|
"api_key": "fake",
|
|
"base_url": "http://unit.test",
|
|
}
|
|
)
|
|
service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""})
|
|
service._chat_completion = AsyncMock(side_effect=ValueError("primary_engine_empty_response"))
|
|
service._resolve_high_tier_engine_for_escalation = AsyncMock(return_value=None)
|
|
service._try_safety_engine = AsyncMock(return_value=("", None))
|
|
service._log_ai_call = AsyncMock()
|
|
|
|
with self.assertRaises(RuntimeError) as ctx:
|
|
await service.generate(
|
|
"query prompt",
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="adaptive_rag_answer_normal",
|
|
use_prompt_profile=False,
|
|
)
|
|
|
|
self.assertIn("模型调用失败：", str(ctx.exception))
|
|
service._try_safety_engine.assert_not_awaited()
|
|
|
|
async def test_generate_keeps_stub_behavior_for_non_query_function(self) -> None:
|
|
service = LLMService()
|
|
service._resolve_engine_for_call = AsyncMock(
|
|
return_value={
|
|
"provider": "glm",
|
|
"model": "GLM-4.7",
|
|
"api_key": "fake",
|
|
"base_url": "http://unit.test",
|
|
}
|
|
)
|
|
service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""})
|
|
service._chat_completion = AsyncMock(side_effect=ValueError("primary_engine_empty_response"))
|
|
service._log_ai_call = AsyncMock()
|
|
|
|
text = await service.generate(
|
|
"reply prompt",
|
|
db=object(),
|
|
tenant_id=1,
|
|
function_name="feishu_humanized_reply",
|
|
use_prompt_profile=False,
|
|
)
|
|
|
|
self.assertIn("模型服务异常", text)
|
|
|
|
|
|
class LLMRoutingSelectionTest(unittest.TestCase):
|
|
def test_inject_function_guardrails_for_dual_model_functions(self) -> None:
|
|
injected = LLMService._inject_function_guardrails(
|
|
system_prompt="",
|
|
function_name="classify_intent_complex",
|
|
)
|
|
self.assertIn("ddl_api_required", injected)
|
|
self.assertIn("DDL", injected)
|
|
|
|
def test_merge_prompt_with_profile_appends_guardrail_for_openclaw_dispatch(self) -> None:
|
|
merged = LLMService._merge_prompt_with_profile(
|
|
prompt='{"instruction":"db ddl"}',
|
|
prompt_profile={"role_name": "planner", "system_prompt": "你是规划器。"},
|
|
function_name="openclaw_command_dispatch",
|
|
)
|
|
self.assertIn("你是规划器。", merged)
|
|
self.assertIn("ddl_api_required", merged)
|
|
|
|
def test_resolve_tier_engine_keeps_configured_openai_base_url(self) -> None:
|
|
service = LLMService()
|
|
env_updates = {
|
|
"OPENCLAW_ROUTING_ENABLED": "1",
|
|
"OPENAI_BASE_URL": "https://api.edgefn.net/v1",
|
|
"LLM_ROUTING_HIGH_PROVIDER": "openai",
|
|
}
|
|
with patch.dict(os.environ, env_updates, clear=False):
|
|
engine = service._resolve_tier_engine(
|
|
tier="high",
|
|
engines=[],
|
|
active_engine={},
|
|
)
|
|
|
|
self.assertIsNotNone(engine)
|
|
assert engine is not None
|
|
self.assertEqual(str(engine.get("base_url")), "https://api.edgefn.net/v1")
|
|
|
|
def test_should_retry_empty_chat_response_for_reasoning_length(self) -> None:
|
|
should_retry = LLMService._should_retry_empty_chat_response(
|
|
structured_output=False,
|
|
message={"content": "", "reasoning_content": "先分析上下文"},
|
|
choice={"finish_reason": "length"},
|
|
current_max_tokens=900,
|
|
)
|
|
self.assertTrue(should_retry)
|
|
|
|
def test_should_not_retry_empty_chat_response_when_not_length(self) -> None:
|
|
should_retry = LLMService._should_retry_empty_chat_response(
|
|
structured_output=False,
|
|
message={"content": "", "reasoning_content": "先分析上下文"},
|
|
choice={"finish_reason": "stop"},
|
|
current_max_tokens=900,
|
|
)
|
|
self.assertFalse(should_retry)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|