import os import unittest from unittest.mock import AsyncMock, MagicMock, patch from app.services.llm import ChatCompletionResult, LLMService, StructuredExtractionResult class LLMStructuredExtractionTest(unittest.TestCase): def setUp(self) -> None: self.service = LLMService() def _extract(self, response: dict, *, function_name: str = "openclaw_command_dispatch") -> StructuredExtractionResult: return self.service._extract_structured_object_from_response( function_name=function_name, response=response, fallback_text="", provider="glm", model="GLM-4.7", ) def test_extracts_object_from_plain_content(self) -> None: result = self._extract( { "choices": [ { "message": { "content": '{"summary":"ok","confidence":0.9,"operations":[]}', } } ] } ) self.assertEqual(result.validation, "ok") self.assertEqual(result.payload["summary"], "ok") def test_extracts_object_from_typed_chunk_content(self) -> None: result = self._extract( { "choices": [ { "message": { "content": [ {"type": "output_text", "text": '{"summary":"chunk","confidence":0.8,"operations":[]}'} ] } } ] } ) self.assertEqual(result.validation, "ok") self.assertEqual(result.payload["summary"], "chunk") def test_extracts_object_from_tool_call_arguments(self) -> None: result = self._extract( { "choices": [ { "message": { "tool_calls": [ { "function": { "arguments": '{"summary":"args","confidence":0.7,"operations":[]}' } } ], "content": "", } } ] } ) self.assertEqual(result.source_field, "tool_calls.function.arguments") self.assertEqual(result.payload["summary"], "args") def test_extracts_json_from_reasoning_content_with_prefix_text(self) -> None: result = self._extract( { "choices": [ { "message": { "content": "", "reasoning_content": "先分析上下文，再给结果：\n{\"summary\":\"from_reasoning\",\"confidence\":0.66,\"operations\":[]}", } } ] } ) self.assertEqual(result.source_field, "message.reasoning_content") self.assertEqual(result.payload["summary"], "from_reasoning") def test_returns_json_not_found_when_reasoning_has_only_natural_language(self) -> None: result = self._extract( { "choices": [ { "message": { "content": "", "reasoning_content": "我认为还需要更多上下文。", } } ] } ) self.assertEqual(result.error_code, "structured_json_not_found") self.assertEqual(result.source_field, "message.reasoning_content") def test_unwraps_nested_json_field_once(self) -> None: result = self._extract( { "choices": [ { "message": { "content": '{"json":"{\\"summary\\":\\"nested\\",\\"confidence\\":0.55,\\"operations\\":[]}"}', } } ] } ) self.assertEqual(result.validation, "ok") self.assertEqual(result.payload["summary"], "nested") def test_rejects_top_level_array_for_object_only_function(self) -> None: result = self._extract( { "choices": [ { "message": { "content": '[{"summary":"bad"}]', } } ] } ) self.assertEqual(result.error_code, "structured_schema_invalid") class LLMStructuredCacheTest(unittest.IsolatedAsyncioTestCase): async def test_generate_structured_object_uses_cache_after_first_hit(self) -> None: service = LLMService() calls: list[str] = [] cache: dict[str, str] = {} class _FakeRedis: async def get(self, key: str): return cache.get(key) async def setex(self, key: str, _ttl: int, value: str): cache[key] = value service._get_redis = AsyncMock(return_value=_FakeRedis()) service._resolve_engine_for_call = AsyncMock( return_value={ "provider": "glm", "model": "GLM-4.7", "api_key": "fake", "base_url": "http://unit.test", } ) service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""}) service._log_ai_call = AsyncMock() async def _fake_chat_completion_raw(**_: object) -> ChatCompletionResult: calls.append("hit") return ChatCompletionResult( text='{"summary":"cached","confidence":0.9,"operations":[]}', raw_response={ "choices": [ { "message": { "content": '{"summary":"cached","confidence":0.9,"operations":[]}', } } ] }, ) service._chat_completion_raw = AsyncMock(side_effect=_fake_chat_completion_raw) first = await service.generate_structured_object( "planner prompt", db=object(), tenant_id=1, function_name="openclaw_command_dispatch", response_mode="fast", ) second = await service.generate_structured_object( "planner prompt", db=object(), tenant_id=1, function_name="openclaw_command_dispatch", response_mode="fast", ) self.assertEqual(first.payload["summary"], "cached") self.assertEqual(second.payload["summary"], "cached") self.assertTrue(second.from_cache) self.assertEqual(len(calls), 1) class LLMQueryFailureBehaviorTest(unittest.IsolatedAsyncioTestCase): async def test_generate_strict_query_retries_high_tier_even_when_rule_not_low(self) -> None: service = LLMService() service._resolve_engine_for_call = AsyncMock( return_value={ "provider": "glm", "model": "GLM-4.7", "api_key": "fake", "base_url": "http://unit.test", } ) service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""}) service._is_low_tier_function = MagicMock(return_value=False) service._resolve_high_tier_engine_for_escalation = AsyncMock( return_value={ "provider": "openai", "model": "GLM-5", "api_key": "fake-openai", "base_url": "http://unit.test/openai", } ) service._chat_completion = AsyncMock( side_effect=[ValueError("provider_circuit_open:glm:GLM-4.7"), "高阶模型正常响应"] ) service._try_safety_engine = AsyncMock(return_value=("", None)) service._log_ai_call = AsyncMock() text = await service.generate( "query prompt", db=object(), tenant_id=1, function_name="adaptive_rag_answer_normal", use_prompt_profile=False, ) self.assertEqual(text, "高阶模型正常响应") self.assertEqual(service._chat_completion.await_count, 2) async def test_generate_raises_runtime_error_for_query_answer_fail_fast(self) -> None: service = LLMService() service._resolve_engine_for_call = AsyncMock( return_value={ "provider": "glm", "model": "GLM-4.7", "api_key": "fake", "base_url": "http://unit.test", } ) service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""}) service._chat_completion = AsyncMock(side_effect=ValueError("primary_engine_empty_response")) service._resolve_high_tier_engine_for_escalation = AsyncMock(return_value=None) service._try_safety_engine = AsyncMock(return_value=("", None)) service._log_ai_call = AsyncMock() with self.assertRaises(RuntimeError) as ctx: await service.generate( "query prompt", db=object(), tenant_id=1, function_name="adaptive_rag_answer_normal", use_prompt_profile=False, ) self.assertIn("模型调用失败：", str(ctx.exception)) service._try_safety_engine.assert_not_awaited() async def test_generate_keeps_stub_behavior_for_non_query_function(self) -> None: service = LLMService() service._resolve_engine_for_call = AsyncMock( return_value={ "provider": "glm", "model": "GLM-4.7", "api_key": "fake", "base_url": "http://unit.test", } ) service._resolve_active_prompt_profile = AsyncMock(return_value={"role_name": "", "system_prompt": ""}) service._chat_completion = AsyncMock(side_effect=ValueError("primary_engine_empty_response")) service._log_ai_call = AsyncMock() text = await service.generate( "reply prompt", db=object(), tenant_id=1, function_name="feishu_humanized_reply", use_prompt_profile=False, ) self.assertIn("模型服务异常", text) class LLMRoutingSelectionTest(unittest.TestCase): def test_inject_function_guardrails_for_dual_model_functions(self) -> None: injected = LLMService._inject_function_guardrails( system_prompt="", function_name="classify_intent_complex", ) self.assertIn("ddl_api_required", injected) self.assertIn("DDL", injected) def test_merge_prompt_with_profile_appends_guardrail_for_openclaw_dispatch(self) -> None: merged = LLMService._merge_prompt_with_profile( prompt='{"instruction":"db ddl"}', prompt_profile={"role_name": "planner", "system_prompt": "你是规划器。"}, function_name="openclaw_command_dispatch", ) self.assertIn("你是规划器。", merged) self.assertIn("ddl_api_required", merged) def test_resolve_tier_engine_keeps_configured_openai_base_url(self) -> None: service = LLMService() env_updates = { "OPENCLAW_ROUTING_ENABLED": "1", "OPENAI_BASE_URL": "https://api.edgefn.net/v1", "LLM_ROUTING_HIGH_PROVIDER": "openai", } with patch.dict(os.environ, env_updates, clear=False): engine = service._resolve_tier_engine( tier="high", engines=[], active_engine={}, ) self.assertIsNotNone(engine) assert engine is not None self.assertEqual(str(engine.get("base_url")), "https://api.edgefn.net/v1") def test_should_retry_empty_chat_response_for_reasoning_length(self) -> None: should_retry = LLMService._should_retry_empty_chat_response( structured_output=False, message={"content": "", "reasoning_content": "先分析上下文"}, choice={"finish_reason": "length"}, current_max_tokens=900, ) self.assertTrue(should_retry) def test_should_not_retry_empty_chat_response_when_not_length(self) -> None: should_retry = LLMService._should_retry_empty_chat_response( structured_output=False, message={"content": "", "reasoning_content": "先分析上下文"}, choice={"finish_reason": "stop"}, current_max_tokens=900, ) self.assertFalse(should_retry) if __name__ == "__main__": unittest.main()