505 lines
21 KiB
Python
505 lines
21 KiB
Python
from __future__ import annotations
|
||
|
||
from .shared import *
|
||
|
||
class LLMTierEscalationMixin:
|
||
def _low_to_high_escalation_enabled(self) -> bool:
|
||
return _env_bool("LLM_LOW_TO_HIGH_ESCALATION_ENABLED", True)
|
||
|
||
@staticmethod
|
||
def _same_runtime_engine(a: dict[str, Any], b: dict[str, Any]) -> bool:
|
||
return (
|
||
str(a.get("provider") or "").strip().lower() == str(b.get("provider") or "").strip().lower()
|
||
and str(a.get("model") or "").strip().lower() == str(b.get("model") or "").strip().lower()
|
||
and str(a.get("base_url") or "").strip().rstrip("/").lower()
|
||
== str(b.get("base_url") or "").strip().rstrip("/").lower()
|
||
)
|
||
|
||
def _is_low_tier_function(self, function_name: str) -> bool:
|
||
rule = self._match_openclaw_rule(function_name)
|
||
if rule is not None:
|
||
tier = str(rule.get("tier") or "").strip().lower()
|
||
if tier:
|
||
return tier == "low"
|
||
return self._resolve_function_tier(function_name) == "low"
|
||
|
||
@staticmethod
|
||
def _looks_like_unanswerable_response(text: str) -> bool:
|
||
cleaned = str(text or "").strip()
|
||
lowered = cleaned.lower()
|
||
if not lowered:
|
||
return True
|
||
if len(cleaned) <= 8:
|
||
return True
|
||
return any(marker in lowered for marker in LOW_TO_HIGH_ESCALATION_FALLBACK_MARKERS)
|
||
|
||
def _should_escalate_low_to_high(self, *, function_name: str, response_text: str) -> bool:
|
||
if not self._low_to_high_escalation_enabled():
|
||
return False
|
||
if not self._is_low_tier_function(function_name):
|
||
return False
|
||
return self._looks_like_unanswerable_response(response_text)
|
||
|
||
@staticmethod
|
||
def _is_glm_47_model(model_name: str) -> bool:
|
||
normalized = re.sub(r"[\s_]+", "", str(model_name or "")).lower()
|
||
if not normalized:
|
||
return False
|
||
if normalized.startswith("qwen"):
|
||
return True
|
||
return "glm" in normalized and ("4.7" in normalized or "47" in normalized)
|
||
|
||
@staticmethod
|
||
def _is_glm_5_model(model_name: str) -> bool:
|
||
normalized = re.sub(r"[\s_]+", "", str(model_name or "")).lower()
|
||
if not normalized:
|
||
return False
|
||
return normalized in {"glm-5", "glm5"} or normalized.startswith("glm-5") or normalized.startswith("glm5")
|
||
|
||
def _should_retry_glm47_with_high_tier(
|
||
self,
|
||
*,
|
||
function_name: str,
|
||
failed_engine: dict[str, Any],
|
||
) -> bool:
|
||
if not self._low_to_high_escalation_enabled():
|
||
return False
|
||
failed_model = str(failed_engine.get("model") or "").strip()
|
||
if self._is_glm_47_model(failed_model):
|
||
return True
|
||
# 兜底兼容：低阶函数遇到 4.x GLM 也允许升级高阶重试。
|
||
if self._is_low_tier_function(function_name):
|
||
lowered = failed_model.lower()
|
||
return "glm" in lowered and "4." in lowered
|
||
return False
|
||
|
||
def _prefer_glm5_high_tier_retry_engine(
|
||
self,
|
||
*,
|
||
bundle: dict[str, Any],
|
||
engines: list[dict[str, Any]],
|
||
function_name: str,
|
||
failed_engine: dict[str, Any],
|
||
current_engine: dict[str, Any],
|
||
) -> Optional[dict[str, Any]]:
|
||
if self._is_glm_5_model(str(current_engine.get("model") or "")):
|
||
return current_engine
|
||
settings = self._normalize_engine_settings(bundle.get("settings"))
|
||
for item in engines:
|
||
candidate = dict(item)
|
||
if not self._is_glm_5_model(str(candidate.get("model") or "")):
|
||
continue
|
||
if self._same_runtime_engine(candidate, failed_engine):
|
||
continue
|
||
try:
|
||
candidate = self._ensure_engine_allowed_for_function(
|
||
engine=candidate,
|
||
function_name=function_name,
|
||
settings=settings,
|
||
engine_label="forced_glm5_failover",
|
||
)
|
||
except Exception as exc:
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"GLM-5强制升级候选被拦截："
|
||
f"function={function_name}, error={type(exc).__name__}:{str(exc)[:180]}"
|
||
),
|
||
)
|
||
continue
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"GLM-5强制高阶升级命中："
|
||
f"function={function_name}, provider={candidate.get('provider')}, model={candidate.get('model')}"
|
||
),
|
||
)
|
||
return candidate
|
||
return None
|
||
|
||
def _match_openclaw_high_tier_rule_for_escalation(self, function_name: str) -> Optional[dict[str, Any]]:
|
||
bundle = self._load_openclaw_routing_bundle()
|
||
rules = bundle.get("rules") if isinstance(bundle.get("rules"), list) else []
|
||
indexed: list[tuple[int, int, dict[str, Any]]] = []
|
||
for idx, row in enumerate(rules):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
if not bool(row.get("enabled", True)):
|
||
continue
|
||
tier_name = str(row.get("tier") or "").strip().lower()
|
||
if tier_name != "high":
|
||
continue
|
||
if not self._rule_matches_function(row, function_name):
|
||
continue
|
||
priority = int(row.get("priority") or 0)
|
||
indexed.append((priority, -idx, row))
|
||
if not indexed:
|
||
_runtime_log("LLM路由", f"OpenClaw高阶规则未命中：function={function_name}")
|
||
return None
|
||
indexed.sort(reverse=True)
|
||
rule = dict(indexed[0][2])
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶规则候选："
|
||
f"function={function_name}, rule={rule.get('name') or 'unnamed'}, "
|
||
f"priority={rule.get('priority')}, provider={rule.get('provider')}, model={rule.get('model')}"
|
||
),
|
||
)
|
||
return rule
|
||
|
||
async def _resolve_openclaw_high_tier_engine_for_escalation(
|
||
self,
|
||
*,
|
||
bundle: dict[str, Any],
|
||
engines: list[dict[str, Any]],
|
||
active_engine: dict[str, Any],
|
||
function_name: str,
|
||
failed_engine: dict[str, Any],
|
||
) -> Optional[dict[str, Any]]:
|
||
rule = self._match_openclaw_high_tier_rule_for_escalation(function_name)
|
||
if not isinstance(rule, dict):
|
||
return None
|
||
routed = self._resolve_engine_from_rule(
|
||
rule=rule,
|
||
engines=engines,
|
||
active_engine=active_engine,
|
||
)
|
||
if not isinstance(routed, dict):
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶规则命中但未解析到可用引擎："
|
||
f"function={function_name}, rule={rule.get('name') or 'unnamed'}"
|
||
),
|
||
)
|
||
return None
|
||
routed = self._ensure_openclaw_control_plane_engine(routed)
|
||
if self._same_runtime_engine(routed, failed_engine):
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶升级命中但与失败引擎同源，跳过："
|
||
f"function={function_name}, provider={routed.get('provider')}, model={routed.get('model')}"
|
||
),
|
||
)
|
||
return None
|
||
try:
|
||
routed = self._ensure_engine_allowed_for_function(
|
||
engine=routed,
|
||
function_name=function_name,
|
||
settings=self._normalize_engine_settings(bundle.get("settings")),
|
||
engine_label="openclaw_high_tier_failover",
|
||
)
|
||
except Exception as exc:
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶升级被拦截："
|
||
f"function={function_name}, rule={rule.get('name') or 'unnamed'}, "
|
||
f"error={type(exc).__name__}:{str(exc)[:180]}"
|
||
),
|
||
)
|
||
return None
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶升级命中："
|
||
f"function={function_name}, rule={rule.get('name') or 'unnamed'}, "
|
||
f"provider={routed.get('provider')}, model={routed.get('model')}"
|
||
),
|
||
)
|
||
return routed
|
||
|
||
async def _retry_with_high_tier_after_low_model_failure(
|
||
self,
|
||
*,
|
||
db: AsyncSession,
|
||
tenant_id: int,
|
||
function_name: str,
|
||
failed_engine: dict[str, Any],
|
||
prompt: str,
|
||
system_prompt: str,
|
||
temperature: float,
|
||
max_tokens: int,
|
||
response_mode: str,
|
||
error: Exception,
|
||
) -> tuple[Optional[str], Optional[dict[str, Any]]]:
|
||
if not self._should_retry_glm47_with_high_tier(function_name=function_name, failed_engine=failed_engine):
|
||
return None, None
|
||
try:
|
||
_, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id)
|
||
engines = [item for item in bundle.get("engines", []) if isinstance(item, dict)]
|
||
active_engine = self._resolve_active_engine_from_bundle(bundle)
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"低阶模型失败准备升级高阶："
|
||
f"function={function_name}, from={str(failed_engine.get('provider') or '').strip().lower()}:"
|
||
f"{str(failed_engine.get('model') or '').strip()}, reason={str(error)[:120]}"
|
||
),
|
||
)
|
||
high_engine = await self._resolve_openclaw_high_tier_engine_for_escalation(
|
||
bundle=bundle,
|
||
engines=engines,
|
||
active_engine=active_engine,
|
||
function_name=function_name,
|
||
failed_engine=failed_engine,
|
||
)
|
||
if not isinstance(high_engine, dict):
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶升级未命中，回退通用 high tier："
|
||
f"function={function_name}, reason={str(error)[:120]}"
|
||
),
|
||
)
|
||
high_engine = self._resolve_tier_engine(
|
||
tier="high",
|
||
engines=engines,
|
||
active_engine=active_engine,
|
||
)
|
||
if not isinstance(high_engine, dict):
|
||
return None, None
|
||
high_engine = self._ensure_engine_allowed_for_function(
|
||
engine=high_engine,
|
||
function_name=function_name,
|
||
settings=self._normalize_engine_settings(bundle.get("settings")),
|
||
engine_label="high_tier_failover",
|
||
)
|
||
forced_glm5_engine = self._prefer_glm5_high_tier_retry_engine(
|
||
bundle=bundle,
|
||
engines=engines,
|
||
function_name=function_name,
|
||
failed_engine=failed_engine,
|
||
current_engine=high_engine,
|
||
)
|
||
if isinstance(forced_glm5_engine, dict):
|
||
high_engine = forced_glm5_engine
|
||
failed_provider = str(failed_engine.get("provider") or "").strip().lower()
|
||
failed_model = str(failed_engine.get("model") or "").strip()
|
||
high_provider = str(high_engine.get("provider") or "").strip().lower()
|
||
high_model = str(high_engine.get("model") or "").strip()
|
||
if failed_provider == high_provider and failed_model == high_model:
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"高阶升级引擎与失败引擎相同，跳过："
|
||
f"function={function_name}, provider={high_provider}, model={high_model}"
|
||
),
|
||
)
|
||
return None, None
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"低阶模型失败自动升级："
|
||
f"function={function_name}, from={failed_provider}:{failed_model}, "
|
||
f"to={high_provider}:{high_model}, reason={str(error)[:120]}"
|
||
),
|
||
)
|
||
text = await self._chat_completion(
|
||
provider=high_provider,
|
||
prompt=prompt,
|
||
system_prompt=system_prompt,
|
||
api_key=str(high_engine.get("api_key") or ""),
|
||
base_url=str(high_engine.get("base_url") or ""),
|
||
model=high_model,
|
||
temperature=temperature,
|
||
max_tokens=max_tokens,
|
||
function_name=function_name,
|
||
response_mode=response_mode,
|
||
)
|
||
if not text.strip():
|
||
raise ValueError("high_tier_empty_response")
|
||
if self._is_invalid_structured_output(function_name=function_name, response_text=text):
|
||
raise ValueError("high_tier_invalid_structured_output")
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"高阶升级成功："
|
||
f"function={function_name}, provider={high_provider}, model={high_model}, text_len={len(text)}"
|
||
),
|
||
)
|
||
return text, high_engine
|
||
except Exception as retry_exc:
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"低阶模型失败升级高阶也失败："
|
||
f"function={function_name}, error={type(retry_exc).__name__}:{str(retry_exc)[:180]}"
|
||
),
|
||
)
|
||
return None, None
|
||
|
||
async def _retry_structured_with_high_tier_after_low_model_failure(
|
||
self,
|
||
*,
|
||
db: AsyncSession,
|
||
tenant_id: int,
|
||
function_name: str,
|
||
failed_engine: dict[str, Any],
|
||
prompt: str,
|
||
system_prompt: str,
|
||
temperature: float,
|
||
max_tokens: int,
|
||
response_mode: str,
|
||
error: Exception,
|
||
) -> tuple[Optional[StructuredExtractionResult], Optional[dict[str, Any]]]:
|
||
fn = str(function_name or "").strip().lower()
|
||
force_intent_retry = fn in {"classify_intent_ai_pmo_kernel", "classify_intent_complex"}
|
||
if not force_intent_retry and not self._should_retry_glm47_with_high_tier(
|
||
function_name=function_name,
|
||
failed_engine=failed_engine,
|
||
):
|
||
return None, None
|
||
try:
|
||
_, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id)
|
||
engines = [item for item in bundle.get("engines", []) if isinstance(item, dict)]
|
||
active_engine = self._resolve_active_engine_from_bundle(bundle)
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"低阶结构化失败准备升级高阶："
|
||
f"function={function_name}, from={str(failed_engine.get('provider') or '').strip().lower()}:"
|
||
f"{str(failed_engine.get('model') or '').strip()}, reason={str(error)[:120]}, "
|
||
f"force_intent_retry={force_intent_retry}"
|
||
),
|
||
)
|
||
high_engine = await self._resolve_openclaw_high_tier_engine_for_escalation(
|
||
bundle=bundle,
|
||
engines=engines,
|
||
active_engine=active_engine,
|
||
function_name=function_name,
|
||
failed_engine=failed_engine,
|
||
)
|
||
if not isinstance(high_engine, dict):
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"OpenClaw高阶结构化升级未命中，回退通用 high tier："
|
||
f"function={function_name}, reason={str(error)[:120]}, "
|
||
f"force_intent_retry={force_intent_retry}"
|
||
),
|
||
)
|
||
high_engine = self._resolve_tier_engine(
|
||
tier="high",
|
||
engines=engines,
|
||
active_engine=active_engine,
|
||
)
|
||
if not isinstance(high_engine, dict):
|
||
return None, None
|
||
high_engine = self._ensure_engine_allowed_for_function(
|
||
engine=high_engine,
|
||
function_name=function_name,
|
||
settings=self._normalize_engine_settings(bundle.get("settings")),
|
||
engine_label="high_tier_structured_failover",
|
||
)
|
||
forced_glm5_engine = self._prefer_glm5_high_tier_retry_engine(
|
||
bundle=bundle,
|
||
engines=engines,
|
||
function_name=function_name,
|
||
failed_engine=failed_engine,
|
||
current_engine=high_engine,
|
||
)
|
||
if isinstance(forced_glm5_engine, dict):
|
||
high_engine = forced_glm5_engine
|
||
failed_provider = str(failed_engine.get("provider") or "").strip().lower()
|
||
failed_model = str(failed_engine.get("model") or "").strip()
|
||
high_provider = str(high_engine.get("provider") or "").strip().lower()
|
||
high_model = str(high_engine.get("model") or "").strip()
|
||
if failed_provider == high_provider and failed_model == high_model:
|
||
if not force_intent_retry:
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"高阶结构化升级引擎与失败引擎相同，跳过："
|
||
f"function={function_name}, provider={high_provider}, model={high_model}"
|
||
),
|
||
)
|
||
return None, None
|
||
fallback_engine: Optional[dict[str, Any]] = None
|
||
for fallback_tier in ("safety", "low"):
|
||
candidate = self._resolve_tier_engine(
|
||
tier=fallback_tier,
|
||
engines=engines,
|
||
active_engine=active_engine,
|
||
)
|
||
if not isinstance(candidate, dict):
|
||
continue
|
||
candidate = self._ensure_engine_allowed_for_function(
|
||
engine=candidate,
|
||
function_name=function_name,
|
||
settings=self._normalize_engine_settings(bundle.get("settings")),
|
||
engine_label=f"intent_structured_failover_{fallback_tier}",
|
||
)
|
||
candidate_provider = str(candidate.get("provider") or "").strip().lower()
|
||
candidate_model = str(candidate.get("model") or "").strip()
|
||
if candidate_provider == failed_provider and candidate_model == failed_model:
|
||
continue
|
||
fallback_engine = candidate
|
||
break
|
||
if fallback_engine is None:
|
||
return None, None
|
||
high_engine = fallback_engine
|
||
high_provider = str(high_engine.get("provider") or "").strip().lower()
|
||
high_model = str(high_engine.get("model") or "").strip()
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"低阶结构化失败自动升级："
|
||
f"function={function_name}, from={failed_provider}:{failed_model}, "
|
||
f"to={high_provider}:{high_model}, reason={str(error)[:120]}, "
|
||
f"force_intent_retry={force_intent_retry}"
|
||
),
|
||
)
|
||
completion = await self._chat_completion_raw(
|
||
provider=high_provider,
|
||
prompt=prompt,
|
||
system_prompt=system_prompt,
|
||
api_key=str(high_engine.get("api_key") or ""),
|
||
base_url=str(high_engine.get("base_url") or ""),
|
||
model=high_model,
|
||
temperature=temperature,
|
||
max_tokens=max_tokens,
|
||
function_name=(
|
||
"intent_structured_runtime_failover"
|
||
if force_intent_retry
|
||
else function_name
|
||
),
|
||
response_mode=response_mode,
|
||
)
|
||
extraction = self._extract_structured_object_from_response(
|
||
function_name=function_name,
|
||
response=completion.raw_response,
|
||
fallback_text=completion.text,
|
||
provider=high_provider,
|
||
model=high_model,
|
||
)
|
||
if extraction.payload is None:
|
||
raise ValueError(str(extraction.error_code or "structured_schema_invalid"))
|
||
_runtime_log(
|
||
"LLM结构化",
|
||
(
|
||
f"function={function_name}, structured_source={extraction.source_field or 'none'}, "
|
||
f"validation={extraction.validation}, fallback=true"
|
||
),
|
||
)
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"高阶结构化升级成功："
|
||
f"function={function_name}, provider={high_provider}, model={high_model}, "
|
||
f"structured_source={extraction.source_field or 'none'}"
|
||
),
|
||
)
|
||
return extraction, high_engine
|
||
except Exception as retry_exc:
|
||
_runtime_log(
|
||
"LLM路由",
|
||
(
|
||
"低阶结构化失败升级高阶也失败："
|
||
f"function={function_name}, error={type(retry_exc).__name__}:{str(retry_exc)[:180]}"
|
||
),
|
||
)
|
||
return None, None
|