from __future__ import annotations from .shared import * _CAPABILITY_PROBE_KEYS = ( "basic_chat", "model_self_report", "structured_output", "json_schema", "tool_calling", "streaming", "long_context_32k", "long_context_128k", "vision_input", "audio_input", "private_deployment", "external_search_ready", "long_context", ) _CAPABILITY_PROBE_STATUSES = {"untested", "running", "passed", "failed", "partial"} _CAPABILITY_EVIDENCE_STATUSES = {"verified", "failed", "skipped", "not_probeable", "declared_only"} _LIVE_EVIDENCE_STRENGTHS = {"live_probe", "smoke_probe", "provider_contract", "operator_verified"} _DECLARED_EVIDENCE_STRENGTHS = {"declared_only", "manual", "inferred", "model_self_report"} _CAPABILITY_TARGET_ALIASES = { "basic": ["basic_chat"], "chat": ["basic_chat"], "basic_chat": ["basic_chat"], "model_self_report": ["model_self_report"], "self_report": ["model_self_report"], "model_report": ["model_self_report"], "structured": ["structured_output", "json_schema"], "structured_output": ["structured_output"], "json": ["json_schema"], "json_schema": ["json_schema"], "tool": ["tool_calling"], "tool_calling": ["tool_calling"], "function_calling": ["tool_calling"], "stream": ["streaming"], "streaming": ["streaming"], "long_context": ["long_context"], "long_context_32k": ["long_context_32k"], "long_context_128k": ["long_context_128k"], "vision": ["vision_input"], "vision_input": ["vision_input"], "audio": ["audio_input"], "audio_input": ["audio_input"], } _DECLARED_CAPABILITY_FIELDS = { "structured_output", "json_schema", "tool_calling", "long_context", "max_context_tokens", "vision", "audio", "local_only", "private_deployment", "external_search_ready", "avg_latency_ms", "cost_tier", "reliability_tier", } _DECLARED_PROBE_REQUIREMENT_FLAGS = { "structured_output": "structured_output", "json_schema": "json_schema", "tool_calling": "tool_calling", "long_context": "long_context", "vision_input": "vision", "audio_input": "audio", "private_deployment": "private_deployment", "external_search_ready": "external_search_ready", } _CAPABILITY_DECLARED_FIELDS_META = "capability_declared_fields" _PROVIDER_MODEL_CONTEXT_CONTRACTS = { ("dashscope", "qwen-plus"): { "context_tokens": 1_000_000, "max_input_tokens": 997_952, "source": "aliyun_model_studio_official_spec", "label": "阿里云百炼 Qwen-Plus 官方模型规格", }, } def _provider_model_context_contract(provider: Any, model: Any) -> dict[str, Any] | None: normalized_provider = str(provider or "").strip().lower() normalized_model = str(model or "").strip().lower() if not normalized_provider or not normalized_model: return None exact = _PROVIDER_MODEL_CONTEXT_CONTRACTS.get((normalized_provider, normalized_model)) if exact: return dict(exact) if normalized_provider == "dashscope" and normalized_model.startswith("qwen-plus-"): return dict(_PROVIDER_MODEL_CONTEXT_CONTRACTS[("dashscope", "qwen-plus")]) return None def _explicit_declared_capability_fields(source: dict[str, Any]) -> set[str]: raw_fields = source.get(_CAPABILITY_DECLARED_FIELDS_META) allowed = set(_DECLARED_PROBE_REQUIREMENT_FLAGS.values()) if isinstance(raw_fields, list): return {str(item) for item in raw_fields if str(item) in allowed} return {field for field in allowed if field in source} def _probe_status_declared_scope(declared: dict[str, Any], explicit_fields: set[str]) -> dict[str, Any]: if not explicit_fields: return {} scoped = dict(declared) for declared_flag in _DECLARED_PROBE_REQUIREMENT_FLAGS.values(): if declared_flag not in explicit_fields: scoped[declared_flag] = False return scoped def _empty_capability_probe(source: str = "inferred") -> dict[str, Any]: normalized_source = str(source or "inferred").strip().lower() if normalized_source not in {"inferred", "manual", "probed"}: normalized_source = "inferred" return { "status": "untested", "last_verified_at": None, "verified_capabilities": {key: False for key in _CAPABILITY_PROBE_KEYS}, "evidence": {}, "source": normalized_source, } def _declared_capability_payload(source: Any) -> dict[str, Any]: raw = source if isinstance(source, dict) else {} return {key: raw.get(key) for key in _DECLARED_CAPABILITY_FIELDS if key in raw} def _declared_capability_signature(source: Any, *, provider: str, model: str) -> str: normalized = LLMEngineAdminMixin._normalize_engine_capabilities( _declared_capability_payload(source), provider=provider, model=model, ) declared = normalized.get("declared_capabilities") if isinstance(normalized.get("declared_capabilities"), dict) else {} return json.dumps(declared, ensure_ascii=False, sort_keys=True, default=str) def _evidence_strength_from_row(row: dict[str, Any], *, key: str) -> str: strength = str(row.get("evidence_strength") or "").strip().lower() if strength: return strength source = str(row.get("source") or "").strip().lower() status = str(row.get("status") or "").strip().lower() message = str(row.get("message") or row.get("summary") or "").strip() if source in {"manual", "default", "inferred"}: return source if key in {"long_context", "long_context_32k", "long_context_128k"} and ("声明" in message or "declared" in message.lower()): return "declared_only" if status == "verified" and source in {"probe", "probed"}: return "live_probe" return "unknown" def _probe_row_is_verified(row: dict[str, Any], *, key: str) -> bool: if not isinstance(row, dict): return False status = str(row.get("status") or "").strip().lower() ok = row.get("ok") if ok is not True and status != "verified": return False strength = _evidence_strength_from_row(row, key=key) return strength in _LIVE_EVIDENCE_STRENGTHS def _derive_effective_capabilities(*, declared: dict[str, Any], probe: dict[str, Any]) -> dict[str, bool]: evidence = probe.get("evidence") if isinstance(probe.get("evidence"), dict) else {} def _row_for(key: str) -> dict[str, Any] | None: row = evidence.get(key) if isinstance(row, dict): return row flat = declared.get("capability_evidence") if isinstance(declared.get("capability_evidence"), dict) else {} flat_row = flat.get(key) return flat_row if isinstance(flat_row, dict) else None def _verified(key: str) -> bool: row = _row_for(key) if isinstance(row, dict) and _probe_row_is_verified(row, key=key): return True return False def _verified_min_tokens(key: str, min_tokens: int) -> bool: row = _row_for(key) if not isinstance(row, dict) or not _probe_row_is_verified(row, key=key): return False try: tested_tokens = int(row.get("tested_tokens") or row.get("max_context_tokens") or 0) except (TypeError, ValueError): tested_tokens = 0 return tested_tokens >= min_tokens long_context_verified = _verified("long_context") or _verified("long_context_32k") or _verified("long_context_128k") long_32k_verified = _verified_min_tokens("long_context", 32768) or _verified_min_tokens("long_context_32k", 32768) long_128k_verified = _verified_min_tokens("long_context", 128000) or _verified_min_tokens("long_context_128k", 128000) return { "basic_chat": _verified("basic_chat"), "structured_output": _verified("structured_output"), "json_schema": _verified("json_schema"), "tool_calling": _verified("tool_calling"), "streaming": _verified("streaming"), "long_context": long_context_verified, "long_context_32k": long_32k_verified, "long_context_128k": long_128k_verified, "vision_input": _verified("vision_input"), "audio_input": _verified("audio_input"), "private_deployment": _verified("private_deployment"), "external_search_ready": _verified("external_search_ready"), "vision": _verified("vision_input"), "audio": _verified("audio_input"), } def _capability_evidence_is_in_scope(key: str, row: dict[str, Any], *, declared: dict[str, Any]) -> bool: normalized_key = str(key or "").strip() if normalized_key == "model_self_report": return False if normalized_key == "basic_chat": return True if normalized_key == "structured_output": return bool(declared.get("structured_output")) if normalized_key == "json_schema": return bool(declared.get("json_schema")) if normalized_key == "tool_calling": return bool(declared.get("tool_calling")) if normalized_key == "streaming": return _probe_row_is_verified(row, key=normalized_key) if normalized_key == "long_context": return bool(declared.get("long_context")) or _probe_row_is_verified(row, key=normalized_key) if normalized_key in {"long_context_32k", "long_context_128k"}: try: declared_tokens = int(declared.get("max_context_tokens") or 0) except (TypeError, ValueError): declared_tokens = 0 required_tokens = 128000 if normalized_key == "long_context_128k" else 32768 return (bool(declared.get("long_context")) and declared_tokens >= required_tokens) or _probe_row_is_verified(row, key=normalized_key) if normalized_key == "vision_input": return bool(declared.get("vision")) or _probe_row_is_verified(row, key=normalized_key) if normalized_key == "audio_input": return bool(declared.get("audio")) or _probe_row_is_verified(row, key=normalized_key) return True def _summarize_capability_probe_status( *, declared: dict[str, Any], probe: dict[str, Any], missing_scope_declared: dict[str, Any] | None = None, ) -> str: evidence = probe.get("evidence") if isinstance(probe.get("evidence"), dict) else {} long_context_row = evidence.get("long_context") if isinstance(evidence.get("long_context"), dict) else None covered_long_context_thresholds: set[str] = set() if isinstance(long_context_row, dict) and _probe_row_is_verified(long_context_row, key="long_context"): try: verified_tokens = int(long_context_row.get("tested_tokens") or long_context_row.get("max_context_tokens") or 0) except (TypeError, ValueError): verified_tokens = 0 if verified_tokens >= 32768: covered_long_context_thresholds.add("long_context_32k") if verified_tokens >= 128000: covered_long_context_thresholds.add("long_context_128k") scoped = { str(key): value for key, value in evidence.items() if isinstance(value, dict) and str(key) not in covered_long_context_thresholds and _capability_evidence_is_in_scope(str(key), value, declared=declared) } placeholder_scope = missing_scope_declared if isinstance(missing_scope_declared, dict) else declared for key, declared_flag in _DECLARED_PROBE_REQUIREMENT_FLAGS.items(): if bool(placeholder_scope.get(declared_flag)) and key not in scoped: scoped[key] = { "status": "declared_only", "ok": None, "evidence_strength": "declared_only", "summary": "已声明但尚未获得真实探测或人工确认依据", } if not scoped: return "untested" verified = sum(1 for key, value in scoped.items() if _probe_row_is_verified(value, key=key)) failed = sum( 1 for value in scoped.values() if str(value.get("status") or "").strip().lower() == "failed" or (value.get("ok") is False and str(value.get("status") or "").strip().lower() not in {"skipped", "not_probeable", "declared_only"}) ) neutral = len(scoped) - verified - failed if verified and not failed and not neutral: return "passed" if verified and (failed or neutral): return "partial" if failed: return "failed" return "untested" def _normalize_legacy_long_context_probe(*, declared: dict[str, Any], probe: dict[str, Any], flat_evidence: dict[str, Any]) -> None: if not bool(declared.get("long_context")): return try: declared_tokens = int(declared.get("max_context_tokens") or 0) except (TypeError, ValueError): declared_tokens = 0 if declared_tokens <= 0: return evidence = probe.get("evidence") if isinstance(probe.get("evidence"), dict) else {} row = evidence.get("long_context") if not isinstance(row, dict): return status = str(row.get("status") or "").strip().lower() failure_code = str(row.get("failure_code") or "").strip() text = " ".join(str(row.get(key) or "") for key in ("summary", "error_summary", "request_summary", "response_summary")) if status != "failed" or failure_code != "declared_window_too_small" or "32768" not in text: return message = f"长上下文真实 smoke 需要高成本调用，本次只记录声明窗口 {declared_tokens} tokens，不作为槽位发布 verified 证据" normalized_row = dict(row) normalized_row.update( { "ok": False, "status": "declared_only", "summary": message, "error_summary": "", "evidence_strength": "declared_only", "failure_code": None, "tested_tokens": declared_tokens, } ) evidence["long_context"] = normalized_row flat_evidence["long_context"] = { **dict(flat_evidence.get("long_context") or {}), "status": "declared_only", "source": "probe", "message": message, "evidence_strength": "declared_only", "failure_code": None, "max_context_tokens": declared_tokens, } def _probe_source_from_flat(source: Any) -> str: normalized = str(source or "").strip().lower() if normalized in {"probe", "mixed", "probed"}: return "probed" if normalized == "manual": return "manual" return "inferred" def _flat_source_from_probe(source: Any, *, fallback: str = "default") -> str: normalized = str(source or "").strip().lower() if normalized == "probed": return "probe" if normalized == "manual": return "manual" return fallback if fallback in {"default", "manual", "probe", "mixed"} else "default" def _normalize_capability_probe( raw: Any, *, flat_evidence: dict[str, Any], flat_status: Any, flat_last_verified_at: Any, flat_source: Any, ) -> dict[str, Any]: source = raw if isinstance(raw, dict) else {} probe = _empty_capability_probe(_probe_source_from_flat(flat_source)) raw_source = str(source.get("source") or "").strip().lower() if raw_source in {"inferred", "manual", "probed"}: probe["source"] = raw_source raw_status = str(source.get("status") or flat_status or "").strip().lower() if raw_status in _CAPABILITY_PROBE_STATUSES: probe["status"] = raw_status raw_last_verified_at = str(source.get("last_verified_at") or flat_last_verified_at or "").strip() probe["last_verified_at"] = raw_last_verified_at or None verified_capabilities = source.get("verified_capabilities") if isinstance(source.get("verified_capabilities"), dict) else {} for key in _CAPABILITY_PROBE_KEYS: probe["verified_capabilities"][key] = bool(verified_capabilities.get(key, False)) nested_evidence = source.get("evidence") if isinstance(source.get("evidence"), dict) else {} evidence_keys = list(dict.fromkeys([*_CAPABILITY_PROBE_KEYS, *nested_evidence.keys(), *flat_evidence.keys()])) normalized_evidence: dict[str, dict[str, Any]] = {} for key in evidence_keys: nested_row = nested_evidence.get(key) if isinstance(nested_evidence.get(key), dict) else {} flat_row = flat_evidence.get(key) if isinstance(flat_evidence.get(key), dict) else {} if not nested_row and not flat_row: continue raw_status = str(nested_row.get("status") or flat_row.get("status") or "").strip().lower() ok_value = nested_row.get("ok") if not isinstance(ok_value, bool): ok_value = raw_status == "verified" checked_at = str(nested_row.get("checked_at") or flat_row.get("verified_at") or probe["last_verified_at"] or "").strip() or None summary = str(nested_row.get("summary") or flat_row.get("message") or "").strip() error_summary = str(nested_row.get("error_summary") or ("" if ok_value else flat_row.get("message") or "")).strip() evidence_strength = _evidence_strength_from_row({**flat_row, **nested_row, "status": raw_status}, key=str(key)) if evidence_strength == "unknown" and ok_value is True and probe.get("source") == "probed": message = str(summary or error_summary or "") evidence_strength = ( "declared_only" if str(key) in {"long_context", "long_context_32k", "long_context_128k"} and ("声明" in message or "declared" in message.lower()) else "live_probe" ) row = { "ok": bool(ok_value), "status": raw_status if raw_status in _CAPABILITY_EVIDENCE_STATUSES else ("verified" if ok_value else "failed"), "method": str(nested_row.get("method") or flat_row.get("method") or "").strip() or None, "evidence_strength": evidence_strength, "checked_at": checked_at, "summary": summary[:300], "error_summary": error_summary[:300], "failure_code": str(nested_row.get("failure_code") or flat_row.get("failure_code") or "").strip() or None, "request_summary": str(nested_row.get("request_summary") or flat_row.get("request_summary") or "").strip()[:500], "response_summary": str(nested_row.get("response_summary") or flat_row.get("response_summary") or "").strip()[:500], "latency_ms": nested_row.get("latency_ms", flat_row.get("latency_ms")), "source": str(nested_row.get("source") or flat_row.get("source") or "").strip() or None, } for passthrough_key in ("source_type", "source_url", "source_note", "verified_by", "verified_at", "expires_at"): passthrough_value = nested_row.get(passthrough_key, flat_row.get(passthrough_key)) if passthrough_value not in (None, ""): row[passthrough_key] = passthrough_value if key in {"long_context", "long_context_32k", "long_context_128k"}: tested_tokens = nested_row.get("tested_tokens", flat_row.get("max_context_tokens")) try: row["tested_tokens"] = int(tested_tokens) if tested_tokens is not None and tested_tokens != "" else None except (TypeError, ValueError): row["tested_tokens"] = None model_self_report = nested_row.get("model_self_report", flat_row.get("model_self_report")) if str(key) == "model_self_report" and isinstance(model_self_report, dict): row["model_self_report"] = model_self_report normalized_evidence[key] = row probe["verified_capabilities"][key] = _probe_row_is_verified(row, key=str(key)) probe["evidence"] = normalized_evidence if probe["status"] == "untested" and normalized_evidence: ok_count = sum(1 for item in normalized_evidence.values() if item.get("ok") is True) failed_count = sum(1 for item in normalized_evidence.values() if item.get("ok") is not True) probe["status"] = "passed" if ok_count and not failed_count else ("partial" if ok_count else "failed") return probe def _flat_evidence_from_probe(probe: dict[str, Any]) -> dict[str, dict[str, Any]]: evidence = probe.get("evidence") if isinstance(probe.get("evidence"), dict) else {} flattened: dict[str, dict[str, Any]] = {} for key, value in evidence.items(): if not isinstance(value, dict): continue flattened[str(key)] = { "status": str(value.get("status") or ("verified" if value.get("ok") is True else "failed")), "source": value.get("source") or "probe", "verified_at": value.get("checked_at"), "message": str(value.get("summary") or value.get("error_summary") or "")[:300], "latency_ms": value.get("latency_ms"), "method": value.get("method"), "evidence_strength": value.get("evidence_strength"), "failure_code": value.get("failure_code"), "request_summary": value.get("request_summary"), "response_summary": value.get("response_summary"), } if value.get("tested_tokens") is not None: flattened[str(key)]["max_context_tokens"] = value.get("tested_tokens") for passthrough_key in ("source_type", "source_url", "source_note", "verified_by", "expires_at"): if value.get(passthrough_key) not in (None, ""): flattened[str(key)][passthrough_key] = value.get(passthrough_key) if str(key) == "model_self_report" and isinstance(value.get("model_self_report"), dict): flattened[str(key)]["model_self_report"] = value.get("model_self_report") return flattened class LLMEngineAdminMixin: _PROBED_CAPABILITY_KEYS = frozenset(_CAPABILITY_PROBE_KEYS) @staticmethod def _default_engine_capabilities(provider: str, model: str = "") -> dict[str, Any]: normalized_provider = str(provider or "").strip().lower() normalized_model = str(model or "").strip().lower() is_local = normalized_provider == "ollama" is_high_reasoning = normalized_provider in {"openai", "deepseek"} or "glm-5" in normalized_model is_dashscope = normalized_provider == "dashscope" context_contract = _provider_model_context_contract(normalized_provider, normalized_model) default_context_tokens = ( int(context_contract["context_tokens"]) if context_contract else (8192 if is_local else (128000 if is_high_reasoning else 32768)) ) return { "structured_output": not is_local, "json_schema": normalized_provider in {"openai", "deepseek"} or is_dashscope, "tool_calling": normalized_provider in {"openai", "deepseek", "dashscope"}, "long_context": not is_local, "max_context_tokens": default_context_tokens, "vision": False, "audio": False, "local_only": is_local, "private_deployment": is_local, "external_search_ready": False, "avg_latency_ms": None, "cost_tier": "low" if normalized_provider in {"dashscope", "ollama"} else "medium", "reliability_tier": "local" if is_local else ("high" if is_high_reasoning else "standard"), "capability_source": "default", "capability_probe_status": "untested", "capability_last_verified_at": None, "capability_blocking_reasons": [], "capability_evidence": {}, "capability_probe": _empty_capability_probe("inferred"), "declared_capabilities": {}, "effective_capabilities": {}, } @classmethod def _normalize_engine_capabilities(cls, raw: Any, *, provider: str, model: str = "") -> dict[str, Any]: defaults = cls._default_engine_capabilities(provider, model) source = raw if isinstance(raw, dict) else {} def _bool(key: str) -> bool: return bool(source[key]) if key in source else bool(defaults[key]) def _int_or_none(key: str) -> int | None: value = source.get(key, defaults.get(key)) if value is None or value == "": return None try: return max(0, int(value)) except (TypeError, ValueError): return defaults.get(key) def _tier(key: str, allowed: set[str]) -> str: value = str(source.get(key) or defaults.get(key) or "").strip().lower() return value if value in allowed else str(defaults.get(key) or "") evidence = source.get("capability_evidence") if isinstance(source.get("capability_evidence"), dict) else {} blocking_reasons = source.get("capability_blocking_reasons") if not isinstance(blocking_reasons, list): blocking_reasons = [] metadata_keys = { "capability_source", "capability_probe_status", "capability_last_verified_at", "capability_blocking_reasons", "capability_evidence", "capability_probe", "declared_capabilities", "effective_capabilities", _CAPABILITY_DECLARED_FIELDS_META, } has_saved_capability_fields = any( key in source for key in defaults if key not in metadata_keys ) capability_source = str(source.get("capability_source") or "").strip().lower() if capability_source not in {"default", "manual", "probe", "mixed"}: capability_source = "manual" if has_saved_capability_fields else "default" probe_status = str(source.get("capability_probe_status") or "").strip().lower() if probe_status not in {"untested", "running", "passed", "partial", "failed"}: probe_status = "untested" capability_probe = _normalize_capability_probe( source.get("capability_probe"), flat_evidence=evidence, flat_status=probe_status, flat_last_verified_at=source.get("capability_last_verified_at"), flat_source=capability_source, ) declared_capabilities = { "structured_output": _bool("structured_output"), "json_schema": _bool("json_schema"), "tool_calling": _bool("tool_calling"), "long_context": _bool("long_context"), "max_context_tokens": _int_or_none("max_context_tokens"), "vision": _bool("vision"), "audio": _bool("audio"), "local_only": _bool("local_only"), "private_deployment": _bool("private_deployment"), "external_search_ready": _bool("external_search_ready"), "avg_latency_ms": _int_or_none("avg_latency_ms"), "cost_tier": _tier("cost_tier", {"low", "medium", "high", "unknown"}), "reliability_tier": _tier("reliability_tier", {"standard", "high", "local", "unknown"}), } explicit_declared_fields = _explicit_declared_capability_fields(source) _normalize_legacy_long_context_probe(declared=declared_capabilities, probe=capability_probe, flat_evidence=evidence) if not evidence and capability_probe.get("evidence"): evidence = _flat_evidence_from_probe(capability_probe) if not blocking_reasons: blocking_reasons = [ f"{key}: {str(value.get('error_summary') or '')[:180]}" for key, value in (capability_probe.get("evidence") or {}).items() if ( isinstance(value, dict) and _capability_evidence_is_in_scope(str(key), value, declared=declared_capabilities) and value.get("ok") is not True and str(value.get("error_summary") or "").strip() ) ] missing_scope_declared = _probe_status_declared_scope(declared_capabilities, explicit_declared_fields) scoped_probe_status = _summarize_capability_probe_status( declared=declared_capabilities, probe=capability_probe, missing_scope_declared=missing_scope_declared, ) if scoped_probe_status != "untested" or probe_status == "untested": probe_status = scoped_probe_status capability_probe["status"] = probe_status last_verified_at = str(source.get("capability_last_verified_at") or capability_probe.get("last_verified_at") or "").strip() or None if capability_source == "default" and capability_probe.get("source") in {"manual", "probed"}: capability_source = _flat_source_from_probe(capability_probe.get("source"), fallback=capability_source) normalized = { **declared_capabilities, "capability_source": capability_source, "capability_probe_status": probe_status, "capability_last_verified_at": last_verified_at, "capability_blocking_reasons": [str(item)[:220] for item in blocking_reasons if str(item or "").strip()][:8], "capability_evidence": { str(key): value for key, value in evidence.items() if str(key or "").strip() and isinstance(value, dict) }, "capability_probe": capability_probe, "declared_capabilities": { "structured_output": declared_capabilities["structured_output"], "json_schema": declared_capabilities["json_schema"], "tool_calling": declared_capabilities["tool_calling"], "long_context": declared_capabilities["long_context"], "max_context_tokens": declared_capabilities["max_context_tokens"], "vision": declared_capabilities["vision"], "audio": declared_capabilities["audio"], "private_deployment": declared_capabilities["private_deployment"], "external_search_ready": declared_capabilities["external_search_ready"], }, _CAPABILITY_DECLARED_FIELDS_META: sorted(explicit_declared_fields), } normalized["effective_capabilities"] = _derive_effective_capabilities(declared=normalized, probe=capability_probe) return normalized @classmethod def _merge_capability_probe_results( cls, *, capabilities: dict[str, Any], results: dict[str, dict[str, Any]], verified_at: str, ) -> dict[str, Any]: merged = cls._normalize_engine_capabilities( capabilities, provider=str(capabilities.get("_provider") or ""), model=str(capabilities.get("_model") or ""), ) evidence = dict(merged.get("capability_evidence") or {}) capability_probe = _normalize_capability_probe( merged.get("capability_probe"), flat_evidence=evidence, flat_status=merged.get("capability_probe_status"), flat_last_verified_at=merged.get("capability_last_verified_at"), flat_source=merged.get("capability_source"), ) probe_evidence = dict(capability_probe.get("evidence") or {}) verified_capabilities = dict(capability_probe.get("verified_capabilities") or {}) blocking_reasons: list[str] = [] verified_count = 0 failed_count = 0 partial_evidence_count = 0 neutral_count = 0 aggregate_result_count = 0 for key, result in results.items(): normalized_key = str(key or "").strip() status = str(result.get("status") or "").strip().lower() ok = status == "verified" evidence_status = status if status in _CAPABILITY_EVIDENCE_STATUSES else ("verified" if ok else "failed") summary = str(result.get("message") or "")[:300] evidence_strength = str(result.get("evidence_strength") or ("live_probe" if ok else "unknown")).strip().lower() evidence[normalized_key] = { "status": evidence_status, "source": "probe", "verified_at": verified_at, "message": summary, "latency_ms": result.get("latency_ms"), "method": result.get("method"), "evidence_strength": evidence_strength, "failure_code": result.get("failure_code"), "request_summary": result.get("request_summary"), "response_summary": result.get("response_summary"), } if result.get("max_context_tokens") is not None: evidence[normalized_key]["max_context_tokens"] = result.get("max_context_tokens") if result.get("model_self_report") is not None: evidence[normalized_key]["model_self_report"] = result.get("model_self_report") probe_row = { "ok": ok, "status": evidence_status, "method": result.get("method"), "evidence_strength": evidence_strength, "checked_at": verified_at, "summary": summary if ok else "", "error_summary": "" if ok else summary, "failure_code": result.get("failure_code"), "request_summary": str(result.get("request_summary") or "")[:500], "response_summary": str(result.get("response_summary") or "")[:500], "latency_ms": result.get("latency_ms"), } if normalized_key in {"long_context", "long_context_32k", "long_context_128k"}: probe_row["tested_tokens"] = result.get("max_context_tokens") if normalized_key == "long_context": probe_evidence["long_context"] = dict(probe_row) verified_capabilities["long_context"] = bool(ok and evidence_strength in _LIVE_EVIDENCE_STRENGTHS) if result.get("model_self_report") is not None: probe_row["model_self_report"] = result.get("model_self_report") probe_evidence[normalized_key] = probe_row verified_capabilities[normalized_key] = bool(ok and evidence_strength in _LIVE_EVIDENCE_STRENGTHS) if normalized_key in {"structured_output", "json_schema"}: merged[normalized_key] = ok or bool(merged.get(normalized_key)) elif normalized_key == "tool_calling": merged["tool_calling"] = ok or bool(merged.get("tool_calling")) elif normalized_key in {"long_context", "long_context_32k", "long_context_128k"}: merged["long_context"] = bool(merged.get("long_context")) or bool(result.get("declared_ok")) if result.get("max_context_tokens"): try: merged["max_context_tokens"] = max(int(merged.get("max_context_tokens") or 0), int(result["max_context_tokens"])) except (TypeError, ValueError): pass if normalized_key == "model_self_report": continue aggregate_result_count += 1 if ok: verified_count += 1 elif evidence_status in {"skipped", "not_probeable", "declared_only"}: neutral_count += 1 else: failed_count += 1 message = str(result.get("message") or "").strip() if message: blocking_reasons.append(f"{normalized_key}: {message[:180]}") if bool(result.get("partial_evidence")) or evidence_strength in _DECLARED_EVIDENCE_STRENGTHS: partial_evidence_count += 1 merged["capability_evidence"] = evidence merged["capability_source"] = "probe" if verified_count and not failed_count else ("mixed" if verified_count else "probe") merged["capability_probe_status"] = ( "passed" if aggregate_result_count and verified_count and not failed_count and not partial_evidence_count and not neutral_count else ("partial" if (aggregate_result_count and (verified_count or partial_evidence_count or neutral_count)) or (results and not aggregate_result_count) else "failed") ) merged["capability_last_verified_at"] = verified_at merged["capability_blocking_reasons"] = blocking_reasons[:8] merged["capability_probe"] = { "status": merged["capability_probe_status"], "last_verified_at": verified_at, "verified_capabilities": {key: bool(verified_capabilities.get(key)) for key in _CAPABILITY_PROBE_KEYS}, "evidence": probe_evidence, "source": "probed", } merged["declared_capabilities"] = { "structured_output": bool(merged.get("structured_output")), "json_schema": bool(merged.get("json_schema")), "tool_calling": bool(merged.get("tool_calling")), "long_context": bool(merged.get("long_context")), "max_context_tokens": merged.get("max_context_tokens"), "vision": bool(merged.get("vision")), "audio": bool(merged.get("audio")), "private_deployment": bool(merged.get("private_deployment")), "external_search_ready": bool(merged.get("external_search_ready")), } explicit_declared_fields = _explicit_declared_capability_fields(merged) merged[_CAPABILITY_DECLARED_FIELDS_META] = sorted(explicit_declared_fields) summarized_status = _summarize_capability_probe_status( declared=merged, probe=merged["capability_probe"], missing_scope_declared=_probe_status_declared_scope(merged, explicit_declared_fields), ) if summarized_status != "untested": merged["capability_probe_status"] = summarized_status merged["capability_probe"]["status"] = summarized_status merged["effective_capabilities"] = _derive_effective_capabilities(declared=merged, probe=merged["capability_probe"]) return merged def _sanitize_engine_for_admin_response(self, item: dict[str, Any]) -> dict[str, Any]: sanitized = self._sanitize_engine_for_response(item) sanitized["capabilities"] = self._normalize_engine_capabilities( item.get("capabilities"), provider=str(item.get("provider") or ""), model=str(item.get("model") or ""), ) return sanitized def _engine_action_permissions(self, item: dict[str, Any], *, default_fallback_engine_id: str | None) -> dict[str, bool]: enabled = self._engine_enabled(item) connection_available = str(item.get("connection_status") or "untested").strip().lower() == "available" is_default_fallback = str(item.get("id") or "") == str(default_fallback_engine_id or "") return { "enable": not enabled, "disable": enabled, "edit": True, "test": enabled, "capability_probe": enabled and connection_available, "set_default_fallback": enabled and connection_available and not is_default_fallback, "delete": (not enabled) and not is_default_fallback, } async def list_engines(self, db: AsyncSession, tenant_id: int = 1) -> dict[str, Any]: _, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) engines = bundle.get("engines", []) default_fallback_engine_id = bundle.get("default_fallback_engine_id") or bundle.get("active_engine_id") return { "active_engine_id": default_fallback_engine_id, "default_fallback_engine_id": default_fallback_engine_id, "engines": [ { **self._sanitize_engine_for_admin_response(item), "is_default_fallback": str(item.get("id") or "") == str(default_fallback_engine_id or ""), "available_actions": self._engine_action_permissions( item, default_fallback_engine_id=str(default_fallback_engine_id or ""), ), } for item in engines ], "settings": self._normalize_engine_settings(bundle.get("settings")), } async def update_engine_settings( self, db: AsyncSession, *, tenant_id: int, enable_local_kb_qa: Optional[bool] = None, ) -> dict[str, Any]: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) settings = self._normalize_engine_settings(bundle.get("settings")) if enable_local_kb_qa is not None: settings["enable_local_kb_qa"] = bool(enable_local_kb_qa) bundle["settings"] = settings integration.config = json.loads(json.dumps(bundle)) await db.commit() return settings async def create_engine( self, db: AsyncSession, *, tenant_id: int, name: str, provider: str, api_key: Optional[str], model: Optional[str] = None, base_url: Optional[str] = None, set_active: bool = True, capabilities: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: provider = _normalize_provider(provider) api_key = str(api_key or "").strip() if provider != "ollama" and not api_key: raise ValueError("api_key 不能为空") integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) now = now_cn_aware().isoformat() engine_id = str(uuid4()) defaults = PROVIDER_DEFAULTS[provider] engine = { "id": engine_id, "name": (name or defaults["name"]).strip()[:80] or defaults["name"], "provider": provider, "api_key": api_key, "model": (model or defaults["model"]).strip() or defaults["model"], "base_url": (base_url or defaults["base_url"]).strip().rstrip("/") or defaults["base_url"], "enabled": True, "connection_status": "untested", "last_test_status": "", "last_test_at": None, "last_error_summary": "", "capabilities": self._normalize_engine_capabilities( _declared_capability_payload(capabilities), provider=provider, model=(model or defaults["model"]).strip() or defaults["model"], ), "created_at": now, "updated_at": now, } engines = bundle.get("engines", []) engines.append(engine) bundle["engines"] = engines if set_active or not bundle.get("default_fallback_engine_id") or not bundle.get("active_engine_id"): bundle["default_fallback_engine_id"] = engine_id bundle["active_engine_id"] = engine_id integration.config = json.loads(json.dumps(bundle)) integration.is_active = True await db.commit() return self._sanitize_engine_for_admin_response(engine) async def update_engine( self, db: AsyncSession, *, tenant_id: int, engine_id: str, name: Optional[str] = None, provider: Optional[str] = None, api_key: Optional[str] = None, model: Optional[str] = None, base_url: Optional[str] = None, capabilities: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) engines = bundle.get("engines", []) target: Optional[dict[str, Any]] = None for item in engines: if str(item.get("id")) == str(engine_id): target = item break if not target: raise ValueError("engine 不存在") old_provider = _normalize_provider(target.get("provider")) old_model = str(target.get("model") or "").strip() old_base_url = str(target.get("base_url") or "").strip().rstrip("/") old_capability_signature = _declared_capability_signature( target.get("capabilities"), provider=old_provider, model=old_model, ) evidence_invalidated = False if provider is not None: new_provider = _normalize_provider(provider) if new_provider != old_provider: evidence_invalidated = True target["connection_status"] = "untested" target["last_test_status"] = "" target["last_error_summary"] = "" target["provider"] = new_provider active_provider = _normalize_provider(target.get("provider")) defaults = PROVIDER_DEFAULTS[active_provider] if name is not None: target["name"] = (name or "").strip()[:80] or target.get("name") or defaults["name"] if api_key is not None: api_key_val = str(api_key).strip() if api_key_val: target["api_key"] = api_key_val target["connection_status"] = "untested" target["last_test_status"] = "" target["last_error_summary"] = "" evidence_invalidated = True if model is not None: next_model = (model or "").strip() or defaults["model"] if next_model != str(target.get("model") or ""): evidence_invalidated = True target["connection_status"] = "untested" target["last_test_status"] = "" target["last_error_summary"] = "" target["model"] = next_model if base_url is not None: next_base_url = (base_url or "").strip().rstrip("/") or defaults["base_url"] if next_base_url != old_base_url: evidence_invalidated = True target["connection_status"] = "untested" target["last_test_status"] = "" target["last_error_summary"] = "" target["base_url"] = next_base_url if capabilities is not None: declared_payload = _declared_capability_payload(capabilities) new_capability_signature = _declared_capability_signature( declared_payload, provider=str(target.get("provider") or ""), model=str(target.get("model") or ""), ) evidence_invalidated = evidence_invalidated or new_capability_signature != old_capability_signature target["capabilities"] = self._normalize_engine_capabilities( declared_payload, provider=str(target.get("provider") or ""), model=str(target.get("model") or ""), ) elif evidence_invalidated: target["capabilities"] = self._normalize_engine_capabilities( _declared_capability_payload(target.get("capabilities")), provider=str(target.get("provider") or ""), model=str(target.get("model") or ""), ) target["updated_at"] = now_cn_aware().isoformat() integration.config = json.loads(json.dumps(bundle)) await db.commit() return self._sanitize_engine_for_admin_response(target) async def delete_engine( self, db: AsyncSession, *, tenant_id: int, engine_id: str, ) -> dict[str, Any]: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) engines = bundle.get("engines", []) target = next((item for item in engines if str(item.get("id")) == str(engine_id)), None) if not target: raise ValueError("engine 不存在") default_fallback_engine_id = str(bundle.get("default_fallback_engine_id") or bundle.get("active_engine_id") or "") if default_fallback_engine_id == str(engine_id): raise ValueError("兜底默认供应商连接不能删除，请先切换兜底或停用后重试") if self._engine_enabled(target): raise ValueError("启用中的供应商连接不能删除，请先停用后再删除") kept = [item for item in engines if str(item.get("id")) != str(engine_id)] bundle["engines"] = kept integration.config = json.loads(json.dumps(bundle)) await db.commit() return { "deleted_engine_id": str(engine_id), "active_engine_id": bundle.get("default_fallback_engine_id"), "default_fallback_engine_id": bundle.get("default_fallback_engine_id"), "remaining_count": len(kept), } async def set_default_fallback_engine( self, db: AsyncSession, *, tenant_id: int, engine_id: str, ) -> dict[str, Any]: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) engines = bundle.get("engines", []) target = next((item for item in engines if str(item.get("id")) == str(engine_id)), None) if not target: raise ValueError("engine 不存在") if not self._engine_enabled(target): raise ValueError("已停用的供应商连接不能设为兜底默认") connection_status = str(target.get("connection_status") or "untested").strip().lower() if connection_status != "available": raise ValueError("未通过连接测试的供应商连接不能设为兜底默认") bundle["default_fallback_engine_id"] = str(engine_id) bundle["active_engine_id"] = str(engine_id) integration.config = json.loads(json.dumps(bundle)) await db.commit() return {"active_engine_id": str(engine_id), "default_fallback_engine_id": str(engine_id)} async def activate_engine( self, db: AsyncSession, *, tenant_id: int, engine_id: str, ) -> dict[str, Any]: return await self.set_default_fallback_engine(db, tenant_id=tenant_id, engine_id=engine_id) async def set_engine_enabled( self, db: AsyncSession, *, tenant_id: int, engine_id: str, enabled: bool, ) -> dict[str, Any]: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) engines = bundle.get("engines", []) target: Optional[dict[str, Any]] = None for item in engines: if str(item.get("id")) == str(engine_id): target = item break if not target: raise ValueError("engine 不存在") target["enabled"] = bool(enabled) target["updated_at"] = now_cn_aware().isoformat() if enabled: if not bundle.get("default_fallback_engine_id"): bundle["default_fallback_engine_id"] = str(engine_id) bundle["active_engine_id"] = str(engine_id) else: default_fallback_engine_id = str(bundle.get("default_fallback_engine_id") or bundle.get("active_engine_id") or "") if default_fallback_engine_id == str(engine_id): replacement = next( (item.get("id") for item in engines if str(item.get("id")) != str(engine_id) and self._engine_enabled(item)), None, ) bundle["default_fallback_engine_id"] = replacement bundle["active_engine_id"] = replacement integration.config = json.loads(json.dumps(bundle)) await db.commit() return { "engine_id": str(engine_id), "enabled": bool(enabled), "active_engine_id": bundle.get("default_fallback_engine_id"), "default_fallback_engine_id": bundle.get("default_fallback_engine_id"), } async def _record_engine_test_result( self, db: AsyncSession, *, tenant_id: int, engine_id: str, ok: bool, error_summary: str = "", ) -> None: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) now = now_cn_aware().isoformat() for item in bundle.get("engines", []): if isinstance(item, dict) and str(item.get("id") or "") == str(engine_id): item["connection_status"] = "available" if ok else "test_failed" item["last_test_status"] = "success" if ok else "failed" item["last_test_at"] = now item["last_error_summary"] = "" if ok else str(error_summary or "")[:220] item["updated_at"] = now break integration.config = json.loads(json.dumps(bundle, ensure_ascii=False)) await db.commit() async def _record_engine_capability_probe_result( self, db: AsyncSession, *, tenant_id: int, engine_id: str, results: dict[str, dict[str, Any]], ) -> dict[str, Any]: integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) now = now_cn_aware().isoformat() for item in bundle.get("engines", []): if not isinstance(item, dict) or str(item.get("id") or "") != str(engine_id): continue raw_capabilities = dict(item.get("capabilities") or {}) raw_capabilities["_provider"] = str(item.get("provider") or "") raw_capabilities["_model"] = str(item.get("model") or "") item["capabilities"] = self._merge_capability_probe_results( capabilities=raw_capabilities, results=results, verified_at=now, ) item["updated_at"] = now integration.config = json.loads(json.dumps(bundle, ensure_ascii=False)) await db.commit() return self._sanitize_engine_for_admin_response(item) raise ValueError("engine 不存在") async def attest_engine_capability( self, db: AsyncSession, *, tenant_id: int, engine_id: str, capability_key: str, tested_tokens: Optional[int] = None, source_type: str = "manual_verification", source_url: Optional[str] = None, source_note: Optional[str] = None, expires_at: Optional[str] = None, user_id: Optional[int] = None, ) -> dict[str, Any]: normalized_key = str(capability_key or "").strip() if normalized_key not in {"long_context"}: raise ValueError("当前仅支持人工确认长上下文能力") token_count = int(tested_tokens or 0) if token_count <= 0: raise ValueError("长上下文人工确认必须填写有效 tokens") integration, bundle = await self._get_or_init_bundle(db, tenant_id=tenant_id) now = now_cn_aware().isoformat() for item in bundle.get("engines", []): if not isinstance(item, dict) or str(item.get("id") or "") != str(engine_id): continue raw_capabilities = dict(item.get("capabilities") or {}) capabilities = self._normalize_engine_capabilities( raw_capabilities, provider=str(item.get("provider") or ""), model=str(item.get("model") or ""), ) capabilities["long_context"] = True capabilities["max_context_tokens"] = max(int(capabilities.get("max_context_tokens") or 0), token_count) capabilities["capability_source"] = "mixed" capabilities["capability_last_verified_at"] = now capabilities["capability_blocking_reasons"] = [ reason for reason in (capabilities.get("capability_blocking_reasons") or []) if "long_context" not in str(reason) and "长上下文" not in str(reason) ] capability_evidence = dict(capabilities.get("capability_evidence") or {}) capability_probe = _normalize_capability_probe( capabilities.get("capability_probe"), flat_evidence=capability_evidence, flat_status=capabilities.get("capability_probe_status"), flat_last_verified_at=capabilities.get("capability_last_verified_at"), flat_source=capabilities.get("capability_source"), ) probe_evidence = dict(capability_probe.get("evidence") or {}) verified_capabilities = dict(capability_probe.get("verified_capabilities") or {}) summary = f"人工确认长上下文窗口 {token_count} tokens" attestation = { "source_type": str(source_type or "manual_verification").strip()[:80] or "manual_verification", "source_url": str(source_url or "").strip()[:500], "source_note": str(source_note or "").strip()[:500], "verified_by": user_id, "verified_at": now, "expires_at": str(expires_at or "").strip()[:80], } flat_row = { "status": "verified", "source": "operator", "verified_at": now, "message": summary, "latency_ms": None, "method": "operator_attestation", "evidence_strength": "operator_verified", "failure_code": None, "max_context_tokens": token_count, "request_summary": "", "response_summary": "", **{key: value for key, value in attestation.items() if value not in (None, "")}, } probe_row = { "ok": True, "status": "verified", "method": "operator_attestation", "evidence_strength": "operator_verified", "checked_at": now, "summary": summary, "error_summary": "", "failure_code": None, "request_summary": "", "response_summary": "", "latency_ms": None, "tested_tokens": token_count, "source": "operator", **{key: value for key, value in attestation.items() if value not in (None, "")}, } for threshold_key, threshold_tokens in (("long_context_32k", 32768), ("long_context_128k", 128000)): if token_count >= threshold_tokens: capability_evidence.pop(threshold_key, None) probe_evidence.pop(threshold_key, None) capability_evidence["long_context"] = flat_row probe_evidence["long_context"] = probe_row verified_capabilities["long_context"] = True verified_capabilities["long_context_32k"] = token_count >= 32768 verified_capabilities["long_context_128k"] = token_count >= 128000 capability_probe["evidence"] = probe_evidence capability_probe["verified_capabilities"] = {key: bool(verified_capabilities.get(key)) for key in _CAPABILITY_PROBE_KEYS} capability_probe["last_verified_at"] = now capability_probe["source"] = "manual" explicit_declared_fields = _explicit_declared_capability_fields(raw_capabilities) explicit_declared_fields.add("long_context") capability_probe["status"] = _summarize_capability_probe_status( declared=capabilities, probe=capability_probe, missing_scope_declared=_probe_status_declared_scope(capabilities, explicit_declared_fields), ) capabilities["capability_evidence"] = capability_evidence capabilities["capability_probe"] = capability_probe capabilities["capability_probe_status"] = capability_probe["status"] capabilities[_CAPABILITY_DECLARED_FIELDS_META] = sorted(explicit_declared_fields) capabilities["effective_capabilities"] = _derive_effective_capabilities(declared=capabilities, probe=capability_probe) item["capabilities"] = self._normalize_engine_capabilities( capabilities, provider=str(item.get("provider") or ""), model=str(item.get("model") or ""), ) item["updated_at"] = now integration.config = json.loads(json.dumps(bundle, ensure_ascii=False)) await db.commit() return self._sanitize_engine_for_admin_response(item) raise ValueError("engine 不存在") @staticmethod def _normalize_capability_probe_targets( targets: list[str] | None, *, mode: str = "full", allow_high_cost: bool = False, capabilities: dict[str, Any] | None = None, ) -> list[str]: explicit_targets = bool(targets) if targets: expanded: list[str] = [] for item in targets: expanded.extend(_CAPABILITY_TARGET_ALIASES.get(str(item or "").strip().lower(), [])) else: normalized_mode = str(mode or "full").strip().lower() if normalized_mode == "quick": expanded = ["basic_chat", "structured_output", "json_schema", "tool_calling"] elif normalized_mode == "advanced": expanded = [ "basic_chat", "model_self_report", "structured_output", "json_schema", "tool_calling", "streaming", "long_context", "vision_input", "audio_input", ] else: expanded = ["basic_chat", "model_self_report", "structured_output", "json_schema", "tool_calling", "streaming", "long_context"] if allow_high_cost and "long_context_128k" not in expanded: expanded.append("long_context_128k") normalized: list[str] = [] for item in expanded: if item in _CAPABILITY_PROBE_KEYS and item not in normalized: normalized.append(item) declared = capabilities or {} if not bool(declared.get("vision")) and "vision_input" in normalized: normalized.remove("vision_input") if not bool(declared.get("audio")) and "audio_input" in normalized: normalized.remove("audio_input") if not explicit_targets and not bool(declared.get("long_context")): normalized = [item for item in normalized if item not in {"long_context", "long_context_32k", "long_context_128k"}] if not explicit_targets: declared_keys = { "structured_output": bool(declared.get("structured_output")), "json_schema": bool(declared.get("json_schema")), "tool_calling": bool(declared.get("tool_calling")), } has_any_declared_core = any(declared_keys.values()) or bool(declared.get("long_context")) or bool(declared.get("vision")) or bool(declared.get("audio")) if has_any_declared_core: normalized = [ item for item in normalized if item in {"basic_chat", "model_self_report", "streaming", "long_context", "long_context_32k", "long_context_128k", "vision_input", "audio_input"} or declared_keys.get(item, False) ] return normalized @staticmethod def _capability_probe_result( *, status: str, message: str, latency_ms: int | None, method: str, evidence_strength: str, request_summary: str, response_summary: str = "", failure_code: str | None = None, max_context_tokens: int | None = None, partial_evidence: bool = False, declared_ok: bool | None = None, provider_request_payload: Any = None, provider_response_payload: Any = None, provider_attempts: Any = None, provider_contract: Any = None, model_self_report: Any = None, upstream_request_sent: bool | None = None, upstream_missing_reason: str | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "status": status, "message": str(message or "")[:500], "latency_ms": latency_ms, "method": method, "evidence_strength": evidence_strength, "request_summary": str(request_summary or "")[:500], "response_summary": str(response_summary or "")[:500], "failure_code": failure_code, "partial_evidence": partial_evidence, } if provider_request_payload is not None: payload["provider_request_payload"] = provider_request_payload if provider_response_payload is not None: payload["provider_response_payload"] = provider_response_payload if provider_attempts is not None: payload["provider_attempts"] = provider_attempts if provider_contract is not None: payload["provider_contract"] = provider_contract if model_self_report is not None: payload["model_self_report"] = model_self_report if upstream_request_sent is not None: payload["upstream_request_sent"] = bool(upstream_request_sent) if upstream_missing_reason: payload["upstream_missing_reason"] = str(upstream_missing_reason or "")[:300] if max_context_tokens is not None: payload["max_context_tokens"] = max_context_tokens if declared_ok is not None: payload["declared_ok"] = declared_ok return payload @staticmethod def _response_audit_payload(response: Any) -> dict[str, Any]: try: parsed = response.json() if isinstance(parsed, dict): return parsed return {"payload": parsed} except Exception: return {"raw_text": str(getattr(response, "text", "") or "")} @staticmethod def _provider_attempt_audit( *, endpoint: str, provider: str, model: str, request_payload: Any, response_payload: Any = None, status: str, status_code: int | None = None, error: str | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "provider": str(provider or ""), "model": str(model or ""), "endpoint": str(endpoint or ""), "request_payload": request_payload, "status": str(status or "unknown"), } if response_payload is not None: payload["response_payload"] = response_payload if status_code is not None: payload["status_code"] = int(status_code) if error: payload["error"] = str(error or "")[:2000] return payload async def _probe_basic_chat_capability(self, *, engine: dict[str, Any]) -> dict[str, Any]: started = time.perf_counter() request_summary = "发送最小连通 prompt，要求模型只返回 OK。" try: result = await self._chat_completion_raw( provider=str(engine.get("provider") or ""), prompt="你是连通性测试助手。请只返回：OK", system_prompt="", api_key=str(engine.get("api_key") or ""), base_url=str(engine.get("base_url") or ""), model=str(engine.get("model") or ""), temperature=0.0, max_tokens=16, function_name="probe_capability_basic_chat", routing_meta={ "scene_key": "platform_ai_ops", "direct_operation": True, "routing_decision": "direct_capability_probe", "route_source": "provider_console", "provider": str(engine.get("provider") or ""), "provider_code": str(engine.get("provider") or ""), "provider_label": _provider_display_label(engine.get("provider")), "model": str(engine.get("model") or ""), "engine_id": str(engine.get("id") or ""), "engine_name": str(engine.get("name") or ""), "probe_target": "basic_chat", "probe_method": "chat_completion", }, ) ok = "OK" in str(result.text or "").upper() return self._capability_probe_result( status="verified" if ok else "failed", message="基础聊天探测通过" if ok else "模型未返回预期 OK", latency_ms=int((time.perf_counter() - started) * 1000), method="chat_completion", evidence_strength="live_probe" if ok else "unknown", request_summary=request_summary, response_summary=str(result.text or "")[:300], failure_code=None if ok else "unexpected_response", provider_request_payload=result.raw_request, provider_response_payload=result.raw_response, provider_attempts=result.provider_attempts, upstream_request_sent=True, ) except Exception as exc: return self._capability_probe_result( status="failed", message=str(exc), latency_ms=int((time.perf_counter() - started) * 1000), method="chat_completion", evidence_strength="unknown", request_summary=request_summary, failure_code="request_failed", provider_request_payload=getattr(exc, "provider_request_payload", None), provider_response_payload=getattr(exc, "provider_response_payload", None), provider_attempts=getattr(exc, "provider_attempts", None), upstream_request_sent=True, ) async def _probe_model_self_report_capability(self, *, engine: dict[str, Any]) -> dict[str, Any]: started = time.perf_counter() model = str(engine.get("model") or "").strip() request_summary = "发送低成本模型自报 prompt，要求模型按 JSON 返回上下文窗口和输出上限的自我认知；该结果只作为辅助参考。" prompt = ( "你正在作为模型能力自报助手。请根据你对当前模型的自我认知返回 JSON，不要编造，不确定填 null。\n" f"当前接口模型名：{model}\n" "只返回 JSON，不要 Markdown，不要解释。字段如下：\n" "{\n" ' "model_name": string,\n' ' "context_window_tokens": number|null,\n' ' "max_input_tokens": number|null,\n' ' "max_output_tokens": number|null,\n' ' "context_tier": "below_32k"|"32k"|"128k"|"1m"|"unknown",\n' ' "supports_32k": boolean|null,\n' ' "supports_128k": boolean|null,\n' ' "supports_1m": boolean|null,\n' ' "confidence": "low"|"medium"|"high",\n' ' "note": string\n' "}" ) try: result = await self._chat_completion_raw( provider=str(engine.get("provider") or ""), prompt=prompt, system_prompt="", api_key=str(engine.get("api_key") or ""), base_url=str(engine.get("base_url") or ""), model=model, temperature=0.0, max_tokens=384, function_name="probe_capability_model_self_report", routing_meta={ "scene_key": "platform_ai_ops", "direct_operation": True, "routing_decision": "direct_capability_probe", "route_source": "provider_console", "provider": str(engine.get("provider") or ""), "provider_code": str(engine.get("provider") or ""), "provider_label": _provider_display_label(engine.get("provider")), "model": model, "engine_id": str(engine.get("id") or ""), "engine_name": str(engine.get("name") or ""), "probe_target": "model_self_report", "probe_method": "model_self_report_json", }, ) raw_text = str(result.text or "").strip() parsed: dict[str, Any] | None = None try: candidate = self._strip_code_fence(raw_text) parsed_value = json.loads(candidate) except Exception: start = raw_text.find("{") end = raw_text.rfind("}") if start >= 0 and end > start: try: parsed_value = json.loads(raw_text[start : end + 1]) except Exception: parsed_value = None else: parsed_value = None if isinstance(parsed_value, dict): parsed = parsed_value message = "模型自报能力已记录，仅作为运营参考，不作为槽位发布 verified 证据" if parsed else "模型自报未返回可解析 JSON，仅保留原始响应作为运营参考" return self._capability_probe_result( status="declared_only", message=message, latency_ms=int((time.perf_counter() - started) * 1000), method="model_self_report_json", evidence_strength="model_self_report", request_summary=request_summary, response_summary=raw_text[:300], failure_code=None if parsed else "self_report_unparseable", partial_evidence=True, model_self_report=parsed, provider_request_payload=result.raw_request, provider_response_payload=result.raw_response, provider_attempts=result.provider_attempts, upstream_request_sent=True, ) except Exception as exc: return self._capability_probe_result( status="declared_only", message=f"模型自报调用失败，仅作为辅助能力未取得参考：{str(exc)[:220]}", latency_ms=int((time.perf_counter() - started) * 1000), method="model_self_report_json", evidence_strength="model_self_report", request_summary=request_summary, response_summary="", failure_code="self_report_request_failed", partial_evidence=True, provider_request_payload=getattr(exc, "provider_request_payload", None), provider_response_payload=getattr(exc, "provider_response_payload", None), provider_attempts=getattr(exc, "provider_attempts", None), upstream_request_sent=True, ) async def _probe_structured_output_capability(self, *, engine: dict[str, Any]) -> dict[str, Any]: started = time.perf_counter() request_summary = '自由文本响应中要求模型只返回 {"ok":true,"label":"pmflow"}，验证可稳定输出可解析 JSON。' try: result = await self._chat_completion_raw( provider=str(engine.get("provider") or ""), prompt='请只返回 JSON：{"ok":true,"label":"pmflow"}', system_prompt="", api_key=str(engine.get("api_key") or ""), base_url=str(engine.get("base_url") or ""), model=str(engine.get("model") or ""), temperature=0.0, max_tokens=96, function_name="probe_capability_structured_output", routing_meta={ "scene_key": "platform_ai_ops", "direct_operation": True, "routing_decision": "direct_capability_probe", "route_source": "provider_console", "provider": str(engine.get("provider") or ""), "provider_code": str(engine.get("provider") or ""), "provider_label": _provider_display_label(engine.get("provider")), "model": str(engine.get("model") or ""), "engine_id": str(engine.get("id") or ""), "engine_name": str(engine.get("name") or ""), "probe_target": "structured_output", "probe_method": "json_parse", "response_contract": "structured", }, ) parsed = json.loads(self._strip_code_fence(result.text)) ok = isinstance(parsed, dict) and parsed.get("ok") is True return self._capability_probe_result( status="verified" if ok else "failed", message="结构化输出探测通过" if ok else "模型未返回预期 JSON 对象", latency_ms=int((time.perf_counter() - started) * 1000), method="json_parse", evidence_strength="live_probe" if ok else "unknown", request_summary=request_summary, response_summary=str(result.text or "")[:300], failure_code=None if ok else "unexpected_json", provider_request_payload=result.raw_request, provider_response_payload=result.raw_response, provider_attempts=result.provider_attempts, upstream_request_sent=True, ) except Exception as exc: return self._capability_probe_result( status="failed", message=str(exc), latency_ms=int((time.perf_counter() - started) * 1000), method="json_parse", evidence_strength="unknown", request_summary=request_summary, failure_code="request_or_parse_failed", provider_request_payload=getattr(exc, "provider_request_payload", None), provider_response_payload=getattr(exc, "provider_response_payload", None), provider_attempts=getattr(exc, "provider_attempts", None), upstream_request_sent=True, ) async def _probe_json_schema_capability(self, *, engine: dict[str, Any]) -> dict[str, Any]: provider = str(engine.get("provider") or "").strip().lower() if provider == "ollama": return self._capability_probe_result( status="not_probeable", message="Ollama OpenAI-compatible JSON Schema 参数未接入探测", latency_ms=None, method="response_format_json_schema", evidence_strength="unknown", request_summary="发送 response_format=json_schema 的 OpenAI-compatible chat/completions 请求。", failure_code="adapter_not_probeable", ) key = str(engine.get("api_key") or "").strip() if not key: return self._capability_probe_result( status="failed", message="未配置 API Key", latency_ms=None, method="response_format_json_schema", evidence_strength="unknown", request_summary="发送 response_format=json_schema 的 OpenAI-compatible chat/completions 请求。", failure_code="missing_api_key", ) endpoint = f"{str(engine.get('base_url') or '').rstrip('/')}/chat/completions" payload = { "model": str(engine.get("model") or ""), "messages": [{"role": "user", "content": "返回 pmflow JSON schema 探测结果。"}], "temperature": 0, "max_tokens": 96, "response_format": { "type": "json_schema", "json_schema": { "name": "pmflow_capability_probe", "schema": { "type": "object", "properties": {"ok": {"type": "boolean"}, "label": {"type": "string"}}, "required": ["ok", "label"], "additionalProperties": False, }, }, }, } started = time.perf_counter() try: async with httpx.AsyncClient(timeout=20.0, trust_env=False) as client: response = await client.post(endpoint, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, json=payload) latency_ms = int((time.perf_counter() - started) * 1000) if response.status_code >= 400: return self._capability_probe_result( status="failed", message=f"json_schema 参数被供应商拒绝: HTTP {response.status_code}", latency_ms=latency_ms, method="response_format_json_schema", evidence_strength="unknown", request_summary="发送 response_format=json_schema 的 OpenAI-compatible chat/completions 请求。", response_summary=response.text[:300], failure_code="json_schema_param_rejected", provider_request_payload=payload, provider_response_payload=self._response_audit_payload(response), provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=self._response_audit_payload(response), status="http_error", status_code=response.status_code)], upstream_request_sent=True, ) data = response.json() content = str(((((data.get("choices") or [{}])[0] or {}).get("message") or {}).get("content") or "")) parsed = json.loads(self._strip_code_fence(content)) ok = isinstance(parsed, dict) and parsed.get("ok") is True return self._capability_probe_result( status="verified" if ok else "failed", message="JSON Schema 参数探测通过" if ok else "供应商接受参数但未返回符合 schema 的 JSON", latency_ms=latency_ms, method="response_format_json_schema", evidence_strength="live_probe" if ok else "unknown", request_summary="发送 response_format=json_schema 的 OpenAI-compatible chat/completions 请求。", response_summary=content[:300], failure_code=None if ok else "schema_response_invalid", provider_request_payload=payload, provider_response_payload=data, provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=data, status="success", status_code=response.status_code)], upstream_request_sent=True, ) except Exception as exc: return self._capability_probe_result( status="failed", message=str(exc), latency_ms=int((time.perf_counter() - started) * 1000), method="response_format_json_schema", evidence_strength="unknown", request_summary="发送 response_format=json_schema 的 OpenAI-compatible chat/completions 请求。", failure_code="request_or_parse_failed", provider_request_payload=payload, provider_response_payload=getattr(exc, "provider_response_payload", None), provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=getattr(exc, "provider_response_payload", None), status="request_error", error=str(exc)[:2000])], upstream_request_sent=True, ) async def _probe_structured_capability(self, *, engine: dict[str, Any]) -> dict[str, dict[str, Any]]: return { "structured_output": await self._probe_structured_output_capability(engine=engine), "json_schema": await self._probe_json_schema_capability(engine=engine), } async def _probe_tool_calling_capability(self, *, engine: dict[str, Any]) -> dict[str, Any]: request_summary = "发送带 tools + tool_choice 的 OpenAI-compatible chat/completions 请求，验证响应是否包含 tool_calls。" provider = str(engine.get("provider") or "").strip().lower() if provider == "ollama": return self._capability_probe_result( status="not_probeable", message="Ollama chat endpoint 未接入 OpenAI-compatible tools 探测", latency_ms=None, method="tools_tool_choice", evidence_strength="unknown", request_summary=request_summary, failure_code="adapter_not_probeable", ) key = str(engine.get("api_key") or "").strip() if not key: return self._capability_probe_result( status="failed", message="未配置 API Key", latency_ms=None, method="tools_tool_choice", evidence_strength="unknown", request_summary=request_summary, failure_code="missing_api_key", ) endpoint = f"{str(engine.get('base_url') or '').rstrip('/')}/chat/completions" payload = { "model": str(engine.get("model") or ""), "messages": [{"role": "user", "content": "调用 echo 工具返回 pmflow"}], "temperature": 0, "max_tokens": 64, "tools": [ { "type": "function", "function": { "name": "pmflow_capability_echo", "description": "Echo capability probe.", "parameters": { "type": "object", "properties": {"value": {"type": "string"}}, "required": ["value"], }, }, } ], "tool_choice": {"type": "function", "function": {"name": "pmflow_capability_echo"}}, } started = time.perf_counter() try: async with httpx.AsyncClient(timeout=20.0, trust_env=False) as client: response = await client.post(endpoint, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, json=payload) if response.status_code >= 400: return self._capability_probe_result( status="failed", message=f"tools 参数被供应商拒绝: HTTP {response.status_code}", latency_ms=int((time.perf_counter() - started) * 1000), method="tools_tool_choice", evidence_strength="unknown", request_summary=request_summary, response_summary=response.text[:300], failure_code="tools_param_rejected", provider_request_payload=payload, provider_response_payload=self._response_audit_payload(response), provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=self._response_audit_payload(response), status="http_error", status_code=response.status_code)], upstream_request_sent=True, ) data = response.json() message = (((data.get("choices") or [{}])[0] or {}).get("message") or {}) if isinstance(data, dict) else {} tool_calls = message.get("tool_calls") if isinstance(message, dict) else None ok = isinstance(tool_calls, list) and len(tool_calls) > 0 return self._capability_probe_result( status="verified" if ok else "failed", message="工具调用探测通过" if ok else "供应商响应中没有 tool_calls", latency_ms=int((time.perf_counter() - started) * 1000), method="tools_tool_choice", evidence_strength="live_probe" if ok else "unknown", request_summary=request_summary, response_summary=json.dumps(message, ensure_ascii=False)[:300], failure_code=None if ok else "missing_tool_calls", provider_request_payload=payload, provider_response_payload=data, provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=data, status="success", status_code=response.status_code)], upstream_request_sent=True, ) except Exception as exc: return self._capability_probe_result( status="failed", message=str(exc), latency_ms=int((time.perf_counter() - started) * 1000), method="tools_tool_choice", evidence_strength="unknown", request_summary=request_summary, failure_code="request_failed", provider_request_payload=payload, provider_response_payload=getattr(exc, "provider_response_payload", None), provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=getattr(exc, "provider_response_payload", None), status="request_error", error=str(exc)[:2000])], upstream_request_sent=True, ) async def _probe_streaming_capability(self, *, engine: dict[str, Any]) -> dict[str, Any]: provider = str(engine.get("provider") or "").strip().lower() request_summary = "发送 stream=true 的 OpenAI-compatible chat/completions 请求，验证是否返回流式分片。" if provider == "ollama": return self._capability_probe_result( status="not_probeable", message="Ollama 流式协议与当前 OpenAI-compatible 探测适配不同，暂不作为验证证据", latency_ms=None, method="stream_chat_completions", evidence_strength="unknown", request_summary=request_summary, failure_code="adapter_not_probeable", ) key = str(engine.get("api_key") or "").strip() if not key: return self._capability_probe_result( status="failed", message="未配置 API Key", latency_ms=None, method="stream_chat_completions", evidence_strength="unknown", request_summary=request_summary, failure_code="missing_api_key", ) endpoint = f"{str(engine.get('base_url') or '').rstrip('/')}/chat/completions" payload = { "model": str(engine.get("model") or ""), "messages": [{"role": "user", "content": "请只返回 OK"}], "temperature": 0, "max_tokens": 16, "stream": True, } started = time.perf_counter() chunks = 0 first_chunk = "" try: async with httpx.AsyncClient(timeout=20.0, trust_env=False) as client: async with client.stream("POST", endpoint, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, json=payload) as response: if response.status_code >= 400: body = await response.aread() return self._capability_probe_result( status="failed", message=f"stream 参数被供应商拒绝: HTTP {response.status_code}", latency_ms=int((time.perf_counter() - started) * 1000), method="stream_chat_completions", evidence_strength="unknown", request_summary=request_summary, response_summary=body.decode("utf-8", errors="ignore")[:300], failure_code="stream_param_rejected", provider_request_payload=payload, provider_response_payload={"raw_text": body.decode("utf-8", errors="ignore")}, provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload={"raw_text": body.decode("utf-8", errors="ignore")}, status="http_error", status_code=response.status_code)], upstream_request_sent=True, ) async for line in response.aiter_lines(): text = str(line or "").strip() if not text: continue chunks += 1 if not first_chunk: first_chunk = text[:300] if chunks >= 2: break ok = chunks > 0 return self._capability_probe_result( status="verified" if ok else "failed", message="流式响应探测通过" if ok else "供应商未返回流式分片", latency_ms=int((time.perf_counter() - started) * 1000), method="stream_chat_completions", evidence_strength="live_probe" if ok else "unknown", request_summary=request_summary, response_summary=first_chunk, failure_code=None if ok else "missing_stream_chunks", provider_request_payload=payload, provider_response_payload={"first_chunk": first_chunk, "chunks_seen": chunks}, provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload={"first_chunk": first_chunk, "chunks_seen": chunks}, status="success" if ok else "missing_stream_chunks")], upstream_request_sent=True, ) except Exception as exc: return self._capability_probe_result( status="failed", message=str(exc), latency_ms=int((time.perf_counter() - started) * 1000), method="stream_chat_completions", evidence_strength="unknown", request_summary=request_summary, failure_code="request_failed", provider_request_payload=payload, provider_response_payload=getattr(exc, "provider_response_payload", None), provider_attempts=[self._provider_attempt_audit(endpoint=endpoint, provider=provider, model=str(engine.get("model") or ""), request_payload=payload, response_payload=getattr(exc, "provider_response_payload", None), status="request_error", error=str(exc)[:2000])], upstream_request_sent=True, ) async def _probe_long_context_capability(self, *, engine: dict[str, Any], min_context_tokens: int | None = None) -> dict[str, Any]: capabilities = self._normalize_engine_capabilities( engine.get("capabilities"), provider=str(engine.get("provider") or ""), model=str(engine.get("model") or ""), ) declared_tokens = int(capabilities.get("max_context_tokens") or 0) target_tokens = int(min_context_tokens) if min_context_tokens is not None else declared_tokens has_declared_window = bool(capabilities.get("long_context")) and declared_tokens > 0 declared_ok = has_declared_window and (min_context_tokens is None or declared_tokens >= target_tokens) context_contract = _provider_model_context_contract(engine.get("provider"), engine.get("model")) contract_context_tokens = int((context_contract or {}).get("context_tokens") or 0) contract_covers_declared = bool(context_contract) and declared_tokens > 0 and contract_context_tokens >= declared_tokens if declared_ok: if contract_covers_declared: message = ( f"供应商官方模型规格覆盖声明窗口 {declared_tokens} tokens；" "本探测不发送高成本长文本正文，记录为供应商合同证据" ) else: message = f"长上下文真实 smoke 需要高成本调用，本次只记录声明窗口 {declared_tokens} tokens，不作为槽位发布 verified 证据" elif not bool(capabilities.get("long_context")): message = "未声明长上下文能力" elif declared_tokens <= 0: message = "已声明长上下文能力，但未配置 max_context_tokens" else: message = f"声明上下文窗口不足 {target_tokens} tokens" return self._capability_probe_result( status="verified" if declared_ok and contract_covers_declared else ("declared_only" if declared_ok else "failed"), message=message, latency_ms=None, method="declared_window_gate", evidence_strength="provider_contract" if declared_ok and contract_covers_declared else "declared_only", request_summary=( f"读取供应商配置中的 max_context_tokens，目标阈值 {target_tokens} tokens；" "未向模型发送长上下文正文。" ), response_summary=( f"{context_contract.get('label')}：上下文 {contract_context_tokens} tokens，" f"最大输入约 {int(context_contract.get('max_input_tokens') or 0)} tokens。" if context_contract else "" ), max_context_tokens=declared_tokens, partial_evidence=not (declared_ok and contract_covers_declared), declared_ok=declared_ok, provider_contract=context_contract, failure_code=None if declared_ok else "declared_window_too_small", provider_attempts=[ { "status": "not_sent", "provider": str(engine.get("provider") or ""), "model": str(engine.get("model") or ""), "reason": "declared_window_gate_only", "summary": f"本探测只读取供应商声明窗口，未向上游发送长上下文请求；目标阈值 {target_tokens} tokens。", } ], upstream_request_sent=False, upstream_missing_reason="本探测只读取供应商声明窗口，未向上游发送请求。", ) async def _log_capability_probe_item( self, *, tenant_id: int, user_id: int | None, engine: dict[str, Any], probe_run_id: str, target: str, result: dict[str, Any], ) -> None: if result.get("upstream_request_sent") is False: return if not result.get("provider_request_payload") and not result.get("provider_response_payload") and not result.get("provider_attempts"): return try: await self._log_ai_call( tenant_id=tenant_id, function_name=f"probe_capability_{target}", prompt=str(result.get("request_summary") or target), response=str(result.get("response_summary") or result.get("message") or ""), status=f"capability_{str(result.get('status') or 'unknown')}", provider=str(engine.get("provider") or ""), model=str(engine.get("model") or ""), user_id=user_id, provider_request_payload=result.get("provider_request_payload"), provider_response_payload=result.get("provider_response_payload"), provider_attempts=result.get("provider_attempts"), routing_meta={ "scene_key": "platform_ai_ops", "direct_operation": True, "routing_decision": "direct_capability_probe", "route_source": "provider_console", "engine_id": str(engine.get("id") or ""), "engine_name": str(engine.get("name") or ""), "provider": str(engine.get("provider") or ""), "provider_code": str(engine.get("provider") or ""), "provider_label": _provider_display_label(engine.get("provider")), "model": str(engine.get("model") or ""), "probe_run_id": probe_run_id, "probe_target": target, "probe_method": result.get("method"), "probe_status": result.get("status"), "evidence_strength": result.get("evidence_strength"), "failure_code": result.get("failure_code"), "latency_ms": result.get("latency_ms"), "upstream_request_sent": result.get("upstream_request_sent"), "upstream_missing_reason": result.get("upstream_missing_reason"), }, ) except Exception as log_exc: _runtime_log("LLM", f"能力子探测日志写入失败：{log_exc}") async def probe_engine_capabilities( self, db: AsyncSession, *, tenant_id: int, engine_id: str, targets: list[str] | None = None, mode: str = "full", allow_high_cost: bool = False, user_id: int | None = None, ) -> dict[str, Any]: engine = await self._resolve_engine_for_test( db, tenant_id=tenant_id, engine_id=engine_id, ) if not self._engine_enabled(engine): raise ValueError("已停用的供应商连接不能执行能力探测") connection_status = str(engine.get("connection_status") or "untested").strip().lower() if connection_status != "available": raise ValueError("供应商连接测试未通过，不能执行能力探测") declared_capabilities = self._normalize_engine_capabilities( engine.get("capabilities"), provider=str(engine.get("provider") or ""), model=str(engine.get("model") or ""), ) requested = self._normalize_capability_probe_targets( targets, mode=mode, allow_high_cost=allow_high_cost, capabilities=declared_capabilities, ) if not requested: raise ValueError("至少选择一个 capability probe 目标") probe_run_id = f"cap-{uuid4().hex}" results: dict[str, dict[str, Any]] = {} if "basic_chat" in requested: results["basic_chat"] = await self._probe_basic_chat_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="basic_chat", result=results["basic_chat"]) dependency_failed = "basic_chat" in results and str(results["basic_chat"].get("status") or "") != "verified" if dependency_failed: for target in requested: if target == "basic_chat": continue results[target] = self._capability_probe_result( status="skipped", message="基础聊天探测未通过，跳过后续能力探测", latency_ms=None, method="dependency_gate", evidence_strength="unknown", request_summary="依赖基础聊天能力通过后才执行该能力探测。", failure_code="basic_chat_failed", ) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target=target, result=results[target]) if not dependency_failed and "model_self_report" in requested: results["model_self_report"] = await self._probe_model_self_report_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="model_self_report", result=results["model_self_report"]) if not dependency_failed and "structured_output" in requested: results["structured_output"] = await self._probe_structured_output_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="structured_output", result=results["structured_output"]) if not dependency_failed and "json_schema" in requested: results["json_schema"] = await self._probe_json_schema_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="json_schema", result=results["json_schema"]) if not dependency_failed and "tool_calling" in requested: results["tool_calling"] = await self._probe_tool_calling_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="tool_calling", result=results["tool_calling"]) if not dependency_failed and "streaming" in requested: results["streaming"] = await self._probe_streaming_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="streaming", result=results["streaming"]) if not dependency_failed and "long_context" in requested: results["long_context"] = await self._probe_long_context_capability(engine=engine) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="long_context", result=results["long_context"]) if not dependency_failed and "long_context_32k" in requested: results["long_context_32k"] = await self._probe_long_context_capability(engine=engine, min_context_tokens=32768) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="long_context_32k", result=results["long_context_32k"]) if not dependency_failed and "long_context_128k" in requested: results["long_context_128k"] = await self._probe_long_context_capability(engine=engine, min_context_tokens=128000) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target="long_context_128k", result=results["long_context_128k"]) for target in ("vision_input", "audio_input"): if not dependency_failed and target in requested: results[target] = self._capability_probe_result( status="not_probeable", message="当前适配器尚未提供该多模态能力的真实探测实现，不能作为 verified 证据", latency_ms=None, method="adapter_capability_gap", evidence_strength="unknown", request_summary=f"检查 {target} 是否有已接入的真实探测器。", failure_code="probe_not_implemented", ) await self._log_capability_probe_item(tenant_id=tenant_id, user_id=user_id, engine=engine, probe_run_id=probe_run_id, target=target, result=results[target]) sanitized = await self._record_engine_capability_probe_result( db, tenant_id=tenant_id, engine_id=str(engine.get("id") or ""), results=results, ) status = str((sanitized.get("capabilities") or {}).get("capability_probe_status") or "untested") return { "ok": status in {"passed", "partial"}, "engine_id": str(engine.get("id") or ""), "provider": str(engine.get("provider") or ""), "model": str(engine.get("model") or ""), "status": status, "probe_run_id": probe_run_id, "mode": str(mode or "full"), "targets": requested, "results": results, "capabilities": sanitized.get("capabilities") or {}, } async def test_engine_connection( self, db: AsyncSession, *, tenant_id: int, engine_id: Optional[str] = None, prompt: Optional[str] = None, max_tokens: int = 64, user_id: int | None = None, ) -> dict[str, Any]: test_prompt = (prompt or "你是连通性测试助手。请只返回: OK").strip() if not test_prompt: raise ValueError("测试 prompt 不能为空") engine = await self._resolve_engine_for_test( db, tenant_id=tenant_id, engine_id=engine_id, ) if not self._engine_enabled(engine): raise ValueError("已停用的供应商连接不能执行连接测试") response_text = "" status = "success" provider_request_payload: Any = None provider_response_payload: Any = None provider_attempts: Any = None started = time.perf_counter() try: transient_retry_budget = 2 transient_retry_attempt = 0 while True: try: completion = await self._chat_completion_raw( provider=str(engine.get("provider") or ""), prompt=test_prompt, system_prompt="", api_key=str(engine.get("api_key") or ""), base_url=str(engine.get("base_url") or ""), model=str(engine.get("model") or ""), temperature=0.0, max_tokens=max_tokens, ) response_text = completion.text provider_request_payload = completion.raw_request provider_response_payload = completion.raw_response provider_attempts = completion.provider_attempts break except Exception as transient_exc: provider_request_payload = provider_request_payload or getattr(transient_exc, "provider_request_payload", None) provider_response_payload = provider_response_payload or getattr(transient_exc, "provider_response_payload", None) provider_attempts = provider_attempts or getattr(transient_exc, "provider_attempts", None) status_code = self._http_status_from_exception(transient_exc) can_retry_transient = ( transient_retry_budget > 0 and ( self._is_retryable_http_status(status_code) or isinstance(transient_exc, httpx.RequestError) ) ) if not can_retry_transient: raise retry_after_header = None if isinstance(transient_exc, httpx.HTTPStatusError) and transient_exc.response is not None: retry_after_header = transient_exc.response.headers.get("Retry-After") retry_delay = self._http_retry_delay_seconds( status_code=status_code, attempt=transient_retry_attempt, retry_after_header=retry_after_header, ) _runtime_log( "LLM", ( "连通测试重试：" f"engine={engine.get('provider')}/{engine.get('model')}, " f"status={status_code or 'request_error'}, " f"retry_in={retry_delay:.2f}s, left={transient_retry_budget - 1}" ), ) transient_retry_budget -= 1 transient_retry_attempt += 1 await asyncio.sleep(retry_delay) if not response_text.strip(): raise ValueError("测试响应为空") latency_ms = int((time.perf_counter() - started) * 1000) await self._record_engine_test_result( db, tenant_id=tenant_id, engine_id=str(engine.get("id") or ""), ok=True, ) return { "ok": True, "engine_id": str(engine.get("id") or ""), "name": str(engine.get("name") or ""), "provider": str(engine.get("provider") or ""), "model": str(engine.get("model") or ""), "base_url": str(engine.get("base_url") or ""), "api_key_masked": _mask_secret(str(engine.get("api_key") or "")), "latency_ms": latency_ms, "response_preview": response_text[:300], } except Exception as exc: status = "failed" response_text = f"[error] {exc}" try: await self._record_engine_test_result( db, tenant_id=tenant_id, engine_id=str(engine.get("id") or ""), ok=False, error_summary=str(exc), ) except Exception as record_exc: _runtime_log("LLM", f"连通测试状态写入失败：{record_exc}") status_code = self._http_status_from_exception(exc) if int(status_code or 0) == 429: raise ValueError("引擎连通测试失败: 上游限流(429)，请稍后重试或更换 API Key。") from exc raise ValueError(f"引擎连通测试失败: {exc}") from exc finally: try: await self._log_ai_call( tenant_id=tenant_id, function_name="test_engine_connection", prompt=test_prompt, response=response_text, status=status, provider=str(engine.get("provider") or ""), model=str(engine.get("model") or ""), user_id=user_id, provider_request_payload=provider_request_payload, provider_response_payload=provider_response_payload, provider_attempts=provider_attempts, routing_meta={ "scene_key": "platform_ai_ops", "engine_id": str(engine.get("id") or ""), "engine_name": str(engine.get("name") or ""), "provider": str(engine.get("provider") or ""), "provider_code": str(engine.get("provider") or ""), "provider_label": _provider_display_label(engine.get("provider")), "model": str(engine.get("model") or ""), "latency_ms": int((time.perf_counter() - started) * 1000), }, ) except Exception as log_exc: _runtime_log("LLM", f"测试调用日志写入失败：{log_exc}")