476 lines
21 KiB
Python
476 lines
21 KiB
Python
"""AI readiness evaluation for protected PMFlow product scenes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from sqlalchemy import and_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ..models import AICallLog, Integration
|
|
from ..services.ai_routing_scene_contracts import AI_SCENE_REGISTRY, scene_contract
|
|
from ..services.ai_routing_governance_service import (
|
|
CAPABILITY_SLOT_INTEGRATION_TYPE,
|
|
LLM_ENGINE_INTEGRATION_TYPE,
|
|
PLATFORM_RUNTIME_TENANT_ID,
|
|
)
|
|
from ..services.pmflow_launch_service import PMFlowLaunchError, PMFlowLaunchService
|
|
from ..utils.timezone import now_cn_naive
|
|
|
|
|
|
def _safe_dict(value: Any) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _safe_list(value: Any) -> list[Any]:
|
|
return value if isinstance(value, list) else []
|
|
|
|
|
|
def _is_success_status(status: str) -> bool:
|
|
normalized = str(status or "").strip().lower()
|
|
return normalized.startswith("success") or normalized in {"ok", "success_cache_hit", "success_failover"}
|
|
|
|
|
|
class AIReadinessService:
|
|
"""Derive customer-facing AI readiness from runtime facts."""
|
|
|
|
SCENE_KEY = "project_launch"
|
|
FUNCTION_NAME = "pmflow_launch_draft_v1"
|
|
|
|
def __init__(self, db: AsyncSession, *, tenant_id: int):
|
|
self.db = db
|
|
self.tenant_id = int(tenant_id)
|
|
|
|
async def get_project_launch_readiness(self) -> dict[str, Any]:
|
|
engine_bundle = await self._load_platform_integration(LLM_ENGINE_INTEGRATION_TYPE)
|
|
slot_bundle = await self._load_platform_integration(CAPABILITY_SLOT_INTEGRATION_TYPE)
|
|
recent_calls = await self._recent_launch_calls()
|
|
latest_call = recent_calls[0] if recent_calls else None
|
|
return self._build_project_launch_readiness(
|
|
engine_bundle=engine_bundle,
|
|
slot_bundle=slot_bundle,
|
|
latest_call=latest_call,
|
|
recent_calls=recent_calls,
|
|
smoke_error=None,
|
|
)
|
|
|
|
async def run_project_launch_check(self, *, user_id: int) -> dict[str, Any]:
|
|
smoke_error: str | None = None
|
|
try:
|
|
await PMFlowLaunchService(self.db, tenant_id=PLATFORM_RUNTIME_TENANT_ID).generate_draft_payload_for_readiness_check(
|
|
user_id=int(user_id),
|
|
)
|
|
except PMFlowLaunchError as exc:
|
|
smoke_error = f"{exc.code}: {exc.message}"
|
|
except Exception as exc: # pragma: no cover - defensive boundary
|
|
smoke_error = str(exc)
|
|
engine_bundle = await self._load_platform_integration(LLM_ENGINE_INTEGRATION_TYPE)
|
|
slot_bundle = await self._load_platform_integration(CAPABILITY_SLOT_INTEGRATION_TYPE)
|
|
recent_calls = await self._recent_launch_calls()
|
|
latest_call = recent_calls[0] if recent_calls else None
|
|
payload = self._build_project_launch_readiness(
|
|
engine_bundle=engine_bundle,
|
|
slot_bundle=slot_bundle,
|
|
latest_call=latest_call,
|
|
recent_calls=recent_calls,
|
|
smoke_error=smoke_error,
|
|
)
|
|
payload["platform"]["last_smoke_error"] = smoke_error
|
|
payload["platform"]["last_checked_at"] = now_cn_naive().isoformat()
|
|
return payload
|
|
|
|
async def _load_platform_integration(self, integration_type: str) -> dict[str, Any]:
|
|
row = (
|
|
await self.db.execute(
|
|
select(Integration).where(
|
|
and_(
|
|
Integration.tenant_id == PLATFORM_RUNTIME_TENANT_ID,
|
|
Integration.type == integration_type,
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
return _safe_dict(row.config if row is not None else {})
|
|
|
|
async def _recent_launch_calls(self, *, limit: int = 20, hours: int = 24) -> list[AICallLog]:
|
|
tenant_ids = sorted({int(self.tenant_id), int(PLATFORM_RUNTIME_TENANT_ID)})
|
|
since = now_cn_naive() - timedelta(hours=hours)
|
|
return (
|
|
await self.db.execute(
|
|
select(AICallLog)
|
|
.where(
|
|
and_(
|
|
AICallLog.tenant_id.in_(tenant_ids),
|
|
AICallLog.function_name == self.FUNCTION_NAME,
|
|
AICallLog.created_at >= since,
|
|
)
|
|
)
|
|
.order_by(AICallLog.created_at.desc(), AICallLog.id.desc())
|
|
.limit(limit)
|
|
)
|
|
).scalars().all()
|
|
|
|
def _build_project_launch_readiness(
|
|
self,
|
|
*,
|
|
engine_bundle: dict[str, Any],
|
|
slot_bundle: dict[str, Any],
|
|
latest_call: AICallLog | None,
|
|
recent_calls: list[AICallLog],
|
|
smoke_error: str | None,
|
|
) -> dict[str, Any]:
|
|
blockers: list[dict[str, Any]] = []
|
|
warnings: list[dict[str, Any]] = []
|
|
checks: dict[str, str] = {
|
|
"provider_runtime": "unknown",
|
|
"capability_probe": "unknown",
|
|
"active_slot": "unknown",
|
|
"scene_route": "unknown",
|
|
"recent_smoke": "missing",
|
|
"recent_failures": "unknown",
|
|
}
|
|
|
|
engines = [item for item in _safe_list(engine_bundle.get("engines")) if isinstance(item, dict)]
|
|
engines_by_id = {str(item.get("id") or ""): item for item in engines if str(item.get("id") or "")}
|
|
active_slots = self._active_slots(slot_bundle)
|
|
contract = scene_contract(self.SCENE_KEY)
|
|
scene_registry_entry = _safe_dict(AI_SCENE_REGISTRY.get(self.SCENE_KEY))
|
|
|
|
provider_ok = any(self._engine_connectable(item) for item in engines)
|
|
checks["provider_runtime"] = "passed" if provider_ok else "failed"
|
|
if not provider_ok:
|
|
blockers.append(
|
|
self._blocker(
|
|
"provider_unavailable",
|
|
"供应商运行时不可用",
|
|
"平台运行租户没有可用的模型运行时，项目草案生成必须保持关闭。",
|
|
"failed",
|
|
"查看供应商与凭据",
|
|
"/platform/ai-routing-providers",
|
|
)
|
|
)
|
|
|
|
scene_ok = (
|
|
self.FUNCTION_NAME in set(scene_registry_entry.get("function_names") or [])
|
|
and "structured" in set(scene_registry_entry.get("required_slots") or [])
|
|
and str(contract.get("expected_slot_key") or "").strip() == "structured"
|
|
)
|
|
checks["scene_route"] = "passed" if scene_ok else "failed"
|
|
if not scene_ok:
|
|
blockers.append(
|
|
self._blocker(
|
|
"scene_route_mismatch",
|
|
"项目启动场景合同不匹配",
|
|
"project_launch 必须绑定结构化草案函数和 structured 槽。",
|
|
"failed",
|
|
"查看场景路由",
|
|
"/platform/ai-routing-scenes?scene=project_launch",
|
|
)
|
|
)
|
|
|
|
structured_slot = _safe_dict(active_slots.get("structured"))
|
|
structured_engine_id = str(structured_slot.get("primary_engine_id") or "").strip()
|
|
structured_engine = engines_by_id.get(structured_engine_id)
|
|
if not structured_engine_id:
|
|
checks["active_slot"] = "failed"
|
|
blockers.append(
|
|
self._blocker(
|
|
"structured_slot_unpublished",
|
|
"结构化槽未发布",
|
|
"project_launch 需要 active structured 槽；当前结构化槽未发布到运行版本。",
|
|
"failed",
|
|
"发布结构化槽",
|
|
"/platform/ai-routing-slots?slot=structured",
|
|
)
|
|
)
|
|
elif not isinstance(structured_engine, dict) or not self._engine_connectable(structured_engine):
|
|
checks["active_slot"] = "failed"
|
|
blockers.append(
|
|
self._blocker(
|
|
"provider_unavailable",
|
|
"结构化槽运行时不可用",
|
|
"active structured 槽绑定的运行时不存在、被停用或连接不可用。",
|
|
"failed",
|
|
"查看结构化槽",
|
|
"/platform/ai-routing-slots?slot=structured",
|
|
)
|
|
)
|
|
else:
|
|
checks["active_slot"] = "passed"
|
|
coverage = self._structured_capability_coverage(structured_engine)
|
|
checks["capability_probe"] = "passed" if coverage["satisfied"] else "failed"
|
|
if not coverage["satisfied"]:
|
|
blockers.append(
|
|
self._blocker(
|
|
"capability_probe_missing",
|
|
"结构化能力证据不足",
|
|
"结构化槽运行时必须具备已验证的结构化输出与 JSON Schema 能力。",
|
|
"failed",
|
|
"执行能力探测",
|
|
"/platform/ai-routing-providers?capability=structured",
|
|
)
|
|
)
|
|
|
|
if latest_call is None:
|
|
checks["recent_smoke"] = "missing"
|
|
checks["recent_failures"] = "unknown"
|
|
warnings.append(
|
|
self._blocker(
|
|
"smoke_not_run",
|
|
"尚未运行草案 smoke test",
|
|
"供应商和槽位恢复后，需要一次项目草案结构化 smoke test 覆盖历史失败。",
|
|
"warning",
|
|
"运行草案 smoke",
|
|
"/platform/ai-routing-overview",
|
|
)
|
|
)
|
|
else:
|
|
latest_success = _is_success_status(str(latest_call.status or ""))
|
|
recent_success = next((call for call in recent_calls if _is_success_status(str(call.status or ""))), None)
|
|
consecutive_failures = 0
|
|
for call in recent_calls:
|
|
if _is_success_status(str(call.status or "")):
|
|
break
|
|
consecutive_failures += 1
|
|
|
|
if latest_success:
|
|
checks["recent_smoke"] = "passed"
|
|
checks["recent_failures"] = "passed"
|
|
elif recent_success is not None and consecutive_failures < 3:
|
|
checks["recent_smoke"] = "passed"
|
|
checks["recent_failures"] = "warning"
|
|
warnings.append(
|
|
self._blocker(
|
|
"recent_generation_failed",
|
|
"最近一次草案生成失败",
|
|
self._customer_safe_failure_summary(str(latest_call.response or latest_call.status or "")),
|
|
"warning",
|
|
"查看失败诊断",
|
|
"/platform/ai-routing-failures",
|
|
)
|
|
)
|
|
else:
|
|
checks["recent_smoke"] = "failed"
|
|
checks["recent_failures"] = "failed"
|
|
warnings.append(
|
|
self._blocker(
|
|
"recent_generation_failed",
|
|
"最近一次草案生成失败",
|
|
self._customer_safe_failure_summary(str(latest_call.response or latest_call.status or "")),
|
|
"warning",
|
|
"运行草案 smoke",
|
|
"/platform/ai-routing-overview",
|
|
)
|
|
)
|
|
|
|
if smoke_error:
|
|
checks["recent_smoke"] = "failed"
|
|
warnings.append(
|
|
self._blocker(
|
|
"smoke_failed",
|
|
"草案 smoke test 失败",
|
|
self._customer_safe_failure_summary(smoke_error),
|
|
"warning",
|
|
"查看失败诊断",
|
|
"/platform/ai-routing-failures",
|
|
)
|
|
)
|
|
|
|
if blockers:
|
|
status = "blocked"
|
|
elif checks["recent_smoke"] == "passed":
|
|
status = "ready"
|
|
elif warnings:
|
|
status = "recovering"
|
|
else:
|
|
status = "unknown"
|
|
|
|
can_generate = status == "ready"
|
|
customer = self._customer_copy(status)
|
|
workspace_owner = self._workspace_owner_copy(status)
|
|
visible_blockers = self._customer_blockers(status=status, warnings=warnings, hard_blockers=blockers)
|
|
platform_blockers = [*blockers, *warnings]
|
|
|
|
return {
|
|
"scene": self.SCENE_KEY,
|
|
"status": status,
|
|
"customer_state": "ai_collaboration_ready" if can_generate else "ai_draft_temporarily_unavailable",
|
|
"short_label": "AI 协作可用" if can_generate else ("AI 协作恢复中" if status == "recovering" else "AI 协作暂不可用"),
|
|
"title": customer["title"],
|
|
"summary": customer["summary"],
|
|
"updated_at_label": "可用" if can_generate else ("恢复中" if status == "recovering" else "暂不可用"),
|
|
"can_generate_project_draft": can_generate,
|
|
"blockers": visible_blockers,
|
|
"customer_actions": [{"label": "新建项目", "to": "/project-launch"}] if can_generate else [
|
|
{"label": "记录项目想法", "to": "/project-launch"},
|
|
{"label": "查看所有项目", "to": "/space/projects"},
|
|
],
|
|
"workspace_owner_actions": [
|
|
{"label": "查看空间 AI 状态", "to": "/ai-config?tab=setup"},
|
|
{"label": "查看团队策略", "to": "/ai-config/team-ai-policy"},
|
|
],
|
|
"platform": {
|
|
"severity": "ok" if can_generate else ("warning" if status == "recovering" else "critical"),
|
|
"blockers": platform_blockers,
|
|
"checks": checks,
|
|
"latest_call": self._latest_call_payload(latest_call),
|
|
"last_checked_at": now_cn_naive().isoformat(),
|
|
},
|
|
"platform_diagnostic": {
|
|
"cause_code": "ready" if can_generate else (platform_blockers[0]["code"] if platform_blockers else "readiness_unknown"),
|
|
"cause_owner": "pmflow_platform",
|
|
"raw_status": status,
|
|
"severity": "ok" if can_generate else ("warning" if status == "recovering" else "critical"),
|
|
"affected_scenes": ["项目草案生成", "暂存想法晋升草案"],
|
|
"customer_title": customer["title"],
|
|
"customer_summary": customer["summary"],
|
|
"operator_summary": workspace_owner["summary"] if status != "ready" else "项目草案生成链路可用。",
|
|
"platform_actions": [
|
|
{"label": "查看供应商与凭据", "to": "/platform/ai-routing-providers"},
|
|
{"label": "查看能力槽", "to": "/platform/ai-routing-slots"},
|
|
{"label": "查看失败诊断", "to": "/platform/ai-routing-failures"},
|
|
],
|
|
"checks": checks,
|
|
"blockers": platform_blockers,
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def _active_slots(slot_bundle: dict[str, Any]) -> dict[str, Any]:
|
|
active = _safe_dict(slot_bundle.get("active"))
|
|
slots = active.get("slots") if isinstance(active.get("slots"), dict) else slot_bundle.get("slots")
|
|
return _safe_dict(slots)
|
|
|
|
@staticmethod
|
|
def _engine_connectable(engine: dict[str, Any]) -> bool:
|
|
if not bool(engine.get("enabled", True)):
|
|
return False
|
|
if not str(engine.get("api_key") or "").strip() and str(engine.get("provider") or "").strip().lower() != "ollama":
|
|
return False
|
|
status = str(engine.get("connection_status") or "").strip().lower()
|
|
last_test = str(engine.get("last_test_status") or "").strip().lower()
|
|
return status == "available" or last_test == "success"
|
|
|
|
@staticmethod
|
|
def _structured_capability_coverage(engine: dict[str, Any]) -> dict[str, Any]:
|
|
capabilities = _safe_dict(engine.get("capabilities"))
|
|
evidence = _safe_dict(capabilities.get("capability_evidence"))
|
|
|
|
def _verified(key: str) -> bool:
|
|
row = _safe_dict(evidence.get(key))
|
|
return str(row.get("status") or "").strip().lower() == "verified"
|
|
|
|
structured = bool(capabilities.get("structured_output"))
|
|
json_schema = bool(capabilities.get("json_schema"))
|
|
probe_status = str(capabilities.get("capability_probe_status") or "").strip().lower()
|
|
has_evidence = _verified("structured_output") and _verified("json_schema")
|
|
return {
|
|
"satisfied": structured and json_schema and (probe_status == "passed" or has_evidence),
|
|
"structured_output": structured,
|
|
"json_schema": json_schema,
|
|
"probe_status": probe_status or "untested",
|
|
"has_evidence": has_evidence,
|
|
}
|
|
|
|
@staticmethod
|
|
def _blocker(
|
|
code: str,
|
|
label: str,
|
|
detail: str,
|
|
status: str,
|
|
action_label: str | None = None,
|
|
action_to: str | None = None,
|
|
) -> dict[str, Any]:
|
|
payload: dict[str, Any] = {
|
|
"code": code,
|
|
"key": code,
|
|
"label": label,
|
|
"description": detail,
|
|
"detail": detail,
|
|
"status": status,
|
|
}
|
|
if action_label and action_to:
|
|
payload["action_label"] = action_label
|
|
payload["action_to"] = action_to
|
|
return payload
|
|
|
|
@staticmethod
|
|
def _customer_copy(status: str) -> dict[str, str]:
|
|
if status == "ready":
|
|
return {
|
|
"title": "AI 协作可用",
|
|
"summary": "PMFlow 可以生成项目草案、提取需求与风险，并把结果交回主流程确认。",
|
|
}
|
|
if status == "recovering":
|
|
return {
|
|
"title": "AI 草案生成正在恢复",
|
|
"summary": "平台能力正在恢复验证中。你可以先记录想法，系统确认稳定后会开放草案生成。",
|
|
}
|
|
return {
|
|
"title": "AI 项目草案暂不可用",
|
|
"summary": "当前空间暂时不能生成 AI 项目草案。你可以先记录项目想法、查看项目和处理待确认事项；系统不会用模板草案冒充 AI 结果。",
|
|
}
|
|
|
|
@staticmethod
|
|
def _workspace_owner_copy(status: str) -> dict[str, str]:
|
|
if status == "ready":
|
|
return {"summary": "项目草案生成链路可用，确认前仍不会写入正式项目。"}
|
|
if status == "recovering":
|
|
return {"summary": "平台 AI 草案能力正在恢复验证。空间配置无需反复调整，可以继续维护团队策略和知识组织。"}
|
|
return {"summary": "平台 AI 草案能力尚未恢复。空间内仍可维护团队策略、工具连接和知识组织。"}
|
|
|
|
def _customer_blockers(
|
|
self,
|
|
*,
|
|
status: str,
|
|
warnings: list[dict[str, Any]],
|
|
hard_blockers: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
if status == "ready":
|
|
return [
|
|
self._blocker("draft_generation", "项目草案生成", "可以把项目想法生成可确认的结构化草案。", "passed"),
|
|
self._blocker("safe_confirmation", "确认保护", "生成结果仍进入项目主流程确认，不会直接写入正式项目状态。", "passed"),
|
|
]
|
|
impact_status = "warning" if status == "recovering" else "failed"
|
|
impact_text = "AI 草案生成正在恢复验证。" if status == "recovering" else "暂时不能生成 AI 项目草案。"
|
|
if hard_blockers:
|
|
impact_text = "平台 AI 草案能力尚未恢复。"
|
|
elif warnings:
|
|
impact_text = "平台正在验证 AI 草案生成链路。"
|
|
return [
|
|
self._blocker("draft_generation", "当前影响", impact_text, impact_status),
|
|
self._blocker("existing_work", "仍可继续", "记录想法、处理待确认、查看项目和推进已有项目不受影响。", "passed"),
|
|
self._blocker("safe_fallback", "保护规则", "平台能力未恢复前，不会开放 AI 草案生成，也不会用模板草案冒充 AI 结果。", "warning"),
|
|
]
|
|
|
|
@staticmethod
|
|
def _latest_call_payload(row: AICallLog | None) -> dict[str, Any] | None:
|
|
if row is None:
|
|
return None
|
|
return {
|
|
"id": int(row.id),
|
|
"created_at": row.created_at.isoformat() if row.created_at else None,
|
|
"function_name": str(row.function_name or ""),
|
|
"status": str(row.status or ""),
|
|
"planned_tier": str(row.planned_tier or ""),
|
|
"executed_tier": str(row.executed_tier or ""),
|
|
"routing_decision": str(row.routing_decision or ""),
|
|
"route_source": str(row.route_source or ""),
|
|
"route_rule_name": str(row.route_rule_name or ""),
|
|
}
|
|
|
|
@staticmethod
|
|
def _customer_safe_failure_summary(raw: str) -> str:
|
|
text = str(raw or "").strip()
|
|
if not text:
|
|
return "系统没有拿到可用的结构化草案结果。"
|
|
if "401" in text or "Unauthorized" in text:
|
|
return "最近一次草案生成失败，平台凭据或供应商授权需要重新验证。"
|
|
if "structured" in text:
|
|
return "最近一次草案生成没有通过结构化结果校验。"
|
|
return text[:180]
|
|
|
|
|
|
__all__ = ["AIReadinessService"]
|