from __future__ import annotations import json from pathlib import Path from typing import Any, Optional from sqlalchemy import case, func, select from sqlalchemy.ext.asyncio import AsyncSession from ..models import ( JudgeRun, MountedAgentTrace, MountedConversationRun, MountedDraftWorkspace, MountedSourceActivation, MountedToolCapabilityPolicy, MountedTopicWorkspace, QualityAsset, QualityAssetPair, TrainingDataset, TrainingEvaluation, TrainingJob, TrainedModelDeployment, ) from ..services.training_pipeline import TrainingPipelineService from ..utils.timezone import now_cn_naive class MountedAIRuntimeService: def __init__(self, db: AsyncSession): self.db = db @staticmethod def _dt(value: Any) -> Optional[str]: return value.isoformat() if value is not None else None @staticmethod def _backend_root() -> Path: return Path(__file__).resolve().parents[2] @classmethod def _read_backend_source(cls, relative_path: str) -> str: try: return (cls._backend_root() / relative_path).read_text(encoding="utf-8") except OSError: return "" @staticmethod def _legacy_cutover_item( *, key: str, title: str, retired: bool, summary: str, evidence: str, ) -> dict[str, Any]: return { "key": key, "title": title, "status": "retired" if retired else "active", "summary": summary, "evidence": evidence, } @classmethod def _build_legacy_cutover_items(cls) -> list[dict[str, Any]]: ingress_render = cls._read_backend_source("app/services/ai_pmo/ai_pmo_kernel/mixins/ingress/ai_pmo_kernel_mixins_ingress_render.py") feishu_reply = cls._read_backend_source("app/services/integrations/feishu_integration_service/mixin_reply_llm_core.py") signal_capture_service = cls._read_backend_source("app/services/signal/signal_capture_service/service.py") feishu_pipeline_service = cls._read_backend_source("app/services/integrations/feishu_signal_pipeline/service.py") feishu_process_core = cls._read_backend_source("app/services/integrations/feishu_signal_pipeline/mixin_process_message_core.py") feishu_followup_a = cls._read_backend_source("app/services/integrations/feishu_signal_pipeline/mixin_followup_a.py") return [ cls._legacy_cutover_item( key="ai_pmo_ingress_consultant_capture", title="AI PMO ingress consultant capture", retired="build_consultant_dispatch_contract" not in ingress_render and '"consultant_dispatch"' not in ingress_render, summary="capture result 不再生成或透传 consultant_dispatch 到 AI PMO ingress 主链。", evidence="ai_pmo_kernel_mixins_ingress_render.py 不再包含 consultant dispatch contract/import 与 rag_result 注入。", ), cls._legacy_cutover_item( key="feishu_mention_consultant_reply", title="Feishu mention consultant reply", retired="build_channel_consultant_reply" not in feishu_reply and "_build_consultant_dispatch_reply(capture_result)" not in feishu_reply, summary="飞书 mention 主回复不再走 consultant reply 分支。", evidence="mixin_reply_llm_core.py 已移除 consultant reply import 和 active reply branch；helper 仅保留 no-op。", ), cls._legacy_cutover_item( key="signal_capture_legacy_entry", title="Signal capture legacy entry", retired="SignalCaptureLegacyMixin" not in signal_capture_service, summary="主动信号捕获服务不再继承 legacy capture mixin。", evidence="signal_capture_service/service.py 的 ProactiveSignalCaptureService 继承链已去掉 SignalCaptureLegacyMixin。", ), cls._legacy_cutover_item( key="legacy_feishu_queue_followup", title="Legacy Feishu queue follow-up", retired=( "FeishuPipelineFollowupAMixin" not in feishu_pipeline_service and "FeishuPipelineFollowupBMixin" not in feishu_pipeline_service and "_maybe_handle_queue_followup(" not in feishu_process_core and "return None" in feishu_followup_a and "Legacy feishu_items queue follow-up is retired." in feishu_followup_a ), summary="旧 feishu_items 队列 continue/status follow-up 已从真实入口退役。", evidence="service.py 不再混入 FollowupA/B，process_message_core 不再调用 queue followup，mixin_followup_a 已降为 retired stub。", ), ] def _topic_snapshot( self, row: MountedTopicWorkspace, *, draft: Optional[MountedDraftWorkspace] = None, ) -> dict[str, Any]: return { "id": int(row.id), "topic_key": row.topic_key, "source_system": row.source_system, "channel_ref": row.channel_ref, "thread_key": row.thread_key, "mode": row.mode, "status": row.status, "plan_status": getattr(row, "plan_status", None), "readiness_score": float(row.readiness_score or 0.0) if row.readiness_score is not None else None, "draft_state": draft.state if draft is not None else None, "draft_version": int(draft.draft_version) if draft is not None else None, "updated_at": self._dt(row.updated_at), } def _draft_snapshot(self, row: MountedDraftWorkspace) -> dict[str, Any]: return { "id": int(row.id), "topic_workspace_id": int(row.topic_workspace_id), "state": row.state, "draft_version": int(row.draft_version or 0), "requirement_card": row.requirement_card, "draft_payload": row.draft_payload, "required_confirmations": row.required_confirmations, "last_ready_at": self._dt(row.last_ready_at), "confirm_message_id": row.confirm_message_id, } @staticmethod def _source_activation_snapshot(row: MountedSourceActivation) -> dict[str, Any]: return { "id": int(row.id), "source_system": row.source_system, "scope_type": row.scope_type, "scope_ref": row.scope_ref, "status": row.status, "cutover_mode": row.cutover_mode, "auto_reply_enabled": bool(row.auto_reply_enabled), "last_seen_at": MountedAIRuntimeService._dt(row.last_seen_at), "notes": row.notes, "metadata_json": row.metadata_json, } @staticmethod def _tool_policy_snapshot(row: MountedToolCapabilityPolicy) -> dict[str, Any]: return { "id": int(row.id), "tool_name": row.tool_name, "risk_level": row.risk_level, "confirm_required": bool(row.confirm_required), "auto_execute_allowed": bool(row.auto_execute_allowed), "is_active": bool(row.is_active), "source_scope": row.source_scope, "tenant_scope": row.tenant_scope, "policy_metadata": row.policy_metadata, "notes": row.notes, } def _run_snapshot(self, row: MountedConversationRun, *, trace_count: int = 0) -> dict[str, Any]: return { "id": int(row.id), "trace_id": row.trace_id, "source_system": row.source_system, "source_message_id": row.source_message_id, "mode_before": row.mode_before, "mode_after": row.mode_after, "active_agent": row.active_agent, "status": row.status, "trigger_reason": row.trigger_reason, "final_reply": row.final_reply, "trace_count": int(trace_count), "updated_at": self._dt(row.updated_at), } @staticmethod def _trace_snapshot(row: MountedAgentTrace) -> dict[str, Any]: return { "id": int(row.id), "agent_name": row.agent_name, "status": row.status, "latency_ms": int(row.latency_ms) if row.latency_ms is not None else None, "failure_reason": row.failure_reason, "input_summary": row.input_summary, "output_summary": row.output_summary, "tool_trace": row.tool_trace, "created_at": row.created_at.isoformat() if row.created_at else None, } @staticmethod def _evaluation_snapshot(row: TrainingEvaluation) -> dict[str, Any]: return { "evaluation_type": row.evaluation_type, "status": row.status, "score": float(row.score or 0.0) if row.score is not None else None, } @staticmethod def _eval_suite_snapshot( row: TrainingDataset, *, latest_score: Optional[float], last_evaluated_at: Optional[Any], ) -> dict[str, Any]: selection_rules = row.selection_rules if isinstance(row.selection_rules, dict) else {} suite_kind = str(selection_rules.get("suite_kind") or "custom").strip() or "custom" return { "id": int(row.id), "dataset_type": row.dataset_type, "version": row.version, "status": row.status, "suite_kind": suite_kind, "sample_count": int(row.sample_count or 0), "latest_score": latest_score, "last_evaluated_at": MountedAIRuntimeService._dt(last_evaluated_at), } @staticmethod def _quality_asset_snapshot(row: QualityAsset) -> dict[str, Any]: return { "id": int(row.id), "asset_type": row.asset_type, "status": row.status, "prompt_key": row.prompt_key, "workflow_type": row.workflow_type, "asked_via": row.asked_via, "question_text": row.question_text, "answer_text": row.answer_text, "corrected_answer": row.corrected_answer, "notes": row.notes, "created_at": MountedAIRuntimeService._dt(row.created_at), } @staticmethod def _judge_run_snapshot(row: JudgeRun, *, asset: Optional[QualityAsset] = None) -> dict[str, Any]: return { "judge_run_id": int(row.id), "asset_id": int(row.asset_id) if row.asset_id is not None else None, "bucket_key": row.bucket_key, "judge_status": row.judge_status, "overall_score": float(row.overall_score or 0.0) if row.overall_score is not None else None, "prompt_key": row.prompt_key, "workflow_type": row.workflow_type, "asked_via": row.asked_via, "question_text": asset.question_text if asset is not None else None, "answer_text": asset.answer_text if asset is not None else None, "corrected_answer": asset.corrected_answer if asset is not None else None, "findings": row.findings, "created_at": MountedAIRuntimeService._dt(row.created_at), } async def list_sources(self, *, tenant_id: int) -> dict[str, Any]: source_map: dict[str, dict[str, Any]] = {} run_rows = ( await self.db.execute( select( MountedConversationRun.source_system, func.count(MountedConversationRun.id).label("run_count"), func.max(MountedConversationRun.updated_at).label("last_seen_at"), ) .where(MountedConversationRun.tenant_id == int(tenant_id)) .group_by(MountedConversationRun.source_system) ) ).all() for row in run_rows: source_system = str(row.source_system or "").strip() source_map[source_system] = { "source_system": source_system, "run_count": int(row.run_count or 0), "topic_count": 0, "active_topic_count": 0, "last_seen_at": self._dt(row.last_seen_at), } topic_rows = ( await self.db.execute( select( MountedTopicWorkspace.source_system, func.count(MountedTopicWorkspace.id).label("topic_count"), func.sum( case((MountedTopicWorkspace.status == "active", 1), else_=0) ).label("active_topic_count"), func.max(MountedTopicWorkspace.updated_at).label("topic_last_seen_at"), ) .where(MountedTopicWorkspace.tenant_id == int(tenant_id)) .group_by(MountedTopicWorkspace.source_system) ) ).all() for row in topic_rows: source_system = str(row.source_system or "").strip() payload = source_map.setdefault( source_system, { "source_system": source_system, "run_count": 0, "topic_count": 0, "active_topic_count": 0, "last_seen_at": None, }, ) payload["topic_count"] = int(row.topic_count or 0) payload["active_topic_count"] = int(row.active_topic_count or 0) topic_last_seen_at = self._dt(row.topic_last_seen_at) if topic_last_seen_at and not payload.get("last_seen_at"): payload["last_seen_at"] = topic_last_seen_at items = sorted(source_map.values(), key=lambda item: (item["source_system"] or "")) return {"items": items} async def list_source_activations(self, *, tenant_id: int) -> dict[str, Any]: rows = ( await self.db.execute( select(MountedSourceActivation) .where(MountedSourceActivation.tenant_id == int(tenant_id)) .order_by( MountedSourceActivation.source_system.asc(), MountedSourceActivation.scope_type.asc(), MountedSourceActivation.scope_ref.asc(), ) ) ).scalars().all() return {"items": [self._source_activation_snapshot(row) for row in rows]} async def upsert_source_activation( self, *, tenant_id: int, source_system: str, scope_type: str, scope_ref: str, status: str, cutover_mode: str, auto_reply_enabled: bool, notes: Optional[str], metadata_json: Optional[Any], ) -> dict[str, Any]: normalized_source = str(source_system or "").strip().lower() normalized_scope_type = str(scope_type or "").strip().lower() normalized_scope_ref = str(scope_ref or "").strip() normalized_status = str(status or "").strip().lower() normalized_cutover_mode = str(cutover_mode or "").strip().lower() if normalized_status not in {"active", "paused"}: raise ValueError("unsupported source activation status") if normalized_cutover_mode not in {"shadow", "mounted_only", "legacy_only"}: raise ValueError("unsupported cutover mode") row = ( await self.db.execute( select(MountedSourceActivation).where( MountedSourceActivation.tenant_id == int(tenant_id), MountedSourceActivation.source_system == normalized_source, MountedSourceActivation.scope_type == normalized_scope_type, MountedSourceActivation.scope_ref == normalized_scope_ref, ) ) ).scalar_one_or_none() if row is None: row = MountedSourceActivation( tenant_id=int(tenant_id), source_system=normalized_source, scope_type=normalized_scope_type, scope_ref=normalized_scope_ref, ) self.db.add(row) row.status = normalized_status row.cutover_mode = normalized_cutover_mode row.auto_reply_enabled = bool(auto_reply_enabled) row.notes = str(notes or "").strip() or None row.metadata_json = metadata_json if isinstance(metadata_json, (dict, list)) else None await self.db.commit() await self.db.refresh(row) return {"item": self._source_activation_snapshot(row)} async def list_tool_policies(self, *, tenant_id: int) -> dict[str, Any]: rows = ( await self.db.execute( select(MountedToolCapabilityPolicy) .where(MountedToolCapabilityPolicy.tenant_id == int(tenant_id)) .order_by(MountedToolCapabilityPolicy.tool_name.asc()) ) ).scalars().all() return {"items": [self._tool_policy_snapshot(row) for row in rows]} async def upsert_tool_policy( self, *, tenant_id: int, tool_name: str, risk_level: str, confirm_required: bool, auto_execute_allowed: bool, is_active: bool, source_scope: Optional[Any], tenant_scope: Optional[Any], policy_metadata: Optional[Any], notes: Optional[str], ) -> dict[str, Any]: normalized_tool_name = str(tool_name or "").strip() normalized_risk_level = str(risk_level or "").strip().lower() if not normalized_tool_name: raise ValueError("tool_name is required") if normalized_risk_level not in {"low", "medium", "high", "critical"}: raise ValueError("unsupported tool policy risk_level") row = ( await self.db.execute( select(MountedToolCapabilityPolicy).where( MountedToolCapabilityPolicy.tenant_id == int(tenant_id), MountedToolCapabilityPolicy.tool_name == normalized_tool_name, ) ) ).scalar_one_or_none() if row is None: row = MountedToolCapabilityPolicy( tenant_id=int(tenant_id), tool_name=normalized_tool_name, ) self.db.add(row) row.risk_level = normalized_risk_level row.confirm_required = bool(confirm_required) row.auto_execute_allowed = bool(auto_execute_allowed) row.is_active = bool(is_active) row.source_scope = source_scope if isinstance(source_scope, (dict, list)) else None row.tenant_scope = tenant_scope if isinstance(tenant_scope, (dict, list)) else None row.policy_metadata = policy_metadata if isinstance(policy_metadata, (dict, list)) else None row.notes = str(notes or "").strip() or None await self.db.commit() await self.db.refresh(row) return {"item": self._tool_policy_snapshot(row)} async def list_topics( self, *, tenant_id: int, limit: int, offset: int, mode: Optional[str] = None, ) -> dict[str, Any]: base_stmt = select(MountedTopicWorkspace).where(MountedTopicWorkspace.tenant_id == int(tenant_id)) if mode: base_stmt = base_stmt.where(MountedTopicWorkspace.mode == str(mode).strip()) total = ( await self.db.execute( select(func.count()).select_from(base_stmt.order_by(None).subquery()) ) ).scalar_one() rows = ( await self.db.execute( base_stmt.order_by(MountedTopicWorkspace.updated_at.desc(), MountedTopicWorkspace.id.desc()) .offset(int(offset)) .limit(int(limit)) ) ).scalars().all() topic_ids = [int(row.id) for row in rows] draft_map: dict[int, MountedDraftWorkspace] = {} if topic_ids: draft_rows = ( await self.db.execute( select(MountedDraftWorkspace).where( MountedDraftWorkspace.tenant_id == int(tenant_id), MountedDraftWorkspace.topic_workspace_id.in_(topic_ids), ) ) ).scalars().all() draft_map = {int(row.topic_workspace_id): row for row in draft_rows} return { "items": [self._topic_snapshot(row, draft=draft_map.get(int(row.id))) for row in rows], "total": int(total or 0), } async def list_runs( self, *, tenant_id: int, limit: int, offset: int, status: Optional[str] = None, ) -> dict[str, Any]: base_stmt = select(MountedConversationRun).where(MountedConversationRun.tenant_id == int(tenant_id)) if status: base_stmt = base_stmt.where(MountedConversationRun.status == str(status).strip()) total = ( await self.db.execute( select(func.count()).select_from(base_stmt.order_by(None).subquery()) ) ).scalar_one() rows = ( await self.db.execute( base_stmt.order_by(MountedConversationRun.updated_at.desc(), MountedConversationRun.id.desc()) .offset(int(offset)) .limit(int(limit)) ) ).scalars().all() run_ids = [int(row.id) for row in rows] trace_counts: dict[int, int] = {} if run_ids: trace_rows = ( await self.db.execute( select( MountedAgentTrace.conversation_run_id, func.count(MountedAgentTrace.id).label("trace_count"), ) .where( MountedAgentTrace.tenant_id == int(tenant_id), MountedAgentTrace.conversation_run_id.in_(run_ids), ) .group_by(MountedAgentTrace.conversation_run_id) ) ).all() trace_counts = {int(row.conversation_run_id): int(row.trace_count or 0) for row in trace_rows} return { "items": [self._run_snapshot(row, trace_count=trace_counts.get(int(row.id), 0)) for row in rows], "total": int(total or 0), } async def get_run_detail(self, *, tenant_id: int, run_id: int) -> dict[str, Any]: row = ( await self.db.execute( select(MountedConversationRun).where( MountedConversationRun.tenant_id == int(tenant_id), MountedConversationRun.id == int(run_id), ) ) ).scalar_one_or_none() if row is None: raise ValueError("mounted ai run not found") trace_rows = ( await self.db.execute( select(MountedAgentTrace) .where( MountedAgentTrace.tenant_id == int(tenant_id), MountedAgentTrace.conversation_run_id == int(row.id), ) .order_by(MountedAgentTrace.id.asc()) ) ).scalars().all() topic_row: MountedTopicWorkspace | None = None draft_row: MountedDraftWorkspace | None = None if row.topic_workspace_id is not None: topic_row = ( await self.db.execute( select(MountedTopicWorkspace).where( MountedTopicWorkspace.tenant_id == int(tenant_id), MountedTopicWorkspace.id == int(row.topic_workspace_id), ) ) ).scalar_one_or_none() if topic_row is not None: draft_row = ( await self.db.execute( select(MountedDraftWorkspace).where( MountedDraftWorkspace.tenant_id == int(tenant_id), MountedDraftWorkspace.topic_workspace_id == int(topic_row.id), ) ) ).scalar_one_or_none() execution_outcome = row.execution_outcome if isinstance(row.execution_outcome, dict) else row.execution_outcome return { "item": self._run_snapshot(row, trace_count=len(trace_rows)), "topic": self._topic_snapshot(topic_row, draft=draft_row) if topic_row is not None else None, "draft": self._draft_snapshot(draft_row) if draft_row is not None else None, "agent_traces": [self._trace_snapshot(trace) for trace in trace_rows], "context_layers_hit": row.context_layers_hit, "tool_calls": row.tool_calls, "execution_outcome": execution_outcome, "semantic_packet_summary": ( execution_outcome.get("semantic_packet_summary") if isinstance(execution_outcome, dict) else None ), "semantic_decision": ( execution_outcome.get("semantic_decision") if isinstance(execution_outcome, dict) else None ), "semantic_gate_result": ( execution_outcome.get("semantic_gate_result") if isinstance(execution_outcome, dict) else None ), "semantic_diff": ( execution_outcome.get("semantic_diff") if isinstance(execution_outcome, dict) else None ), } async def replay_run( self, *, tenant_id: int, actor_id: Optional[int], run_id: int, asset_type: str, bucket_key: Optional[str], question_text: Optional[str], answer_text: Optional[str], corrected_answer: Optional[str], notes: Optional[str], persist_quality_asset: bool, overall_score: Optional[float], rule_score: Optional[float], llm_score: Optional[float], findings: Optional[Any], ) -> dict[str, Any]: run = ( await self.db.execute( select(MountedConversationRun).where( MountedConversationRun.tenant_id == int(tenant_id), MountedConversationRun.id == int(run_id), ) ) ).scalar_one_or_none() if run is None: raise ValueError("mounted ai run not found") topic = None if run.topic_workspace_id is not None: topic = ( await self.db.execute( select(MountedTopicWorkspace).where( MountedTopicWorkspace.tenant_id == int(tenant_id), MountedTopicWorkspace.id == int(run.topic_workspace_id), ) ) ).scalar_one_or_none() topic_metadata = topic.metadata_json if isinstance(getattr(topic, "metadata_json", None), dict) else {} normalized_question = str( question_text or topic_metadata.get("last_message_preview") or run.source_message_id or f"mounted run {run.id}" ).strip() normalized_answer = str(answer_text or run.final_reply or run.error_text or "").strip() if not normalized_answer and run.execution_outcome is not None: normalized_answer = json.dumps(run.execution_outcome, ensure_ascii=False) if not normalized_answer: raise ValueError("mounted ai run missing replayable answer") normalized_asset_type = str(asset_type or "").strip().lower() or "failure" if normalized_asset_type not in {"golden", "failure"}: raise ValueError("unsupported replay asset_type") asset_row: Optional[QualityAsset] = None if persist_quality_asset: asset_row = QualityAsset( tenant_id=int(tenant_id), project_id=None, asset_type=normalized_asset_type, status="approved", prompt_key="mounted.runtime.reply", workflow_type=f"mounted.{run.mode_after or run.mode_before or 'chat'}", asked_via=run.source_system, route_rule_name=run.trigger_reason, question_text=normalized_question, answer_text=normalized_answer, corrected_answer=str(corrected_answer or "").strip() or None, source_snapshot={ "run_id": int(run.id), "trace_id": run.trace_id, "topic_workspace_id": int(run.topic_workspace_id) if run.topic_workspace_id is not None else None, "context_layers_hit": run.context_layers_hit, "execution_outcome": run.execution_outcome, }, tags=["mounted_runtime", run.active_agent or "ConversationOrchestrator"], notes=str(notes or "").strip() or None, approved_by=actor_id, approved_at=now_cn_naive().isoformat(), ) self.db.add(asset_row) await self.db.flush() judge_row = JudgeRun( tenant_id=int(tenant_id), project_id=None, asset_id=int(asset_row.id) if asset_row is not None else None, prompt_key="mounted.runtime.reply", workflow_type=f"mounted.{run.mode_after or run.mode_before or 'chat'}", asked_via=run.source_system, judge_status="completed", bucket_key=str(bucket_key or f"mounted_runtime.{(run.active_agent or 'conversation_orchestrator').lower()}").strip(), rule_score=rule_score, llm_score=llm_score, overall_score=overall_score, findings=findings if findings is not None else [], metadata_json={ "run_id": int(run.id), "trace_id": run.trace_id, "topic_key": getattr(topic, "topic_key", None), "trigger_reason": run.trigger_reason, }, ) self.db.add(judge_row) await self.db.commit() await self.db.refresh(judge_row) if asset_row is not None: await self.db.refresh(asset_row) return { "run_id": int(run.id), "asset": self._quality_asset_snapshot(asset_row) if asset_row is not None else None, "judge_run": self._judge_run_snapshot(judge_row, asset=asset_row), } async def get_cutover_status(self, *, tenant_id: int) -> dict[str, Any]: topic_workspace_count = ( await self.db.execute( select(func.count(MountedTopicWorkspace.id)).where( MountedTopicWorkspace.tenant_id == int(tenant_id) ) ) ).scalar_one() conversation_run_count = ( await self.db.execute( select(func.count(MountedConversationRun.id)).where( MountedConversationRun.tenant_id == int(tenant_id) ) ) ).scalar_one() tool_policy_count = ( await self.db.execute( select(func.count(MountedToolCapabilityPolicy.id)).where( MountedToolCapabilityPolicy.tenant_id == int(tenant_id) ) ) ).scalar_one() observed_sources = ( await self.db.execute( select(MountedConversationRun.source_system) .where(MountedConversationRun.tenant_id == int(tenant_id)) .distinct() .order_by(MountedConversationRun.source_system.asc()) ) ).scalars().all() legacy_cutover_items = self._build_legacy_cutover_items() active_item_count = sum(1 for item in legacy_cutover_items if item["status"] == "active") retired_item_count = sum(1 for item in legacy_cutover_items if item["status"] == "retired") legacy_cutover_state = "retired" if active_item_count == 0 else "active_legacy_paths_remaining" return { "mounted_runtime_ready": True, "topic_workspace_count": int(topic_workspace_count or 0), "conversation_run_count": int(conversation_run_count or 0), "tool_policy_count": int(tool_policy_count or 0), "observed_source_systems": [str(item) for item in observed_sources if str(item or "").strip()], "legacy_cutover_state": legacy_cutover_state, "retired_item_count": int(retired_item_count), "active_item_count": int(active_item_count), "legacy_cutover_items": legacy_cutover_items, } async def list_training_ops(self, *, tenant_id: int) -> dict[str, Any]: dataset_count = ( await self.db.execute( select(func.count(TrainingDataset.id)).where(TrainingDataset.tenant_id == int(tenant_id)) ) ).scalar_one() ready_eval_suite_count = ( await self.db.execute( select(func.count(TrainingDataset.id)).where( TrainingDataset.tenant_id == int(tenant_id), TrainingDataset.dataset_type == "eval", TrainingDataset.status == "ready", ) ) ).scalar_one() pending_job_count = ( await self.db.execute( select(func.count(TrainingJob.id)).where( TrainingJob.tenant_id == int(tenant_id), TrainingJob.status.in_(("queued", "running")), ) ) ).scalar_one() candidate_deployment_count = ( await self.db.execute( select(func.count(TrainedModelDeployment.id)).where( TrainedModelDeployment.tenant_id == int(tenant_id), TrainedModelDeployment.deployment_status == "candidate", ) ) ).scalar_one() approved_golden_asset_count = ( await self.db.execute( select(func.count(QualityAsset.id)).where( QualityAsset.tenant_id == int(tenant_id), QualityAsset.asset_type == "golden", QualityAsset.status == "approved", ) ) ).scalar_one() approved_failure_asset_count = ( await self.db.execute( select(func.count(QualityAsset.id)).where( QualityAsset.tenant_id == int(tenant_id), QualityAsset.asset_type == "failure", QualityAsset.status == "approved", ) ) ).scalar_one() preference_pair_count = ( await self.db.execute( select(func.count(QualityAssetPair.id)).where( QualityAssetPair.tenant_id == int(tenant_id), QualityAssetPair.status == "approved", ) ) ).scalar_one() hard_case_filter = case((JudgeRun.overall_score.is_(None), True), else_=JudgeRun.overall_score < 90) hard_case_count = ( await self.db.execute( select(func.count(JudgeRun.id)).where( JudgeRun.tenant_id == int(tenant_id), hard_case_filter, ) ) ).scalar_one() last_training_job_at = ( await self.db.execute( select(func.max(TrainingJob.created_at)).where(TrainingJob.tenant_id == int(tenant_id)) ) ).scalar_one() bucket_rows = ( await self.db.execute( select( JudgeRun.bucket_key, func.count(JudgeRun.id).label("run_count"), func.avg(JudgeRun.overall_score).label("avg_score"), func.max(JudgeRun.created_at).label("last_seen_at"), ) .where( JudgeRun.tenant_id == int(tenant_id), hard_case_filter, ) .group_by(JudgeRun.bucket_key) .order_by(func.count(JudgeRun.id).desc(), func.max(JudgeRun.created_at).desc()) .limit(5) ) ).all() recent_datasets = ( await self.db.execute( select(TrainingDataset) .where(TrainingDataset.tenant_id == int(tenant_id)) .order_by(TrainingDataset.id.desc()) .limit(4) ) ).scalars().all() recent_jobs = ( await self.db.execute( select(TrainingJob) .where(TrainingJob.tenant_id == int(tenant_id)) .order_by(TrainingJob.id.desc()) .limit(6) ) ).scalars().all() acceptance_gate = await TrainingPipelineService(self.db).get_acceptance_gate(tenant_id=tenant_id) return { "dataset_count": int(dataset_count or 0), "ready_eval_suite_count": int(ready_eval_suite_count or 0), "pending_job_count": int(pending_job_count or 0), "candidate_deployment_count": int(candidate_deployment_count or 0), "approved_golden_asset_count": int(approved_golden_asset_count or 0), "approved_failure_asset_count": int(approved_failure_asset_count or 0), "preference_pair_count": int(preference_pair_count or 0), "hard_case_count": int(hard_case_count or 0), "last_training_job_at": self._dt(last_training_job_at), "regression_buckets": [ { "bucket_key": str(row.bucket_key or "unbucketed"), "run_count": int(row.run_count or 0), "avg_score": float(row.avg_score or 0.0) if row.avg_score is not None else None, "last_seen_at": self._dt(row.last_seen_at), } for row in bucket_rows ], "recent_datasets": [ { "id": int(row.id), "dataset_type": row.dataset_type, "version": row.version, "status": row.status, "sample_count": int(row.sample_count or 0), "suite_kind": ( str((row.selection_rules or {}).get("suite_kind") or "").strip() or None if isinstance(row.selection_rules, dict) else None ), } for row in recent_datasets ], "recent_jobs": [ { "id": int(row.id), "job_type": row.job_type, "target_slot": row.target_slot, "status": row.status, "provider_backend": row.provider_backend, "dataset_id": int(row.dataset_id), "artifact_ref": row.artifact_ref, } for row in recent_jobs ], "acceptance_gate": acceptance_gate, } async def list_hard_cases(self, *, tenant_id: int, limit: int = 20, offset: int = 0) -> dict[str, Any]: hard_case_filter = case((JudgeRun.overall_score.is_(None), True), else_=JudgeRun.overall_score < 90) base_stmt = ( select(JudgeRun) .where( JudgeRun.tenant_id == int(tenant_id), hard_case_filter, ) ) total = ( await self.db.execute( select(func.count()).select_from(base_stmt.order_by(None).subquery()) ) ).scalar_one() rows = ( await self.db.execute( base_stmt.order_by(JudgeRun.created_at.desc(), JudgeRun.id.desc()) .offset(int(offset)) .limit(int(limit)) ) ).scalars().all() asset_ids = [int(row.asset_id) for row in rows if row.asset_id is not None] asset_map: dict[int, QualityAsset] = {} if asset_ids: asset_rows = ( await self.db.execute( select(QualityAsset).where( QualityAsset.tenant_id == int(tenant_id), QualityAsset.id.in_(asset_ids), ) ) ).scalars().all() asset_map = {int(row.id): row for row in asset_rows} return { "items": [self._judge_run_snapshot(row, asset=asset_map.get(int(row.asset_id))) for row in rows], "total": int(total or 0), } async def list_eval_suites(self, *, tenant_id: int) -> dict[str, Any]: datasets = ( await self.db.execute( select(TrainingDataset) .where( TrainingDataset.tenant_id == int(tenant_id), TrainingDataset.dataset_type == "eval", ) .order_by(TrainingDataset.id.desc()) ) ).scalars().all() if not datasets: return {"items": [], "total": 0} dataset_ids = [int(row.id) for row in datasets] jobs = ( await self.db.execute( select(TrainingJob) .where( TrainingJob.tenant_id == int(tenant_id), TrainingJob.dataset_id.in_(dataset_ids), ) .order_by(TrainingJob.id.desc()) ) ).scalars().all() latest_job_by_dataset: dict[int, TrainingJob] = {} for job in jobs: latest_job_by_dataset.setdefault(int(job.dataset_id), job) job_ids = [int(job.id) for job in latest_job_by_dataset.values()] evaluations = [] if job_ids: evaluations = ( await self.db.execute( select(TrainingEvaluation) .where( TrainingEvaluation.tenant_id == int(tenant_id), TrainingEvaluation.training_job_id.in_(job_ids), ) .order_by(TrainingEvaluation.id.desc()) ) ).scalars().all() evaluations_by_job: dict[int, list[TrainingEvaluation]] = {} for item in evaluations: evaluations_by_job.setdefault(int(item.training_job_id), []).append(item) items: list[dict[str, Any]] = [] for dataset in datasets: latest_job = latest_job_by_dataset.get(int(dataset.id)) latest_evaluations = evaluations_by_job.get(int(latest_job.id), []) if latest_job is not None else [] completed_scores = [ float(row.score or 0.0) for row in latest_evaluations if row.score is not None and str(row.status or "").strip().lower() == "completed" ] latest_score = round(sum(completed_scores) / len(completed_scores), 2) if completed_scores else None last_evaluated_at = None if latest_evaluations: last_evaluated_at = latest_evaluations[0].updated_at items.append( self._eval_suite_snapshot( dataset, latest_score=latest_score, last_evaluated_at=last_evaluated_at, ) ) return {"items": items, "total": len(items)} async def get_eval_suite_detail(self, *, tenant_id: int, dataset_id: int) -> dict[str, Any]: dataset = ( await self.db.execute( select(TrainingDataset).where( TrainingDataset.tenant_id == int(tenant_id), TrainingDataset.id == int(dataset_id), TrainingDataset.dataset_type == "eval", ) ) ).scalar_one_or_none() if dataset is None: raise ValueError("mounted ai eval suite not found") latest_job = ( await self.db.execute( select(TrainingJob) .where( TrainingJob.tenant_id == int(tenant_id), TrainingJob.dataset_id == int(dataset.id), ) .order_by(TrainingJob.id.desc()) .limit(1) ) ).scalar_one_or_none() evaluations: list[TrainingEvaluation] = [] if latest_job is not None: evaluations = ( await self.db.execute( select(TrainingEvaluation) .where( TrainingEvaluation.tenant_id == int(tenant_id), TrainingEvaluation.training_job_id == int(latest_job.id), ) .order_by(TrainingEvaluation.id.asc()) ) ).scalars().all() completed_scores = [ float(row.score or 0.0) for row in evaluations if row.score is not None and str(row.status or "").strip().lower() == "completed" ] latest_score = round(sum(completed_scores) / len(completed_scores), 2) if completed_scores else None last_evaluated_at = evaluations[-1].updated_at if evaluations else None return { "item": self._eval_suite_snapshot( dataset, latest_score=latest_score, last_evaluated_at=last_evaluated_at, ), "selection_rules": dataset.selection_rules if isinstance(dataset.selection_rules, dict) else {}, "notes": dataset.notes, "evaluations": [self._evaluation_snapshot(row) for row in evaluations], }