"""PMFlow OS AI answer feedback service.""" from __future__ import annotations from typing import Any, Optional from fastapi import HTTPException from sqlalchemy import and_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified from ..models import AICallLog, JudgeRun, OptimizerCandidate, PMFlowAIFeedbackEvent, PMFlowChangeSet, PMFlowThreadMessage from ..schemas.pmflow_os import PMFlowAIFeedbackCreateRequest, PMFlowAIFeedbackResponse from .pmflow_change_review_service import PMFlowChangeReviewService class PMFlowAIFeedbackService: def __init__(self, db: AsyncSession, *, tenant_id: int): self.db = db self.tenant_id = int(tenant_id) async def create_or_update( self, *, project_id: int, body: PMFlowAIFeedbackCreateRequest, user_id: int, ) -> PMFlowAIFeedbackResponse: target_type = body.target_type target_id = body.target_id.strip() message = await self._resolve_message(project_id=project_id, body=body) change_set = await self._resolve_change_set(project_id=project_id, body=body) if target_type == "thread_message" and message is None: raise HTTPException(status_code=404, detail="feedback_target_message_not_found") if target_type == "change_set" and change_set is None: raise HTTPException(status_code=404, detail="feedback_target_change_set_not_found") if target_type == "copilot_answer" and message is None and change_set is None: raise HTTPException(status_code=422, detail="feedback_target_anchor_required") prompt_meta = await self._prompt_meta(body=body, message=message, change_set=change_set) result = await self.db.execute( select(PMFlowAIFeedbackEvent).where( and_( PMFlowAIFeedbackEvent.tenant_id == self.tenant_id, PMFlowAIFeedbackEvent.project_id == int(project_id), PMFlowAIFeedbackEvent.actor_user_id == int(user_id), PMFlowAIFeedbackEvent.target_type == target_type, PMFlowAIFeedbackEvent.target_id == target_id, ) ) ) row = result.scalar_one_or_none() if row is None: row = PMFlowAIFeedbackEvent( tenant_id=self.tenant_id, project_id=int(project_id), actor_user_id=int(user_id), target_type=target_type, target_id=target_id, ) self.db.add(row) row.thread_id = self._int_or_none(body.thread_id) or (int(message.thread_id) if message is not None else None) or ( int(change_set.thread_id) if change_set is not None else None ) row.message_id = self._int_or_none(body.message_id) or (int(message.id) if message is not None else None) row.change_set_id = self._int_or_none(body.change_set_id) or (int(change_set.id) if change_set is not None else None) row.ai_call_log_id = prompt_meta.get("ai_call_log_id") row.answer_kind = body.answer_kind row.rating = body.rating row.reason_tags = [tag.strip()[:80] for tag in body.reason_tags if tag and tag.strip()][:12] row.comment = body.comment.strip() if body.comment else None row.scene_key = prompt_meta.get("scene_key") row.prompt_key = prompt_meta.get("prompt_key") row.prompt_release_id = prompt_meta.get("prompt_release_id") row.runtime_profile_id = prompt_meta.get("runtime_profile_id") row.model_provider = prompt_meta.get("model_provider") row.model_name = prompt_meta.get("model_name") current_metadata = row.feedback_metadata if isinstance(row.feedback_metadata, dict) else {} row.feedback_metadata = { **current_metadata, **(body.feedback_metadata or {}), "promptOptimizationUse": "quality_signal_for_scene_prompt_iteration", "source": "pmflow_os_command_center", } await self.db.flush() await self._ensure_prompt_optimization_candidate(row) await self.db.commit() await self.db.refresh(row) return self.serialize(row) async def list_project_feedback(self, *, project_id: int, user_id: Optional[int] = None) -> list[PMFlowAIFeedbackResponse]: clauses = [ PMFlowAIFeedbackEvent.tenant_id == self.tenant_id, PMFlowAIFeedbackEvent.project_id == int(project_id), ] if user_id is not None: clauses.append(PMFlowAIFeedbackEvent.actor_user_id == int(user_id)) result = await self.db.execute( select(PMFlowAIFeedbackEvent) .where(and_(*clauses)) .order_by(PMFlowAIFeedbackEvent.updated_at.desc(), PMFlowAIFeedbackEvent.id.desc()) ) return [self.serialize(row) for row in result.scalars().all()] async def _resolve_message( self, *, project_id: int, body: PMFlowAIFeedbackCreateRequest, ) -> Optional[PMFlowThreadMessage]: message_id = self._int_or_none(body.message_id) or ( self._int_or_none(body.target_id) if body.target_type == "thread_message" else None ) if message_id is None: return None result = await self.db.execute( select(PMFlowThreadMessage).where( and_( PMFlowThreadMessage.tenant_id == self.tenant_id, PMFlowThreadMessage.project_id == int(project_id), PMFlowThreadMessage.id == int(message_id), ) ) ) return result.scalar_one_or_none() async def _resolve_change_set( self, *, project_id: int, body: PMFlowAIFeedbackCreateRequest, ) -> Optional[PMFlowChangeSet]: change_set_id = self._int_or_none(body.change_set_id) or ( self._int_or_none(body.target_id) if body.target_type == "change_set" else None ) if change_set_id is None: return None result = await self.db.execute( select(PMFlowChangeSet).where( and_( PMFlowChangeSet.tenant_id == self.tenant_id, PMFlowChangeSet.project_id == int(project_id), PMFlowChangeSet.id == int(change_set_id), ) ) ) return result.scalar_one_or_none() async def _prompt_meta( self, *, body: PMFlowAIFeedbackCreateRequest, message: Optional[PMFlowThreadMessage], change_set: Optional[PMFlowChangeSet], ) -> dict[str, Any]: meta: dict[str, Any] = { "scene_key": body.scene_key, "prompt_key": body.prompt_key, "prompt_release_id": body.prompt_release_id, "runtime_profile_id": body.runtime_profile_id, "model_provider": body.model_provider, "model_name": body.model_name, "ai_call_log_id": self._int_or_none(body.ai_call_log_id), } if message is not None and isinstance(message.message_metadata, dict): msg_meta = message.message_metadata meta["scene_key"] = meta["scene_key"] or msg_meta.get("scene_key") or msg_meta.get("scene") meta["prompt_key"] = meta["prompt_key"] or msg_meta.get("prompt_key") meta["prompt_release_id"] = meta["prompt_release_id"] or self._int_or_none(msg_meta.get("prompt_release_id")) meta["runtime_profile_id"] = meta["runtime_profile_id"] or msg_meta.get("runtime_profile_id") meta["model_provider"] = meta["model_provider"] or msg_meta.get("provider") meta["model_name"] = meta["model_name"] or msg_meta.get("model") routing_meta = change_set.routing_meta if change_set is not None and isinstance(change_set.routing_meta, dict) else {} if change_set is not None: meta["scene_key"] = meta["scene_key"] or PMFlowChangeReviewService.AI_EXTRACTION_SCENE_KEY meta["prompt_key"] = meta["prompt_key"] or PMFlowChangeReviewService.AI_EXTRACTION_PROMPT_KEY meta["prompt_release_id"] = meta["prompt_release_id"] or self._int_or_none(routing_meta.get("prompt_release_id")) meta["runtime_profile_id"] = ( meta["runtime_profile_id"] or routing_meta.get("function_runtime_profile_version") or routing_meta.get("runtime_profile_id") or routing_meta.get("profile_id") ) meta["model_provider"] = meta["model_provider"] or routing_meta.get("provider") meta["model_name"] = meta["model_name"] or routing_meta.get("model") meta["ai_call_log_id"] = meta["ai_call_log_id"] or self._int_or_none(getattr(change_set, "llm_call_id", None)) if meta["ai_call_log_id"] is None and change_set is not None and change_set.llm_call_id: trace_result = await self.db.execute( select(AICallLog) .where( and_( AICallLog.tenant_id == self.tenant_id, AICallLog.trace_id == str(change_set.llm_call_id), ) ) .order_by(AICallLog.created_at.desc(), AICallLog.id.desc()) ) ai_call = trace_result.scalars().first() if ai_call is not None: meta["ai_call_log_id"] = int(ai_call.id) meta["prompt_key"] = meta["prompt_key"] or ai_call.prompt_key meta["prompt_release_id"] = meta["prompt_release_id"] or ai_call.prompt_release_id meta["model_provider"] = meta["model_provider"] or ai_call.provider meta["model_name"] = meta["model_name"] or ai_call.model routing = ai_call.routing_meta if isinstance(ai_call.routing_meta, dict) else {} meta["runtime_profile_id"] = ( meta["runtime_profile_id"] or routing.get("function_runtime_profile_version") or routing.get("runtime_profile_id") ) return meta async def _ensure_prompt_optimization_candidate(self, row: PMFlowAIFeedbackEvent) -> None: metadata = dict(row.feedback_metadata or {}) prompt_optimization = dict(metadata.get("promptOptimization") or {}) candidate_id = self._int_or_none(prompt_optimization.get("candidateId")) if row.rating != "negative": if candidate_id is not None: candidate = ( await self.db.execute( select(OptimizerCandidate).where( and_( OptimizerCandidate.tenant_id == self.tenant_id, OptimizerCandidate.id == int(candidate_id), ) ) ) ).scalar_one_or_none() if candidate is not None and candidate.status == "draft": candidate.status = "superseded" candidate.notes = "用户将 PMFlow OS AI 回答反馈改为赞同，自动挂起此前负反馈生成的优化候选。" prompt_optimization["status"] = "superseded_by_positive_feedback" metadata["promptOptimization"] = prompt_optimization row.feedback_metadata = metadata flag_modified(row, "feedback_metadata") return if not row.prompt_key: return judge_run_id = self._int_or_none(prompt_optimization.get("judgeRunId")) findings = self._feedback_findings(row) suggestion = self._feedback_prompt_suggestion(row, findings=findings) evidence = { "feedback_event_id": int(row.id), "target_type": row.target_type, "target_id": row.target_id, "answer_kind": row.answer_kind, "rating": row.rating, "reason_tags": list(row.reason_tags or []), "comment": row.comment, "scene_key": row.scene_key, "prompt_release_id": row.prompt_release_id, "runtime_profile_id": row.runtime_profile_id, "ai_call_log_id": row.ai_call_log_id, "findings": findings, } judge: Optional[JudgeRun] = None if judge_run_id is not None: judge = ( await self.db.execute( select(JudgeRun).where( and_( JudgeRun.tenant_id == self.tenant_id, JudgeRun.id == int(judge_run_id), ) ) ) ).scalar_one_or_none() if judge is None: judge = JudgeRun( tenant_id=self.tenant_id, project_id=int(row.project_id), prompt_key=row.prompt_key, workflow_type=row.scene_key or row.answer_kind, asked_via="pmflow_os", judge_status="feedback_review", bucket_key=f"pmflow_ai_feedback:{row.prompt_key}:{row.answer_kind}", rule_score=0.2, overall_score=0.2, findings=findings, metadata_json=evidence, ) self.db.add(judge) await self.db.flush() else: judge.findings = findings judge.metadata_json = evidence flag_modified(judge, "findings") flag_modified(judge, "metadata_json") candidate: Optional[OptimizerCandidate] = None if candidate_id is not None: candidate = ( await self.db.execute( select(OptimizerCandidate).where( and_( OptimizerCandidate.tenant_id == self.tenant_id, OptimizerCandidate.id == int(candidate_id), ) ) ) ).scalar_one_or_none() if candidate is None: candidate = OptimizerCandidate( tenant_id=self.tenant_id, project_id=int(row.project_id), judge_run_id=int(judge.id), candidate_type="prompt_change", target_type="prompt_key", target_ref=row.prompt_key, status="draft", suggestion=suggestion, evidence=evidence, notes="PMFlow OS AI feedback 自动生成的 prompt 优化候选，需平台 AI Ops 人工复核后再创建 release。", ) self.db.add(candidate) await self.db.flush() else: candidate.suggestion = suggestion candidate.evidence = evidence flag_modified(candidate, "suggestion") flag_modified(candidate, "evidence") metadata["promptOptimization"] = { "status": "draft_optimizer_candidate", "judgeRunId": int(judge.id), "candidateId": int(candidate.id), "targetPromptKey": row.prompt_key, } row.feedback_metadata = metadata flag_modified(row, "feedback_metadata") @staticmethod def _feedback_findings(row: PMFlowAIFeedbackEvent) -> list[dict[str, Any]]: tags = [str(tag).strip() for tag in (row.reason_tags or []) if str(tag).strip()] if not tags and row.comment: tags = ["用户补充说明"] return [ { "code": PMFlowAIFeedbackService._reason_code(tag), "label": tag, "severity": "high" if tag in {"不准确", "不该生成这个结果"} else "medium", "message": f"用户对 {row.answer_kind} 反馈为不赞同：{tag}", } for tag in tags ] or [ { "code": "negative_without_reason", "label": "未填写原因", "severity": "medium", "message": f"用户对 {row.answer_kind} 反馈为不赞同，但未选择原因标签。", } ] @staticmethod def _feedback_prompt_suggestion(row: PMFlowAIFeedbackEvent, *, findings: list[dict[str, Any]]) -> dict[str, Any]: labels = [str(item.get("label") or "").strip() for item in findings if isinstance(item, dict)] actions: list[str] = [] if "不准确" in labels: actions.append("强化事实约束、证据引用和不确定性表达，避免把候选对象说成已确认结论。") if "漏掉重点" in labels: actions.append("在抽取 prompt 中补充关键对象覆盖检查，优先覆盖用户最新输入里的需求、风险、决策和任务。") if "不够可执行" in labels: actions.append("要求输出包含可审查对象、可执行下一步、边界条件和验收信号。") if "不符合项目上下文" in labels: actions.append("提高项目目标、当前阶段、已有 ChangeSet 与 Evidence 的上下文权重。") if "表达不清" in labels: actions.append("约束回答结构，先给结论，再给依据和待确认动作。") if "不该生成这个结果" in labels: actions.append("增加生成前 gate：证据不足时显式待补充，不创建候选对象。") if not actions: actions.append("复核负反馈样本，补充场景化约束和回归 golden case。") return { "summary": f"基于 PMFlow OS 用户负反馈优化 {row.prompt_key}", "reason_tags": labels, "recommended_actions": actions, "target_answer_kind": row.answer_kind, "scene_key": row.scene_key, } @staticmethod def _reason_code(label: str) -> str: mapping = { "不准确": "inaccurate", "漏掉重点": "missed_key_point", "不够可执行": "not_actionable", "不符合项目上下文": "context_mismatch", "表达不清": "unclear_expression", "不该生成这个结果": "should_not_generate", "用户补充说明": "user_comment", } return mapping.get(label, "negative_feedback") @staticmethod def _int_or_none(value: Any) -> Optional[int]: if value is None: return None try: text = str(value).strip() if not text: return None return int(text) except (TypeError, ValueError): return None @staticmethod def serialize(row: PMFlowAIFeedbackEvent) -> PMFlowAIFeedbackResponse: return PMFlowAIFeedbackResponse( id=str(row.id), projectId=str(row.project_id), threadId=str(row.thread_id) if row.thread_id is not None else None, messageId=str(row.message_id) if row.message_id is not None else None, changeSetId=str(row.change_set_id) if row.change_set_id is not None else None, aiCallLogId=str(row.ai_call_log_id) if row.ai_call_log_id is not None else None, targetType=row.target_type, targetId=row.target_id, answerKind=row.answer_kind, rating=row.rating, reasonTags=list(row.reason_tags or []), comment=row.comment, sceneKey=row.scene_key, promptKey=row.prompt_key, promptReleaseId=row.prompt_release_id, runtimeProfileId=row.runtime_profile_id, modelProvider=row.model_provider, modelName=row.model_name, metadata=dict(row.feedback_metadata or {}), createdAt=row.created_at.isoformat() if row.created_at else "", updatedAt=row.updated_at.isoformat() if row.updated_at else "", )