434 lines
20 KiB
Python
434 lines
20 KiB
Python
"""PMFlow OS AI answer feedback service."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any, Optional
|
||
|
||
from fastapi import HTTPException
|
||
from sqlalchemy import and_, select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
from sqlalchemy.orm.attributes import flag_modified
|
||
|
||
from ..models import AICallLog, JudgeRun, OptimizerCandidate, PMFlowAIFeedbackEvent, PMFlowChangeSet, PMFlowThreadMessage
|
||
from ..schemas.pmflow_os import PMFlowAIFeedbackCreateRequest, PMFlowAIFeedbackResponse
|
||
from .pmflow_change_review_service import PMFlowChangeReviewService
|
||
|
||
|
||
class PMFlowAIFeedbackService:
|
||
def __init__(self, db: AsyncSession, *, tenant_id: int):
|
||
self.db = db
|
||
self.tenant_id = int(tenant_id)
|
||
|
||
async def create_or_update(
|
||
self,
|
||
*,
|
||
project_id: int,
|
||
body: PMFlowAIFeedbackCreateRequest,
|
||
user_id: int,
|
||
) -> PMFlowAIFeedbackResponse:
|
||
target_type = body.target_type
|
||
target_id = body.target_id.strip()
|
||
message = await self._resolve_message(project_id=project_id, body=body)
|
||
change_set = await self._resolve_change_set(project_id=project_id, body=body)
|
||
if target_type == "thread_message" and message is None:
|
||
raise HTTPException(status_code=404, detail="feedback_target_message_not_found")
|
||
if target_type == "change_set" and change_set is None:
|
||
raise HTTPException(status_code=404, detail="feedback_target_change_set_not_found")
|
||
if target_type == "copilot_answer" and message is None and change_set is None:
|
||
raise HTTPException(status_code=422, detail="feedback_target_anchor_required")
|
||
|
||
prompt_meta = await self._prompt_meta(body=body, message=message, change_set=change_set)
|
||
result = await self.db.execute(
|
||
select(PMFlowAIFeedbackEvent).where(
|
||
and_(
|
||
PMFlowAIFeedbackEvent.tenant_id == self.tenant_id,
|
||
PMFlowAIFeedbackEvent.project_id == int(project_id),
|
||
PMFlowAIFeedbackEvent.actor_user_id == int(user_id),
|
||
PMFlowAIFeedbackEvent.target_type == target_type,
|
||
PMFlowAIFeedbackEvent.target_id == target_id,
|
||
)
|
||
)
|
||
)
|
||
row = result.scalar_one_or_none()
|
||
if row is None:
|
||
row = PMFlowAIFeedbackEvent(
|
||
tenant_id=self.tenant_id,
|
||
project_id=int(project_id),
|
||
actor_user_id=int(user_id),
|
||
target_type=target_type,
|
||
target_id=target_id,
|
||
)
|
||
self.db.add(row)
|
||
|
||
row.thread_id = self._int_or_none(body.thread_id) or (int(message.thread_id) if message is not None else None) or (
|
||
int(change_set.thread_id) if change_set is not None else None
|
||
)
|
||
row.message_id = self._int_or_none(body.message_id) or (int(message.id) if message is not None else None)
|
||
row.change_set_id = self._int_or_none(body.change_set_id) or (int(change_set.id) if change_set is not None else None)
|
||
row.ai_call_log_id = prompt_meta.get("ai_call_log_id")
|
||
row.answer_kind = body.answer_kind
|
||
row.rating = body.rating
|
||
row.reason_tags = [tag.strip()[:80] for tag in body.reason_tags if tag and tag.strip()][:12]
|
||
row.comment = body.comment.strip() if body.comment else None
|
||
row.scene_key = prompt_meta.get("scene_key")
|
||
row.prompt_key = prompt_meta.get("prompt_key")
|
||
row.prompt_release_id = prompt_meta.get("prompt_release_id")
|
||
row.runtime_profile_id = prompt_meta.get("runtime_profile_id")
|
||
row.model_provider = prompt_meta.get("model_provider")
|
||
row.model_name = prompt_meta.get("model_name")
|
||
current_metadata = row.feedback_metadata if isinstance(row.feedback_metadata, dict) else {}
|
||
row.feedback_metadata = {
|
||
**current_metadata,
|
||
**(body.feedback_metadata or {}),
|
||
"promptOptimizationUse": "quality_signal_for_scene_prompt_iteration",
|
||
"source": "pmflow_os_command_center",
|
||
}
|
||
await self.db.flush()
|
||
await self._ensure_prompt_optimization_candidate(row)
|
||
await self.db.commit()
|
||
await self.db.refresh(row)
|
||
return self.serialize(row)
|
||
|
||
async def list_project_feedback(self, *, project_id: int, user_id: Optional[int] = None) -> list[PMFlowAIFeedbackResponse]:
|
||
clauses = [
|
||
PMFlowAIFeedbackEvent.tenant_id == self.tenant_id,
|
||
PMFlowAIFeedbackEvent.project_id == int(project_id),
|
||
]
|
||
if user_id is not None:
|
||
clauses.append(PMFlowAIFeedbackEvent.actor_user_id == int(user_id))
|
||
result = await self.db.execute(
|
||
select(PMFlowAIFeedbackEvent)
|
||
.where(and_(*clauses))
|
||
.order_by(PMFlowAIFeedbackEvent.updated_at.desc(), PMFlowAIFeedbackEvent.id.desc())
|
||
)
|
||
return [self.serialize(row) for row in result.scalars().all()]
|
||
|
||
async def _resolve_message(
|
||
self,
|
||
*,
|
||
project_id: int,
|
||
body: PMFlowAIFeedbackCreateRequest,
|
||
) -> Optional[PMFlowThreadMessage]:
|
||
message_id = self._int_or_none(body.message_id) or (
|
||
self._int_or_none(body.target_id) if body.target_type == "thread_message" else None
|
||
)
|
||
if message_id is None:
|
||
return None
|
||
result = await self.db.execute(
|
||
select(PMFlowThreadMessage).where(
|
||
and_(
|
||
PMFlowThreadMessage.tenant_id == self.tenant_id,
|
||
PMFlowThreadMessage.project_id == int(project_id),
|
||
PMFlowThreadMessage.id == int(message_id),
|
||
)
|
||
)
|
||
)
|
||
return result.scalar_one_or_none()
|
||
|
||
async def _resolve_change_set(
|
||
self,
|
||
*,
|
||
project_id: int,
|
||
body: PMFlowAIFeedbackCreateRequest,
|
||
) -> Optional[PMFlowChangeSet]:
|
||
change_set_id = self._int_or_none(body.change_set_id) or (
|
||
self._int_or_none(body.target_id) if body.target_type == "change_set" else None
|
||
)
|
||
if change_set_id is None:
|
||
return None
|
||
result = await self.db.execute(
|
||
select(PMFlowChangeSet).where(
|
||
and_(
|
||
PMFlowChangeSet.tenant_id == self.tenant_id,
|
||
PMFlowChangeSet.project_id == int(project_id),
|
||
PMFlowChangeSet.id == int(change_set_id),
|
||
)
|
||
)
|
||
)
|
||
return result.scalar_one_or_none()
|
||
|
||
async def _prompt_meta(
|
||
self,
|
||
*,
|
||
body: PMFlowAIFeedbackCreateRequest,
|
||
message: Optional[PMFlowThreadMessage],
|
||
change_set: Optional[PMFlowChangeSet],
|
||
) -> dict[str, Any]:
|
||
meta: dict[str, Any] = {
|
||
"scene_key": body.scene_key,
|
||
"prompt_key": body.prompt_key,
|
||
"prompt_release_id": body.prompt_release_id,
|
||
"runtime_profile_id": body.runtime_profile_id,
|
||
"model_provider": body.model_provider,
|
||
"model_name": body.model_name,
|
||
"ai_call_log_id": self._int_or_none(body.ai_call_log_id),
|
||
}
|
||
if message is not None and isinstance(message.message_metadata, dict):
|
||
msg_meta = message.message_metadata
|
||
meta["scene_key"] = meta["scene_key"] or msg_meta.get("scene_key") or msg_meta.get("scene")
|
||
meta["prompt_key"] = meta["prompt_key"] or msg_meta.get("prompt_key")
|
||
meta["prompt_release_id"] = meta["prompt_release_id"] or self._int_or_none(msg_meta.get("prompt_release_id"))
|
||
meta["runtime_profile_id"] = meta["runtime_profile_id"] or msg_meta.get("runtime_profile_id")
|
||
meta["model_provider"] = meta["model_provider"] or msg_meta.get("provider")
|
||
meta["model_name"] = meta["model_name"] or msg_meta.get("model")
|
||
|
||
routing_meta = change_set.routing_meta if change_set is not None and isinstance(change_set.routing_meta, dict) else {}
|
||
if change_set is not None:
|
||
meta["scene_key"] = meta["scene_key"] or PMFlowChangeReviewService.AI_EXTRACTION_SCENE_KEY
|
||
meta["prompt_key"] = meta["prompt_key"] or PMFlowChangeReviewService.AI_EXTRACTION_PROMPT_KEY
|
||
meta["prompt_release_id"] = meta["prompt_release_id"] or self._int_or_none(routing_meta.get("prompt_release_id"))
|
||
meta["runtime_profile_id"] = (
|
||
meta["runtime_profile_id"]
|
||
or routing_meta.get("function_runtime_profile_version")
|
||
or routing_meta.get("runtime_profile_id")
|
||
or routing_meta.get("profile_id")
|
||
)
|
||
meta["model_provider"] = meta["model_provider"] or routing_meta.get("provider")
|
||
meta["model_name"] = meta["model_name"] or routing_meta.get("model")
|
||
meta["ai_call_log_id"] = meta["ai_call_log_id"] or self._int_or_none(getattr(change_set, "llm_call_id", None))
|
||
|
||
if meta["ai_call_log_id"] is None and change_set is not None and change_set.llm_call_id:
|
||
trace_result = await self.db.execute(
|
||
select(AICallLog)
|
||
.where(
|
||
and_(
|
||
AICallLog.tenant_id == self.tenant_id,
|
||
AICallLog.trace_id == str(change_set.llm_call_id),
|
||
)
|
||
)
|
||
.order_by(AICallLog.created_at.desc(), AICallLog.id.desc())
|
||
)
|
||
ai_call = trace_result.scalars().first()
|
||
if ai_call is not None:
|
||
meta["ai_call_log_id"] = int(ai_call.id)
|
||
meta["prompt_key"] = meta["prompt_key"] or ai_call.prompt_key
|
||
meta["prompt_release_id"] = meta["prompt_release_id"] or ai_call.prompt_release_id
|
||
meta["model_provider"] = meta["model_provider"] or ai_call.provider
|
||
meta["model_name"] = meta["model_name"] or ai_call.model
|
||
routing = ai_call.routing_meta if isinstance(ai_call.routing_meta, dict) else {}
|
||
meta["runtime_profile_id"] = (
|
||
meta["runtime_profile_id"]
|
||
or routing.get("function_runtime_profile_version")
|
||
or routing.get("runtime_profile_id")
|
||
)
|
||
return meta
|
||
|
||
async def _ensure_prompt_optimization_candidate(self, row: PMFlowAIFeedbackEvent) -> None:
|
||
metadata = dict(row.feedback_metadata or {})
|
||
prompt_optimization = dict(metadata.get("promptOptimization") or {})
|
||
candidate_id = self._int_or_none(prompt_optimization.get("candidateId"))
|
||
if row.rating != "negative":
|
||
if candidate_id is not None:
|
||
candidate = (
|
||
await self.db.execute(
|
||
select(OptimizerCandidate).where(
|
||
and_(
|
||
OptimizerCandidate.tenant_id == self.tenant_id,
|
||
OptimizerCandidate.id == int(candidate_id),
|
||
)
|
||
)
|
||
)
|
||
).scalar_one_or_none()
|
||
if candidate is not None and candidate.status == "draft":
|
||
candidate.status = "superseded"
|
||
candidate.notes = "用户将 PMFlow OS AI 回答反馈改为赞同，自动挂起此前负反馈生成的优化候选。"
|
||
prompt_optimization["status"] = "superseded_by_positive_feedback"
|
||
metadata["promptOptimization"] = prompt_optimization
|
||
row.feedback_metadata = metadata
|
||
flag_modified(row, "feedback_metadata")
|
||
return
|
||
if not row.prompt_key:
|
||
return
|
||
judge_run_id = self._int_or_none(prompt_optimization.get("judgeRunId"))
|
||
findings = self._feedback_findings(row)
|
||
suggestion = self._feedback_prompt_suggestion(row, findings=findings)
|
||
evidence = {
|
||
"feedback_event_id": int(row.id),
|
||
"target_type": row.target_type,
|
||
"target_id": row.target_id,
|
||
"answer_kind": row.answer_kind,
|
||
"rating": row.rating,
|
||
"reason_tags": list(row.reason_tags or []),
|
||
"comment": row.comment,
|
||
"scene_key": row.scene_key,
|
||
"prompt_release_id": row.prompt_release_id,
|
||
"runtime_profile_id": row.runtime_profile_id,
|
||
"ai_call_log_id": row.ai_call_log_id,
|
||
"findings": findings,
|
||
}
|
||
|
||
judge: Optional[JudgeRun] = None
|
||
if judge_run_id is not None:
|
||
judge = (
|
||
await self.db.execute(
|
||
select(JudgeRun).where(
|
||
and_(
|
||
JudgeRun.tenant_id == self.tenant_id,
|
||
JudgeRun.id == int(judge_run_id),
|
||
)
|
||
)
|
||
)
|
||
).scalar_one_or_none()
|
||
if judge is None:
|
||
judge = JudgeRun(
|
||
tenant_id=self.tenant_id,
|
||
project_id=int(row.project_id),
|
||
prompt_key=row.prompt_key,
|
||
workflow_type=row.scene_key or row.answer_kind,
|
||
asked_via="pmflow_os",
|
||
judge_status="feedback_review",
|
||
bucket_key=f"pmflow_ai_feedback:{row.prompt_key}:{row.answer_kind}",
|
||
rule_score=0.2,
|
||
overall_score=0.2,
|
||
findings=findings,
|
||
metadata_json=evidence,
|
||
)
|
||
self.db.add(judge)
|
||
await self.db.flush()
|
||
else:
|
||
judge.findings = findings
|
||
judge.metadata_json = evidence
|
||
flag_modified(judge, "findings")
|
||
flag_modified(judge, "metadata_json")
|
||
|
||
candidate: Optional[OptimizerCandidate] = None
|
||
if candidate_id is not None:
|
||
candidate = (
|
||
await self.db.execute(
|
||
select(OptimizerCandidate).where(
|
||
and_(
|
||
OptimizerCandidate.tenant_id == self.tenant_id,
|
||
OptimizerCandidate.id == int(candidate_id),
|
||
)
|
||
)
|
||
)
|
||
).scalar_one_or_none()
|
||
if candidate is None:
|
||
candidate = OptimizerCandidate(
|
||
tenant_id=self.tenant_id,
|
||
project_id=int(row.project_id),
|
||
judge_run_id=int(judge.id),
|
||
candidate_type="prompt_change",
|
||
target_type="prompt_key",
|
||
target_ref=row.prompt_key,
|
||
status="draft",
|
||
suggestion=suggestion,
|
||
evidence=evidence,
|
||
notes="PMFlow OS AI feedback 自动生成的 prompt 优化候选，需平台 AI Ops 人工复核后再创建 release。",
|
||
)
|
||
self.db.add(candidate)
|
||
await self.db.flush()
|
||
else:
|
||
candidate.suggestion = suggestion
|
||
candidate.evidence = evidence
|
||
flag_modified(candidate, "suggestion")
|
||
flag_modified(candidate, "evidence")
|
||
|
||
metadata["promptOptimization"] = {
|
||
"status": "draft_optimizer_candidate",
|
||
"judgeRunId": int(judge.id),
|
||
"candidateId": int(candidate.id),
|
||
"targetPromptKey": row.prompt_key,
|
||
}
|
||
row.feedback_metadata = metadata
|
||
flag_modified(row, "feedback_metadata")
|
||
|
||
@staticmethod
|
||
def _feedback_findings(row: PMFlowAIFeedbackEvent) -> list[dict[str, Any]]:
|
||
tags = [str(tag).strip() for tag in (row.reason_tags or []) if str(tag).strip()]
|
||
if not tags and row.comment:
|
||
tags = ["用户补充说明"]
|
||
return [
|
||
{
|
||
"code": PMFlowAIFeedbackService._reason_code(tag),
|
||
"label": tag,
|
||
"severity": "high" if tag in {"不准确", "不该生成这个结果"} else "medium",
|
||
"message": f"用户对 {row.answer_kind} 反馈为不赞同：{tag}",
|
||
}
|
||
for tag in tags
|
||
] or [
|
||
{
|
||
"code": "negative_without_reason",
|
||
"label": "未填写原因",
|
||
"severity": "medium",
|
||
"message": f"用户对 {row.answer_kind} 反馈为不赞同，但未选择原因标签。",
|
||
}
|
||
]
|
||
|
||
@staticmethod
|
||
def _feedback_prompt_suggestion(row: PMFlowAIFeedbackEvent, *, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||
labels = [str(item.get("label") or "").strip() for item in findings if isinstance(item, dict)]
|
||
actions: list[str] = []
|
||
if "不准确" in labels:
|
||
actions.append("强化事实约束、证据引用和不确定性表达，避免把候选对象说成已确认结论。")
|
||
if "漏掉重点" in labels:
|
||
actions.append("在抽取 prompt 中补充关键对象覆盖检查，优先覆盖用户最新输入里的需求、风险、决策和任务。")
|
||
if "不够可执行" in labels:
|
||
actions.append("要求输出包含可审查对象、可执行下一步、边界条件和验收信号。")
|
||
if "不符合项目上下文" in labels:
|
||
actions.append("提高项目目标、当前阶段、已有 ChangeSet 与 Evidence 的上下文权重。")
|
||
if "表达不清" in labels:
|
||
actions.append("约束回答结构，先给结论，再给依据和待确认动作。")
|
||
if "不该生成这个结果" in labels:
|
||
actions.append("增加生成前 gate：证据不足时显式待补充，不创建候选对象。")
|
||
if not actions:
|
||
actions.append("复核负反馈样本，补充场景化约束和回归 golden case。")
|
||
return {
|
||
"summary": f"基于 PMFlow OS 用户负反馈优化 {row.prompt_key}",
|
||
"reason_tags": labels,
|
||
"recommended_actions": actions,
|
||
"target_answer_kind": row.answer_kind,
|
||
"scene_key": row.scene_key,
|
||
}
|
||
|
||
@staticmethod
|
||
def _reason_code(label: str) -> str:
|
||
mapping = {
|
||
"不准确": "inaccurate",
|
||
"漏掉重点": "missed_key_point",
|
||
"不够可执行": "not_actionable",
|
||
"不符合项目上下文": "context_mismatch",
|
||
"表达不清": "unclear_expression",
|
||
"不该生成这个结果": "should_not_generate",
|
||
"用户补充说明": "user_comment",
|
||
}
|
||
return mapping.get(label, "negative_feedback")
|
||
|
||
@staticmethod
|
||
def _int_or_none(value: Any) -> Optional[int]:
|
||
if value is None:
|
||
return None
|
||
try:
|
||
text = str(value).strip()
|
||
if not text:
|
||
return None
|
||
return int(text)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
@staticmethod
|
||
def serialize(row: PMFlowAIFeedbackEvent) -> PMFlowAIFeedbackResponse:
|
||
return PMFlowAIFeedbackResponse(
|
||
id=str(row.id),
|
||
projectId=str(row.project_id),
|
||
threadId=str(row.thread_id) if row.thread_id is not None else None,
|
||
messageId=str(row.message_id) if row.message_id is not None else None,
|
||
changeSetId=str(row.change_set_id) if row.change_set_id is not None else None,
|
||
aiCallLogId=str(row.ai_call_log_id) if row.ai_call_log_id is not None else None,
|
||
targetType=row.target_type,
|
||
targetId=row.target_id,
|
||
answerKind=row.answer_kind,
|
||
rating=row.rating,
|
||
reasonTags=list(row.reason_tags or []),
|
||
comment=row.comment,
|
||
sceneKey=row.scene_key,
|
||
promptKey=row.prompt_key,
|
||
promptReleaseId=row.prompt_release_id,
|
||
runtimeProfileId=row.runtime_profile_id,
|
||
modelProvider=row.model_provider,
|
||
modelName=row.model_name,
|
||
metadata=dict(row.feedback_metadata or {}),
|
||
createdAt=row.created_at.isoformat() if row.created_at else "",
|
||
updatedAt=row.updated_at.isoformat() if row.updated_at else "",
|
||
)
|