"""全自适应 RAG 服务。 能力： 1. 意图识别（question / statement / mixed） 2. 多域路由（结构化 DB + 向量检索） 3. 回答生成与来源引用 4. QA 记录、反馈、JSONL 导出 5. 知识库文档/URL/飞书消息入库向量化 """ from __future__ import annotations import ast import asyncio import hashlib import json import math import os from pathlib import Path import re import time from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Optional import httpx from sqlalchemy import and_, bindparam, delete, func, or_, select, text from sqlalchemy.ext.asyncio import AsyncSession from ..config.ai_prompts import PROMPT_RAG_ANSWER, PROMPT_RAG_DB_PROJECT_QA, PROMPT_RAG_INTENT_CLASSIFICATION from ..database import AsyncSessionLocal from ..models import ( KnowledgeBaseChunk, KnowledgeBaseDoc, KGEvidence, KGEntity, KGRelation, Project, ProjectSignalLink, QAFeedback, QARecord, QASession, QASessionState, Signal, Task, TruthNode, User, UserProject, UserWorkload, ) from .context_cache import context_cache_service from .llm import LLMService from .signal_source_orchestration_service import SignalSourceOrchestrationService from .symbol_index_service import SymbolIndexService from ..utils.timezone import CN_TZ, now_cn_aware, now_cn_naive def _env_int(name: str, default: int, *, minimum: int, maximum: int) -> int: raw = str(os.getenv(name, "") or "").strip() if not raw: value = int(default) else: try: value = int(raw) except Exception: value = int(default) return max(minimum, min(maximum, value)) QUESTION_HINTS = ( "吗", "么", "呢", "呀", "啊", "？", "?", "如何", "怎么", "为什么", "为何", "谁", "哪里", "有没有", "有无", "有吗", "是否", "是否有", "是否存在", "哪些", "哪个", "什么", "是什么", "多少", "几", ) PROJECT_EXISTENCE_RE = re.compile(r"(有没有|是否有|有).*?(项目|系统)|项目.*?(有没有|是否有|有).*(吗|\?)") PROJECT_PROGRESS_RE = re.compile(r"(进度|阶段|完成度|完成率|推进|里程碑|状态)") QUESTION_NOISE_RE = re.compile( r"(目前|当前|现在|请问|一下|系统中|项目中|系统内|项目内|是否|有没有|有无|有|吗|呢|呀|啊|什么|哪个|多少|查询|看看)" ) STATEMENT_HINTS = ( "需要", "请", "新增", "优化", "修复", "安排", "立项", "开发", "上线", ) PROJECT_KEYWORDS = ( "项目", "信号", "任务", "宪章", "进度", "状态", "工作量", "负责人", ) PEOPLE_KEYWORDS = ( "成员", "人员", "同事", "负责人", "owner", "assignee", "谁负责", "谁在做", "名单", ) DOC_KEYWORDS = ( "文档", "规范", "spec", "说明", "流程", "模板", "手册", "how to", "怎么做", ) FEATURE_QUERY_KEYWORDS = ( "功能", "用途", "是干嘛", "做什么", "有什么用", "页面", "菜单", "入口", "在哪", "ai配置", "智能配置", "配置中心", "能力管理", "用户管理", "项目列表", "团队负载", "ai管家", ) FEATURE_PURPOSE_HINTS = ( "用途", "作用", "干嘛", "做什么", "是什么", "入口", "在哪", "页面", ) CORRECTION_CUE_KEYWORDS = ( "我说的是", "我说的不是", "不是这个", "不是系统架构", "不是啊", "不对", "错了", "答错", "跑偏", "偏了", "不是这个意思", "你理解错", "你答偏了", "答非所问", "请纠正", "纠正一下", "请更正", "上一条", "上一个回答", "不要模板", "模板化", "我要的是", "我问的是", "不是在问", ) ENGINE_PROCESS_KEYWORDS = ( "ai引擎", "引擎驱动", "驱动方式", "调用链", "调用流程", "调用路径", "模型路由", "模型调用", "路由策略", "openclaw", "网关", "ai架构", "ai 管家", "ai管家", "第一层", "第二层", "第三层", "fallback", "兜底", ) DEEP_EXPLICIT_KEYWORDS = ( "深入", "深度", "详细分析", "证据链", "调用链", "逐层", "逐步", "详细展开", "deep", ) BRIEF_NEGATIVE_FEEDBACK_RE = re.compile( r"^(不是啊?|不对|错了|答错了?|跑偏了?|偏了|不是这个(意思)?|不(是|对)啊)$" ) HISTORY_KEYWORDS = ("之前", "上次", "历史", "曾经", "之前说", "last time") PMFLOW_KEYWORDS = ("复盘", "charter", "宪章", "task", "任务描述") FOLLOWUP_REFERENTIAL_KEYWORDS = ( "这些", "它们", "这个", "那个", "上面", "前面", "刚才", "分别", "具体", "都有哪些", "分别是什么", ) FOLLOWUP_ENUMERATION_KEYWORDS = ( "分别是什么", "分别是啥", "都有哪些", "具体有哪些", "列出来", "清单", "名单", ) IDENTITY_KEYWORDS = ( "你是谁", "你是", "自我介绍", "介绍你自己", "你能做什么", "你会什么", "你负责什么", "你叫什么", ) ARCHITECTURE_KEYWORDS = ( "系统架构", "架构", "运行机制", "技术栈", "模块", "服务组成", "docker", "数据库", "redis", "后端", "前端", ) CODE_KEYWORDS = ( "代码", "函数", "方法", "类", "模块", "文件", "api", "接口实现", "service", "model", "schema", "where defined", ) URL_RE = re.compile(r"https?://\S+", flags=re.IGNORECASE) SENTENCE_SPLIT_RE = re.compile(r"(?<=[。！？!?\n])") PAGE_MARKER_RE = re.compile(r"\[\[PAGE:(\d{1,6})\]\]") MAX_CONTEXT_TOKENS = 8000 DEFAULT_TOP_K = 5 DB_QUERY_TIMEOUT_SECONDS = 5.0 VECTOR_SEARCH_TIMEOUT_SECONDS = 10.0 GRAPH_QUERY_TIMEOUT_SECONDS = 4.0 AGENT_MAX_RETRIEVAL_STEPS = 3 SESSION_GAP_MINUTES = 30 SESSION_DEFAULT_TITLE = "新会话" SESSION_TITLE_MIN_LEN = 5 THINKING_MODE_NORMAL = "normal" THINKING_MODE_DEEP = "deep" DEEP_RETRIEVAL_STEPS = _env_int("RAG_DEEP_RETRIEVAL_STEPS", 5, minimum=2, maximum=8) DEEP_VECTOR_TOP_K = _env_int("RAG_DEEP_VECTOR_TOP_K", 12, minimum=6, maximum=40) DEEP_CONTEXT_BLOCKS = _env_int("RAG_DEEP_CONTEXT_BLOCKS", 16, minimum=8, maximum=32) DEEP_ANSWER_MAX_TOKENS = _env_int("RAG_DEEP_ANSWER_MAX_TOKENS", 1400, minimum=900, maximum=3000) QUERY_TYPE_COUNT = "count" QUERY_TYPE_LIST = "list" QUERY_TYPE_STATUS = "status" QUERY_TYPE_EXPLAIN = "explain" QUERY_TYPE_UNKNOWN = "unknown" ACTION_MODE_AUTO = "auto" ACTION_MODE_READ = "read" ACTION_MODE_WRITE = "write" ACTION_MODE_READ_WRITE = "read_write" REQUIREMENT_WRITE_VERB_KEYWORDS = ( "新增", "增加", "添加", "接入", "支持", "实现", "创建", "新建", "记录", "入库", "同步", "补充", ) REQUIREMENT_WRITE_OBJECT_KEYWORDS = ( "信号", "需求", "任务", "项目", "功能", "接口", "页面", "按钮", "规则", "模块", "卡路里", "食谱", ) WRITE_EXPRESSIONS = ( "给", "帮我", "请帮", "请给", "麻烦", ) COUNT_HINTS = ("多少", "几个", "数量", "总数", "count") LIST_HINTS = ("哪些", "列表", "清单", "列出", "分别", "名单", "都有哪些") STATUS_HINTS = ("进度", "状态", "阶段", "完成率", "完成度", "情况") ENTITY_ALIAS_MAP: dict[str, str] = { "能力": "skill", "技能": "skill", "ability": "skill", "abilities": "skill", "skill": "skill", "skills": "skill", "负责人": "owner", "owner": "owner", "assignee": "owner", "成员": "member", "人员": "member", "member": "member", "任务": "task", "task": "task", "tasks": "task", "项目": "project", "project": "project", "projects": "project", "信号": "signal", "signal": "signal", "signals": "signal", "函数": "function", "function": "function", "method": "function", "类": "class", "class": "class", "模块": "module", "module": "module", "智能配置": "ai_config", "ai配置": "ai_config", "配置中心": "ai_config", "能力管理": "skills", "技能管理": "skills", "用户管理": "users", "项目列表": "projects", "团队负载": "workloads", "ai管家": "ai_steward", } TS_SYMBOL_RE = re.compile( r"(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|const)\s+([A-Za-z_][A-Za-z0-9_]*)" ) @dataclass(slots=True) class RetrievedItem: source: str content: str score: float source_label: str metadata: dict[str, Any] class AdaptiveRAGService: """项目内统一 RAG 服务。""" _symbol_snapshot_cache: dict[str, tuple[float, dict[str, Any]]] = {} def __init__(self, db: AsyncSession, *, tenant_id: int) -> None: self.db = db self.tenant_id = int(tenant_id) self.llm = LLMService() self.context_cache = context_cache_service @staticmethod def _normalize_thinking_mode(value: Optional[str]) -> str: text = str(value or "").strip().lower() if text == THINKING_MODE_DEEP: return THINKING_MODE_DEEP return THINKING_MODE_NORMAL @staticmethod def _normalize_conversation_key(value: Optional[str]) -> str: raw = str(value or "").strip() if not raw: return "" cleaned = re.sub(r"\s+", "", raw.lower()) cleaned = re.sub(r"[^a-z0-9:_\-\.]", "", cleaned) return cleaned[:160] def _build_external_conversation_key( self, *, asked_via: str, source_chat_id: Optional[str], conversation_key: Optional[str], ) -> str: explicit = self._normalize_conversation_key(conversation_key) if explicit: return explicit channel = str(asked_via or "").strip().lower() if channel == "feishu": chat = str(source_chat_id or "").strip() if chat: return self._normalize_conversation_key(f"feishu:{chat}") return "" @staticmethod def _external_session_id_from_key(conversation_key: str) -> Optional[int]: key = str(conversation_key or "").strip() if not key: return None digest = hashlib.sha1(key.encode("utf-8")).hexdigest() bucket = int(digest[:10], 16) % 500_000_000 return 1_500_000_000 + bucket # ------------------------------------------------------------------ # Intent # ------------------------------------------------------------------ async def classify_intent(self, message: str) -> dict[str, Any]: text = str(message or "").strip() if not text: return { "intent": "statement", "confidence": 1.0, "question_part": "", "statement_part": "", } heuristic = self._classify_intent_rule(text) if heuristic["confidence"] >= 0.86: return heuristic prompt = PROMPT_RAG_INTENT_CLASSIFICATION.format(message=text[:1200]).strip() try: raw = await self.llm.generate( prompt, db=self.db, tenant_id=self.tenant_id, function_name="rag_intent_classification", temperature=0.0, max_tokens=220, use_prompt_profile=False, ) parsed = self._parse_json_object(raw) intent = str(parsed.get("intent") or heuristic["intent"]).strip().lower() if intent not in {"question", "statement", "mixed"}: intent = heuristic["intent"] confidence = self._to_float(parsed.get("confidence"), heuristic["confidence"]) question_part = str(parsed.get("question_part") or "").strip() statement_part = str(parsed.get("statement_part") or "").strip() if not question_part and intent in {"question", "mixed"}: question_part = heuristic["question_part"] if not statement_part and intent in {"statement", "mixed"}: statement_part = heuristic["statement_part"] return { "intent": intent, "confidence": max(0.0, min(1.0, confidence)), "question_part": question_part, "statement_part": statement_part, } except Exception: return heuristic def _classify_intent_rule(self, text: str) -> dict[str, Any]: question_score = 0.0 statement_score = 0.0 lowered = text.lower() if "?" in text or "？" in text: question_score += 0.35 if lowered.endswith(("吗", "么", "呢", "呀", "啊")): question_score += 0.24 for hint in QUESTION_HINTS: if hint in lowered: question_score += 0.08 if PROJECT_EXISTENCE_RE.search(text): question_score += 0.26 if re.search(r"(现在|当前|目前|请问).*(项目|任务|负责人|状态|进度|列表|清单)", text): question_score += 0.18 for hint in STATEMENT_HINTS: if hint in lowered: statement_score += 0.10 sentences = [s.strip() for s in SENTENCE_SPLIT_RE.split(text) if s.strip()] question_sentences = [s for s in sentences if "?" in s or "？" in s or any(k in s.lower() for k in QUESTION_HINTS)] statement_sentences = [s for s in sentences if s not in question_sentences] if question_sentences and statement_sentences: return { "intent": "mixed", "confidence": 0.88, "question_part": " ".join(question_sentences)[:1000], "statement_part": " ".join(statement_sentences)[:1000], } if question_score >= 0.28 and question_score >= statement_score: return { "intent": "question", "confidence": min(0.9, 0.62 + question_score), "question_part": text[:1000], "statement_part": "", } return { "intent": "statement", "confidence": min(0.9, 0.66 + statement_score), "question_part": "", "statement_part": text[:1000], } # ------------------------------------------------------------------ # Public ask flow # ------------------------------------------------------------------ async def answer_question( self, *, question: str, asked_by: Optional[int], asked_via: str, project_id: Optional[int] = None, source_chat_id: Optional[str] = None, session_id: Optional[int] = None, conversation_key: Optional[str] = None, force_new_session: bool = False, thinking_mode: str = THINKING_MODE_NORMAL, action_mode: str = ACTION_MODE_AUTO, sync_requirement: bool = True, ) -> dict[str, Any]: started = time.perf_counter() clean_q = str(question or "").strip() external_conversation_key = self._build_external_conversation_key( asked_via=asked_via, source_chat_id=source_chat_id, conversation_key=conversation_key, ) active_thinking_mode = self._normalize_thinking_mode(thinking_mode) deep_thinking = active_thinking_mode == THINKING_MODE_DEEP asked_via_lower = str(asked_via or "").strip().lower() deep_retrieval_steps = DEEP_RETRIEVAL_STEPS deep_vector_top_k = DEEP_VECTOR_TOP_K if asked_via_lower == "feishu": # 飞书链路优先低时延，保留 deep 能力但收紧预算。 deep_retrieval_steps = min(DEEP_RETRIEVAL_STEPS, 3) deep_vector_top_k = min(DEEP_VECTOR_TOP_K, 8) if not clean_q: return self._build_response_payload( answer="问题为空，无法检索。", sources=[], domains_used=[], response_time_ms=0, db_query_ms=0, vector_search_ms=0, db_timeout=False, vector_timeout=False, qa_id=None, session_id=None, turn_index=1, relevance_max=0.0, query_type=QUERY_TYPE_UNKNOWN, confidence=0.0, missing_fields=["question"], thinking_mode=active_thinking_mode, ) session_id, turn_index = await self._touch_session_for_new_record( asked_by=asked_by, asked_via=asked_via, project_id=project_id, question=clean_q, preferred_session_id=session_id, force_new_session=bool(force_new_session), ) context_session_id: Optional[int] if session_id is not None: context_session_id = int(session_id) elif asked_by is None: context_session_id = self._external_session_id_from_key(external_conversation_key) else: context_session_id = None conversation = await self._load_recent_conversation( asked_by=asked_by, asked_via=asked_via, project_id=project_id, session_id=session_id, external_session_id=context_session_id, limit=10, ) state_row = await self._ensure_session_state(session_id=session_id) effective_q, rewrite_meta = self._rewrite_question_with_state( question=clean_q, conversation=conversation, session_state=state_row, ) query_type = self._classify_query_type(effective_q) requested_action_mode = self._normalize_action_mode(action_mode) effective_action_mode = self._resolve_action_mode( requested_mode=requested_action_mode, question=clean_q, effective_question=effective_q, ) if ( bool(sync_requirement) and asked_via_lower != "feishu" and effective_action_mode in {ACTION_MODE_WRITE, ACTION_MODE_READ_WRITE} ): ack_result = await self._handle_ack_first_requirement( question=clean_q, effective_question=effective_q, asked_by=asked_by, asked_via=asked_via, project_id=project_id, session_id=session_id, turn_index=turn_index, state_row=state_row, rewrite_meta=rewrite_meta, context_session_id=context_session_id, query_type=query_type, action_mode=effective_action_mode, started_at=started, ) if ack_result is not None: return ack_result forced_deep_reason = "" if not deep_thinking: forced_deep_reason = self._forced_deep_reason( question=clean_q, effective_question=effective_q, conversation=conversation, ) if forced_deep_reason and self._should_force_deep_for_channel( asked_via=asked_via, question=clean_q, effective_question=effective_q, reason=forced_deep_reason, ): deep_thinking = True active_thinking_mode = THINKING_MODE_DEEP else: forced_deep_reason = "" clarification_answer = self._build_clarification_question( original_question=clean_q, effective_question=effective_q, session_state=state_row, ) if clarification_answer: elapsed = int((time.perf_counter() - started) * 1000) clarify_sources = [ { "source": "system", "label": "Session State", "score": 0.95, "metadata": {"mode": "clarification", "reason": "missing_subject"}, } ] qa = QARecord( tenant_id=self.tenant_id, question=clean_q, answer=clarification_answer, sources=clarify_sources, intent="question", domains_used=["system_context"], response_time_ms=elapsed, asked_by=asked_by, asked_via=(asked_via or "pmflow")[:20], project_id=project_id, session_id=session_id, turn_index=turn_index, ) self.db.add(qa) await self._update_session_state_after_turn( session_state=state_row, question=clean_q, effective_question=effective_q, answer=clarification_answer, query_type=query_type, ambiguity="needs_clarification", rewrite_meta=rewrite_meta, sources=clarify_sources, ) await self.db.commit() await self.db.refresh(qa) await self.context_cache.append_session_messages( tenant_id=self.tenant_id, session_id=context_session_id, question=clean_q, answer=clarification_answer, ) return self._build_response_payload( answer=clarification_answer, sources=clarify_sources, domains_used=["system_context"], response_time_ms=elapsed, db_query_ms=0, vector_search_ms=0, db_timeout=False, vector_timeout=False, qa_id=int(qa.id), session_id=int(session_id) if session_id is not None else None, turn_index=int(turn_index), relevance_max=0.95, query_type=query_type, confidence=0.92, missing_fields=["subject"], thinking_mode=active_thinking_mode, ) system_snapshot = await self.context_cache.get_system_snapshot( db=self.db, tenant_id=self.tenant_id, ) user_snapshot = await self.context_cache.get_user_snapshot( db=self.db, tenant_id=self.tenant_id, user_id=asked_by, ) if not deep_thinking: deterministic_result = await self._run_deterministic_query( question=effective_q, project_id=project_id, query_type=query_type, ) if deterministic_result is not None: return await self._persist_fast_answer( question=clean_q, asked_by=asked_by, asked_via=asked_via, project_id=project_id, fast_result=deterministic_result, session_id=session_id, external_session_id=context_session_id, turn_index=turn_index, query_type=str(deterministic_result.get("query_type") or query_type), missing_fields=list(deterministic_result.get("missing_fields") or []), state_row=state_row, effective_question=effective_q, rewrite_meta=rewrite_meta, thinking_mode=active_thinking_mode, ) direct_answer = None if not deep_thinking: direct_answer = self._build_snapshot_direct_answer( effective_q, system_snapshot=system_snapshot, user_snapshot=user_snapshot, ) if direct_answer: elapsed = int((time.perf_counter() - started) * 1000) direct_sources = [ { "source": "system", "label": "System Snapshot", "score": 0.99, "metadata": {"mode": "snapshot_direct", "topic": "identity_or_architecture"}, } ] if user_snapshot: direct_sources.append( { "source": "system", "label": "User Snapshot", "score": 0.98, "metadata": {"mode": "snapshot_direct", "topic": "current_user"}, } ) qa = QARecord( tenant_id=self.tenant_id, question=clean_q, answer=direct_answer, sources=direct_sources, intent="question", domains_used=["system_context"], response_time_ms=elapsed, asked_by=asked_by, asked_via=(asked_via or "pmflow")[:20], project_id=project_id, session_id=session_id, turn_index=turn_index, ) self.db.add(qa) await self._update_session_state_after_turn( session_state=state_row, question=clean_q, effective_question=effective_q, answer=direct_answer, query_type=QUERY_TYPE_EXPLAIN, ambiguity="clear", rewrite_meta=rewrite_meta, sources=direct_sources, ) await self.db.commit() await self.db.refresh(qa) await self.context_cache.append_session_messages( tenant_id=self.tenant_id, session_id=context_session_id, question=clean_q, answer=direct_answer, ) return self._build_response_payload( answer=direct_answer, sources=direct_sources, domains_used=["system_context"], response_time_ms=elapsed, db_query_ms=0, vector_search_ms=0, db_timeout=False, vector_timeout=False, qa_id=int(qa.id), session_id=int(session_id) if session_id is not None else None, turn_index=int(turn_index), relevance_max=1.0, query_type=QUERY_TYPE_EXPLAIN, confidence=0.99, missing_fields=[], thinking_mode=active_thinking_mode, ) is_snapshot_direct_question = self._is_snapshot_direct_question(effective_q) if not deep_thinking and not is_snapshot_direct_question: glm_semantic_result = await self._try_glm_project_semantic_answer(clean_q=effective_q, project_id=project_id) if glm_semantic_result is not None: return await self._persist_fast_answer( question=clean_q, asked_by=asked_by, asked_via=asked_via, project_id=project_id, fast_result=glm_semantic_result, session_id=session_id, external_session_id=context_session_id, turn_index=turn_index, query_type=query_type if query_type != QUERY_TYPE_UNKNOWN else QUERY_TYPE_EXPLAIN, missing_fields=[], state_row=state_row, effective_question=effective_q, rewrite_meta=rewrite_meta, thinking_mode=active_thinking_mode, ) domains = self._route_domains(effective_q) if str(asked_via or "").strip().lower() == "feishu" and source_chat_id and "feishu" not in domains: domains.append("feishu") retrieved, search_meta = await self._multi_step_search( question=effective_q, domains=domains, project_id=project_id, source_chat_id=source_chat_id, session_state=state_row, max_steps_override=deep_retrieval_steps if deep_thinking else None, vector_top_k=deep_vector_top_k if deep_thinking else None, ) db_query_ms = int(search_meta.get("db_query_ms") or 0) vector_search_ms = int(search_meta.get("vector_search_ms") or 0) graph_query_ms = int(search_meta.get("graph_query_ms") or 0) db_timeout = bool(search_meta.get("db_timeout")) vector_timeout = bool(search_meta.get("vector_timeout")) graph_timeout = bool(search_meta.get("graph_timeout")) tool_trace = list(search_meta.get("tool_trace") or []) tool_trace.insert( 0, { "step": 0, "tool": "thinking_mode", "mode": active_thinking_mode, "fast_path_bypassed": bool(deep_thinking), "reason": forced_deep_reason or ("manual_deep" if self._normalize_thinking_mode(thinking_mode) == THINKING_MODE_DEEP else "normal"), }, ) final_domains = [str(x) for x in (search_meta.get("domains_used") or domains)] max_score = retrieved[0].score if retrieved else 0.0 answer_text = await self._generate_answer( effective_q, retrieved, conversation, project_id, system_snapshot=system_snapshot, user_snapshot=user_snapshot, relevance_max=max_score, thinking_mode=active_thinking_mode, context_block_limit=DEEP_CONTEXT_BLOCKS if deep_thinking else None, answer_max_tokens=DEEP_ANSWER_MAX_TOKENS if deep_thinking else None, ) used_sources = self._dedupe_sources(retrieved) if not used_sources: used_sources = [ { "source": "system", "label": "System Snapshot", "score": round(max(max_score, 0.6), 4), "metadata": {"mode": "snapshot_only", "relevance_max": round(max_score, 4)}, } ] if user_snapshot: used_sources.append( { "source": "system", "label": "User Snapshot", "score": round(max(max_score, 0.58), 4), "metadata": {"mode": "snapshot_only", "relevance_max": round(max_score, 4)}, } ) auto_retry_reason = "" if not deep_thinking: auto_retry_reason = self._should_auto_deep_retry( question=clean_q, effective_question=effective_q, sources=used_sources, relevance_max=max_score, ) if auto_retry_reason and not self._allow_auto_deep_retry_for_channel( asked_via=asked_via, question=clean_q, effective_question=effective_q, reason=auto_retry_reason, ): auto_retry_reason = "" if auto_retry_reason: deep_retrieved, deep_meta = await self._multi_step_search( question=effective_q, domains=domains, project_id=project_id, source_chat_id=source_chat_id, session_state=state_row, max_steps_override=deep_retrieval_steps, vector_top_k=deep_vector_top_k, ) deep_max_score = deep_retrieved[0].score if deep_retrieved else 0.0 deep_answer_text = await self._generate_answer( effective_q, deep_retrieved, conversation, project_id, system_snapshot=system_snapshot, user_snapshot=user_snapshot, relevance_max=deep_max_score, thinking_mode=THINKING_MODE_DEEP, context_block_limit=DEEP_CONTEXT_BLOCKS, answer_max_tokens=DEEP_ANSWER_MAX_TOKENS, ) deep_sources = self._dedupe_sources(deep_retrieved) if not deep_sources: deep_sources = [ { "source": "system", "label": "System Snapshot", "score": round(max(deep_max_score, 0.6), 4), "metadata": {"mode": "snapshot_only", "relevance_max": round(deep_max_score, 4)}, } ] if user_snapshot: deep_sources.append( { "source": "system", "label": "User Snapshot", "score": round(max(deep_max_score, 0.58), 4), "metadata": {"mode": "snapshot_only", "relevance_max": round(deep_max_score, 4)}, } ) merged_metrics = self._merge_search_metrics( { "db_query_ms": db_query_ms, "vector_search_ms": vector_search_ms, "graph_query_ms": graph_query_ms, "db_timeout": db_timeout, "vector_timeout": vector_timeout, "graph_timeout": graph_timeout, }, deep_meta, ) db_query_ms = int(merged_metrics.get("db_query_ms") or 0) vector_search_ms = int(merged_metrics.get("vector_search_ms") or 0) graph_query_ms = int(merged_metrics.get("graph_query_ms") or 0) db_timeout = bool(merged_metrics.get("db_timeout")) vector_timeout = bool(merged_metrics.get("vector_timeout")) graph_timeout = bool(merged_metrics.get("graph_timeout")) deep_trace = list(deep_meta.get("tool_trace") or []) tool_trace.append( { "step": 98, "tool": "auto_deep_retry", "reason": auto_retry_reason, "prev_relevance_max": round(float(max_score or 0.0), 4), "prev_snapshot_only": self._is_snapshot_only_sources(used_sources), } ) for row in deep_trace: tool_trace.append(row) final_domains = [str(x) for x in (deep_meta.get("domains_used") or final_domains or domains)] retrieved = deep_retrieved max_score = deep_max_score answer_text = deep_answer_text used_sources = deep_sources deep_thinking = True active_thinking_mode = THINKING_MODE_DEEP evidence_guard_triggered = self._should_block_answer_with_guard( question=clean_q, effective_question=effective_q, sources=used_sources, relevance_max=max_score, ) if evidence_guard_triggered: answer_text = self._build_evidence_guard_answer(clean_q) used_sources = [ { "source": "system", "label": "Evidence Guard", "score": 0.99, "metadata": {"mode": "guard", "reason": "snapshot_only_process_query"}, } ] max_score = min(float(max_score or 0.0), 0.35) if "system_context" not in final_domains: final_domains.append("system_context") tool_trace.append( { "step": 99, "tool": "answer_guard", "reason": "insufficient_call_chain_evidence", } ) elapsed = int((time.perf_counter() - started) * 1000) qa = QARecord( tenant_id=self.tenant_id, question=clean_q, answer=answer_text, sources=used_sources, intent="question", domains_used=final_domains, response_time_ms=elapsed, asked_by=asked_by, asked_via=(asked_via or "pmflow")[:20], project_id=project_id, session_id=session_id, turn_index=turn_index, ) self.db.add(qa) confidence = round(max(0.2, min(1.0, max_score if max_score > 0 else 0.55)), 4) if evidence_guard_triggered: confidence = min(confidence, 0.35) missing_fields = [] if max_score < 0.45: missing_fields = ["high_relevance_evidence"] if evidence_guard_triggered: missing_fields = sorted(set([*missing_fields, "call_chain_evidence", "target_scope"])) await self._update_session_state_after_turn( session_state=state_row, question=clean_q, effective_question=effective_q, answer=answer_text, query_type=query_type if query_type != QUERY_TYPE_UNKNOWN else QUERY_TYPE_EXPLAIN, ambiguity="clear" if not missing_fields else "low_evidence", rewrite_meta=rewrite_meta, sources=used_sources, ) await self.db.commit() await self.db.refresh(qa) await self.context_cache.append_session_messages( tenant_id=self.tenant_id, session_id=context_session_id, question=clean_q, answer=answer_text, ) return self._build_response_payload( answer=answer_text, sources=used_sources, domains_used=final_domains, response_time_ms=elapsed, db_query_ms=db_query_ms, vector_search_ms=vector_search_ms, graph_query_ms=graph_query_ms, db_timeout=db_timeout, vector_timeout=vector_timeout, graph_timeout=graph_timeout, tool_trace=tool_trace, qa_id=int(qa.id), session_id=int(session_id) if session_id is not None else None, turn_index=int(turn_index), relevance_max=round(max_score, 4), query_type=query_type if query_type != QUERY_TYPE_UNKNOWN else QUERY_TYPE_EXPLAIN, confidence=confidence, missing_fields=missing_fields, thinking_mode=active_thinking_mode, ) def _build_response_payload( self, *, answer: str, sources: list[dict[str, Any]], domains_used: list[str], response_time_ms: int, db_query_ms: int, vector_search_ms: int, graph_query_ms: int = 0, db_timeout: bool = False, vector_timeout: bool = False, graph_timeout: bool = False, qa_id: Optional[int] = None, session_id: Optional[int] = None, turn_index: int = 1, relevance_max: float = 0.0, query_type: str = QUERY_TYPE_UNKNOWN, confidence: float = 0.0, missing_fields: Optional[list[str]] = None, tool_trace: Optional[list[dict[str, Any]]] = None, thinking_mode: str = THINKING_MODE_NORMAL, action_taken: Optional[str] = None, signal_id: Optional[int] = None, reply_required: bool = False, reply_text: Optional[str] = None, ack_latency_ms: Optional[int] = None, trace_id: Optional[str] = None, processing_stage: Optional[str] = None, ) -> dict[str, Any]: evidence: list[dict[str, Any]] = [] for item in sources[:8]: if not isinstance(item, dict): continue evidence.append( { "source": str(item.get("source") or "unknown"), "label": str(item.get("label") or "unknown"), "score": round(self._to_float(item.get("score"), 0.0), 4), "metadata": item.get("metadata") if isinstance(item.get("metadata"), dict) else {}, } ) return { "answer": str(answer or "").strip(), "evidence": evidence, "confidence": round(max(0.0, min(1.0, self._to_float(confidence, 0.0))), 4), "thinking_mode": self._normalize_thinking_mode(thinking_mode), "missing_fields": [str(x) for x in (missing_fields or []) if str(x).strip()], "query_type": str(query_type or QUERY_TYPE_UNKNOWN), "sources": sources, "domains_used": domains_used, "response_time_ms": int(response_time_ms or 0), "retrieval_metrics": { "db_query_ms": int(db_query_ms or 0), "vector_search_ms": int(vector_search_ms or 0), "graph_query_ms": int(graph_query_ms or 0), "db_timeout": bool(db_timeout), "vector_timeout": bool(vector_timeout), "graph_timeout": bool(graph_timeout), }, "tool_trace": tool_trace if isinstance(tool_trace, list) else [], "qa_id": int(qa_id) if qa_id is not None else None, "session_id": int(session_id) if session_id is not None else None, "turn_index": int(turn_index or 1), "relevance_max": round(max(0.0, self._to_float(relevance_max, 0.0)), 4), "action_taken": str(action_taken).strip() if action_taken else None, "signal_id": int(signal_id) if signal_id is not None else None, "reply_required": bool(reply_required), "reply_text": str(reply_text or "").strip() if reply_text else None, "ack_latency_ms": int(ack_latency_ms) if ack_latency_ms is not None else None, "trace_id": str(trace_id or "").strip() or None, "processing_stage": str(processing_stage or "").strip() or None, } @staticmethod def _extract_ack_metadata_from_sources( sources: Any, *, answer: str = "", ) -> dict[str, Any]: rows = sources if isinstance(sources, list) else [] action_taken = "" signal_id: Optional[int] = None reply_required = False ack_latency_ms: Optional[int] = None trace_id = "" processing_stage = "" for source in rows: if not isinstance(source, dict): continue meta = source.get("metadata") if isinstance(source.get("metadata"), dict) else {} candidate_action = str(meta.get("action_taken") or "").strip() if not candidate_action: continue action_taken = candidate_action try: signal_id = int(meta.get("signal_id")) if meta.get("signal_id") is not None else None except Exception: signal_id = None trace_id = str(meta.get("trace_id") or "").strip() processing_stage = str(meta.get("processing_stage") or meta.get("action_mode") or "").strip() try: ack_latency_ms = ( int(meta.get("ack_latency_ms")) if meta.get("ack_latency_ms") is not None else None ) except Exception: ack_latency_ms = None reply_required = True break reply_text = str(answer or "").strip() if reply_required else "" normalized_stage = processing_stage if normalized_stage == "write": normalized_stage = "ack_only" return { "action_taken": action_taken or None, "signal_id": signal_id, "reply_required": bool(reply_required), "reply_text": reply_text or None, "ack_latency_ms": ack_latency_ms, "trace_id": trace_id or None, "processing_stage": normalized_stage or None, } @staticmethod def _normalize_action_mode(value: Optional[str]) -> str: text = str(value or "").strip().lower() if text in {ACTION_MODE_READ, ACTION_MODE_WRITE, ACTION_MODE_READ_WRITE}: return text return ACTION_MODE_AUTO @classmethod def _looks_like_write_intent(cls, question: str) -> bool: text = str(question or "").strip().lower() if not text: return False has_verb = any(k in text for k in REQUIREMENT_WRITE_VERB_KEYWORDS) has_object = any(k in text for k in REQUIREMENT_WRITE_OBJECT_KEYWORDS) if not has_verb or not has_object: return False if any(prefix in text for prefix in WRITE_EXPRESSIONS): return True if "新增" in text or "增加" in text or "记录" in text or "入库" in text: return True return False @classmethod def _resolve_action_mode( cls, *, requested_mode: str, question: str, effective_question: str, ) -> str: mode = cls._normalize_action_mode(requested_mode) if mode != ACTION_MODE_AUTO: return mode merged = f"{question} {effective_question}".strip() if not cls._looks_like_write_intent(merged): return ACTION_MODE_READ has_question_form = ("?" in merged) or ("？" in merged) or cls._contains_any(merged, QUESTION_HINTS) if has_question_form: return ACTION_MODE_READ_WRITE return ACTION_MODE_WRITE @staticmethod def _normalize_requirement_summary(text: str) -> str: normalized = re.sub(r"\s+", " ", str(text or "").strip()) normalized = normalized.strip("，,。.!！？?：:;；") return normalized[:300] @staticmethod def _build_ack_trace_id(question: str) -> str: base = str(question or "").strip() digest = hashlib.sha1(base.encode("utf-8")).hexdigest()[:10] if base else "empty" return f"rag-ack-{int(time.time() * 1000)}-{digest}" async def _ensure_project_signal_link( self, *, signal_id: int, project_id: int, link_type: str = "related", ) -> None: existing = ( await self.db.execute( select(ProjectSignalLink.id).where( and_( ProjectSignalLink.tenant_id == self.tenant_id, ProjectSignalLink.project_id == int(project_id), ProjectSignalLink.signal_id == int(signal_id), ) ).limit(1) ) ).scalar_one_or_none() if existing is not None: return self.db.add( ProjectSignalLink( tenant_id=self.tenant_id, project_id=int(project_id), signal_id=int(signal_id), link_type=link_type, relevance_score=1.0, ) ) async def _create_or_reuse_requirement_signal( self, *, question: str, effective_question: str, asked_by: Optional[int], project_id: Optional[int], ) -> tuple[str, Optional[int], Optional[int], str]: summary = self._normalize_requirement_summary(effective_question or question) resolved_project_id: Optional[int] = project_id if resolved_project_id is None: scoped_id, _, _ = await self._resolve_project_scope_for_question( question=effective_question or question, project_id=project_id, ) resolved_project_id = scoped_id existing_stmt = select(Signal).where( and_( Signal.tenant_id == self.tenant_id, Signal.source == "PMFlow", Signal.summary == summary, ) ) if resolved_project_id is None: existing_stmt = existing_stmt.where(Signal.matched_project_id.is_(None)) else: existing_stmt = existing_stmt.where(Signal.matched_project_id == int(resolved_project_id)) existing = ( await self.db.execute( existing_stmt.order_by(Signal.created_at.desc(), Signal.id.desc()).limit(1) ) ).scalar_one_or_none() if existing is not None: if resolved_project_id is not None: try: await self._ensure_project_signal_link( signal_id=int(existing.id), project_id=int(resolved_project_id), ) await self.db.commit() except Exception as exc: await self.db.rollback() return ("signal_failed", None, resolved_project_id, str(exc)[:240]) return ("signal_deduped", int(existing.id), resolved_project_id, "") actor = f"PMFlow用户#{int(asked_by)}" if asked_by is not None else "PMFlow用户" content = f"[{actor}]: {summary}" strategic_payload = { "source": "pmflow_qa", "original_question": str(question or "")[:4000], "normalized_question": summary, "write_mode": "ack_first", "created_at": now_cn_aware().isoformat(), } signal = Signal( source="PMFlow", content=content[:2000], impact_score=0.65, priority="medium", processed=False, strategic_analysis=json.dumps(strategic_payload, ensure_ascii=False), tasks_generated=json.dumps([], ensure_ascii=False), matched_project_id=int(resolved_project_id) if resolved_project_id is not None else None, title=summary[:120] or "PMFlow 新需求信号", signal_type="新需求", summary=summary, reasoning=json.dumps(["来自 PMFlow 问答写入意图，按 ACK-first 策略快速入库。"], ensure_ascii=False), confidence=0.74, recommended_action="enter_only", status="待处理", tenant_id=self.tenant_id, ) try: self.db.add(signal) await self.db.flush() if resolved_project_id is not None: await self._ensure_project_signal_link( signal_id=int(signal.id), project_id=int(resolved_project_id), ) await self.db.commit() await self.db.refresh(signal) return ("signal_created", int(signal.id), resolved_project_id, "") except Exception as exc: await self.db.rollback() return ("signal_failed", None, resolved_project_id, str(exc)[:240]) @staticmethod def _build_requirement_ack_text( *, action_taken: str, signal_id: Optional[int], project_id: Optional[int], error_reason: str, ) -> str: project_text = f"项目#{int(project_id)}" if project_id is not None else "未指定项目" if action_taken == "signal_created" and signal_id is not None: return f"已入库：已为{project_text}创建信号 #{int(signal_id)}，并进入待处理队列。" if action_taken == "signal_deduped" and signal_id is not None: return f"已入库：检测到重复需求，已复用{project_text}的信号 #{int(signal_id)}。" if action_taken == "signal_failed": reason = error_reason or "写入异常" return f"未完成入库：{reason}。我已记录本次失败，请稍后重试。" return "已收到需求，正在处理中。" async def _handle_ack_first_requirement( self, *, question: str, effective_question: str, asked_by: Optional[int], asked_via: str, project_id: Optional[int], session_id: Optional[int], turn_index: int, state_row: Optional[QASessionState], rewrite_meta: dict[str, Any], context_session_id: Optional[int], query_type: str, action_mode: str, started_at: float, ) -> Optional[dict[str, Any]]: action_taken, linked_signal_id, resolved_project_id, error_reason = await self._create_or_reuse_requirement_signal( question=question, effective_question=effective_question, asked_by=asked_by, project_id=project_id, ) if not action_taken: return None ack_latency_ms = int((time.perf_counter() - started_at) * 1000) trace_id = self._build_ack_trace_id(question) reply_text = self._build_requirement_ack_text( action_taken=action_taken, signal_id=linked_signal_id, project_id=resolved_project_id, error_reason=error_reason, ) source_orchestration_result: Optional[dict[str, Any]] = None if ( linked_signal_id is not None and resolved_project_id is not None and action_taken in {"signal_created", "signal_deduped"} and SignalSourceOrchestrationService.looks_like_signal_source_request(effective_question or question) ): try: source_orchestration_result = await SignalSourceOrchestrationService(self.db).orchestrate_from_signal( tenant_id=self.tenant_id, project_id=int(resolved_project_id), signal_id=int(linked_signal_id), initiating_text=effective_question or question, requested_via="pmflow", operator_user_id=asked_by, dry_run=False, ) except Exception as exc: source_orchestration_result = { "status": "failed", "reason": str(exc), } orchestration_status = str(source_orchestration_result.get("status") or "").lower() if orchestration_status in {"completed", "partial", "planned", "pending_manual"}: source_payload = source_orchestration_result.get("signal_source") if isinstance(source_orchestration_result.get("signal_source"), dict) else {} worker_executions = ( source_orchestration_result.get("worker_executions") if isinstance(source_orchestration_result.get("worker_executions"), list) else [] ) if orchestration_status == "pending_manual": reply_text = ( f"{reply_text}" f" 已记录信号源“{str(source_payload.get('source_name') or '未命名信号源')}”的接入请求，" "但当前仍缺少可直接执行的接入配置，已转待确认。" ) else: reply_text = ( f"{reply_text}" f" 已登记信号源“{str(source_payload.get('source_name') or '未命名信号源')}”，" f"并已下发 {len(worker_executions)} 个 AI worker 执行。" ) answer_text = reply_text sources = [ { "source": "pmflow", "label": "Signal Queue", "score": 0.98 if action_taken != "signal_failed" else 0.42, "metadata": { "action_taken": action_taken, "signal_id": linked_signal_id, "project_id": resolved_project_id, "action_mode": action_mode, "processing_stage": "ack_only", "ack_latency_ms": ack_latency_ms, "trace_id": trace_id, "source_orchestration": source_orchestration_result, }, } ] qa_id: Optional[int] = None try: qa = QARecord( tenant_id=self.tenant_id, question=question, answer=answer_text, sources=sources, intent="statement", domains_used=["signal_write"], response_time_ms=ack_latency_ms, asked_by=asked_by, asked_via=(asked_via or "pmflow")[:20], project_id=resolved_project_id if resolved_project_id is not None else project_id, session_id=session_id, turn_index=int(turn_index or 1), ) self.db.add(qa) await self._update_session_state_after_turn( session_state=state_row, question=question, effective_question=effective_question, answer=answer_text, query_type=query_type if query_type != QUERY_TYPE_UNKNOWN else QUERY_TYPE_STATUS, ambiguity="clear" if action_taken != "signal_failed" else "low_evidence", rewrite_meta=rewrite_meta, sources=sources, ) await self.db.commit() await self.db.refresh(qa) qa_id = int(qa.id) except Exception: await self.db.rollback() try: await self.context_cache.append_session_messages( tenant_id=self.tenant_id, session_id=context_session_id, question=question, answer=answer_text, ) except Exception: pass confidence = 0.94 if action_taken in {"signal_created", "signal_deduped"} else 0.35 missing_fields = [] if resolved_project_id is not None else ["project_id"] return self._build_response_payload( answer=answer_text, sources=sources, domains_used=["signal_write"], response_time_ms=ack_latency_ms, db_query_ms=0, vector_search_ms=0, db_timeout=False, vector_timeout=False, qa_id=qa_id, session_id=int(session_id) if session_id is not None else None, turn_index=int(turn_index or 1), relevance_max=0.99 if action_taken != "signal_failed" else 0.30, query_type=query_type if query_type != QUERY_TYPE_UNKNOWN else QUERY_TYPE_STATUS, confidence=confidence, missing_fields=missing_fields, thinking_mode=THINKING_MODE_NORMAL, action_taken=action_taken, signal_id=linked_signal_id, reply_required=True, reply_text=reply_text, ack_latency_ms=ack_latency_ms, trace_id=trace_id, processing_stage="ack_only", ) def _classify_query_type(self, question: str) -> str: text = str(question or "").strip().lower() if not text: return QUERY_TYPE_UNKNOWN # 若同一句同时出现“多少/几个”和“列出/分别”，优先按 list 处理，避免追问被误判为 count。 if any(k in text for k in LIST_HINTS): return QUERY_TYPE_LIST if any(k in text for k in COUNT_HINTS): return QUERY_TYPE_COUNT if any(k in text for k in STATUS_HINTS): return QUERY_TYPE_STATUS if "为什么" in text or "如何" in text or "怎么" in text: return QUERY_TYPE_EXPLAIN return QUERY_TYPE_UNKNOWN async def _run_deterministic_query( self, *, question: str, project_id: Optional[int], query_type: str, ) -> Optional[dict[str, Any]]: q_type = str(query_type or QUERY_TYPE_UNKNOWN) if q_type == QUERY_TYPE_COUNT: return await self._run_count_query(question=question, project_id=project_id) if q_type == QUERY_TYPE_LIST: return await self._run_list_query(question=question, project_id=project_id) if q_type == QUERY_TYPE_STATUS: progress = await self._try_fast_project_progress_answer(clean_q=question, project_id=project_id) if progress: progress["query_type"] = QUERY_TYPE_STATUS progress.setdefault("confidence", 0.98) progress.setdefault("missing_fields", []) return progress existence = await self._try_fast_project_existence_answer(clean_q=question, project_id=project_id) if existence: existence["query_type"] = QUERY_TYPE_STATUS existence.setdefault("confidence", 0.96) existence.setdefault("missing_fields", []) return existence return None async def _resolve_project_scope_for_question( self, *, question: str, project_id: Optional[int], ) -> tuple[Optional[int], Optional[str], str]: if project_id is not None: row = ( await self.db.execute( select(Project).where( and_( Project.tenant_id == self.tenant_id, Project.id == int(project_id), ) ) ) ).scalar_one_or_none() if row is None: return (None, None, "") return (int(row.id), str(row.title or row.name or f"项目#{int(row.id)}"), "") direct_project_id, direct_project_name = await self._match_project_name_in_question(question=question) if direct_project_id is not None and direct_project_name: return (int(direct_project_id), str(direct_project_name), str(direct_project_name)[:24]) keyword = self._extract_project_lookup_keyword(question) terms = self._extract_query_terms(question, preferred=keyword) for term in terms: matched = await self._search_projects_by_keyword(keyword=term, project_id=None, limit=5) if matched: target = matched[0] return (int(target.id), str(target.title or target.name or f"项目#{int(target.id)}"), term) return (None, None, keyword[:24] if keyword else "") @staticmethod def _normalize_project_subject_text(text: str) -> str: clean = str(text or "").strip().lower() if not clean: return "" clean = clean.replace("—", "-").replace("_", " ") clean = re.sub(r"[^\w\u4e00-\u9fff\- ]+", " ", clean) clean = re.sub( r"(我说的是|我说的|请问|请|帮我|一下|这个里面|这里面|里面|这个|那个|一下子|目前|当前|现在|有多少个|多少个|信号源|信号|项目|系统)", " ", clean, ) clean = re.sub(r"[-]+", " ", clean) clean = re.sub(r"\s+", " ", clean).strip() return clean async def _match_project_name_in_question(self, *, question: str) -> tuple[Optional[int], Optional[str]]: normalized_question = self._normalize_project_subject_text(question) if not normalized_question: return (None, None) projects = ( await self.db.execute( select(Project) .where(Project.tenant_id == self.tenant_id) .order_by(Project.updated_at.desc(), Project.id.desc()) .limit(200) ) ).scalars().all() best_id: Optional[int] = None best_name: Optional[str] = None best_score = 0 for row in projects: candidates = [ str(row.title or "").strip(), str(row.name or "").strip(), ] for name in candidates: if len(name) < 2: continue normalized_name = self._normalize_project_subject_text(name) if len(normalized_name) < 2: continue score = 0 if normalized_name in normalized_question: score = len(normalized_name) else: tokens = [t for t in normalized_name.split(" ") if len(t) >= 2] if tokens and all(t in normalized_question for t in tokens): score = sum(len(t) for t in tokens) if score > best_score: best_score = score best_id = int(row.id) best_name = str(row.title or row.name or f"项目#{int(row.id)}") return (best_id, best_name) async def _run_count_query(self, *, question: str, project_id: Optional[int]) -> Optional[dict[str, Any]]: text = str(question or "").lower() started = time.perf_counter() has_project = "项目" in text has_task = "任务" in text has_signal = "信号" in text or "信号源" in text has_people = self._is_people_query(text) # 先处理更具体的对象（信号/任务），避免“xxx项目里有多少信号”误落到项目总数分支。 if has_signal: scoped_project_id, scoped_project_name, scoped_keyword = await self._resolve_project_scope_for_question( question=text, project_id=project_id, ) if has_project and scoped_project_id is None and scoped_keyword: elapsed = int((time.perf_counter() - started) * 1000) return { "answer": f"未检索到名称或描述包含“{scoped_keyword}”的项目，无法统计信号数。", "sources": [ { "source": "db", "label": "Project database", "score": 0.94, "metadata": {"table": "projects", "query_keyword": scoped_keyword, "match_count": 0}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.94, "missing_fields": ["project_id"], } stmt = select(func.count(Signal.id)).where(Signal.tenant_id == self.tenant_id) if scoped_project_id is not None: stmt = stmt.where(Signal.matched_project_id == int(scoped_project_id)) total = int((await self.db.execute(stmt)).scalar() or 0) elapsed = int((time.perf_counter() - started) * 1000) if scoped_project_id is not None and scoped_project_name: answer = f"项目「{scoped_project_name}」当前关联信号总数为 {total} 条。" else: answer = f"当前可见信号总数为 {total} 条。" return { "answer": answer, "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": { "table": "signals", "count": total, "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.99, "missing_fields": [], } if has_task: scoped_project_id, scoped_project_name, scoped_keyword = await self._resolve_project_scope_for_question( question=text, project_id=project_id, ) if has_project and scoped_project_id is None and scoped_keyword: elapsed = int((time.perf_counter() - started) * 1000) return { "answer": f"未检索到名称或描述包含“{scoped_keyword}”的项目，无法统计任务数。", "sources": [ { "source": "db", "label": "Project database", "score": 0.94, "metadata": {"table": "projects", "query_keyword": scoped_keyword, "match_count": 0}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.94, "missing_fields": ["project_id"], } stmt = select(func.count(Task.id)).where(Task.tenant_id == self.tenant_id) if scoped_project_id is not None: stmt = stmt.where(Task.project_id == int(scoped_project_id)) total = int((await self.db.execute(stmt)).scalar() or 0) elapsed = int((time.perf_counter() - started) * 1000) if scoped_project_id is not None and scoped_project_name: answer = f"项目「{scoped_project_name}」当前任务总数为 {total} 条。" else: answer = f"当前可见任务总数为 {total} 条。" return { "answer": answer, "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": { "table": "tasks", "count": total, "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.99, "missing_fields": [], } if has_people: scoped_project_id: Optional[int] = None scoped_project_name: Optional[str] = None scoped_keyword = "" if has_project or project_id is not None: scoped_project_id, scoped_project_name, scoped_keyword = await self._resolve_project_scope_for_question( question=text, project_id=project_id, ) if has_project and scoped_project_id is None and scoped_keyword and not self._is_people_query(scoped_keyword): elapsed = int((time.perf_counter() - started) * 1000) return { "answer": f"未检索到名称或描述包含“{scoped_keyword}”的项目，无法统计成员人数。", "sources": [ { "source": "db", "label": "Project database", "score": 0.94, "metadata": {"table": "projects", "query_keyword": scoped_keyword, "match_count": 0}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.94, "missing_fields": ["project_id"], } user_stmt = select(func.count(User.id)).where( and_( User.tenant_id == self.tenant_id, User.is_active.is_(True), ) ) if scoped_project_id is not None: member_subq = select(UserProject.user_id).where( and_( UserProject.tenant_id == self.tenant_id, UserProject.project_id == int(scoped_project_id), ) ) user_stmt = user_stmt.where(User.id.in_(member_subq)) total = int((await self.db.execute(user_stmt)).scalar() or 0) elapsed = int((time.perf_counter() - started) * 1000) if scoped_project_id is not None and scoped_project_name: answer = f"项目「{scoped_project_name}」当前活跃成员总数为 {total} 人。" else: answer = f"当前系统里活跃成员总数为 {total} 人。" return { "answer": answer, "sources": [ { "source": "db", "label": "User database", "score": 0.99, "metadata": { "table": "users", "count": total, "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.99, "missing_fields": [], } if has_project: stmt = select(func.count(Project.id)).where(Project.tenant_id == self.tenant_id) if project_id is not None: stmt = stmt.where(Project.id == int(project_id)) total = int((await self.db.execute(stmt)).scalar() or 0) elapsed = int((time.perf_counter() - started) * 1000) return { "answer": f"当前系统里项目总数为 {total} 个。", "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": {"table": "projects", "count": total}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_COUNT, "confidence": 0.99, "missing_fields": [], } return None async def _run_list_query(self, *, question: str, project_id: Optional[int]) -> Optional[dict[str, Any]]: text = str(question or "").lower() started = time.perf_counter() if self._is_people_query(text): scoped_project_id: Optional[int] = None scoped_project_name: Optional[str] = None scoped_keyword = "" if "项目" in text or project_id is not None: scoped_project_id, scoped_project_name, scoped_keyword = await self._resolve_project_scope_for_question( question=text, project_id=project_id, ) if "项目" in text and scoped_project_id is None and scoped_keyword and not self._is_people_query(scoped_keyword): elapsed = int((time.perf_counter() - started) * 1000) return { "answer": f"未检索到名称或描述包含“{scoped_keyword}”的项目，无法列出成员清单。", "sources": [ { "source": "db", "label": "Project database", "score": 0.94, "metadata": {"table": "projects", "query_keyword": scoped_keyword, "match_count": 0}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.94, "missing_fields": ["project_id"], } user_stmt = select(User).where( and_( User.tenant_id == self.tenant_id, User.is_active.is_(True), ) ) if scoped_project_id is not None: member_subq = select(UserProject.user_id).where( and_( UserProject.tenant_id == self.tenant_id, UserProject.project_id == int(scoped_project_id), ) ) user_stmt = user_stmt.where(User.id.in_(member_subq)) members = ( await self.db.execute( user_stmt.order_by(User.updated_at.desc(), User.id.desc()).limit(30) ) ).scalars().all() elapsed = int((time.perf_counter() - started) * 1000) if not members: answer = ( f"项目「{scoped_project_name}」当前没有可列举的成员。" if scoped_project_name else "当前没有可列举的成员。" ) return { "answer": answer, "sources": [ { "source": "db", "label": "User database", "score": 0.95, "metadata": { "table": "users", "count": 0, "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.95, "missing_fields": [], } member_ids = [int(row.id) for row in members] workload_rows = ( await self.db.execute( select(UserWorkload.user_id, UserWorkload.workload_ratio).where( and_( UserWorkload.tenant_id == self.tenant_id, UserWorkload.user_id.in_(member_ids), ) ) ) ).all() workload_map = {int(uid): float(ratio or 0.0) for uid, ratio in workload_rows} project_count_rows = ( await self.db.execute( select(UserProject.user_id, func.count(UserProject.project_id)) .where( and_( UserProject.tenant_id == self.tenant_id, UserProject.user_id.in_(member_ids), ) ) .group_by(UserProject.user_id) ) ).all() project_count_map = {int(uid): int(cnt or 0) for uid, cnt in project_count_rows} lines = [] for row in members: uid = int(row.id) ratio = workload_map.get(uid) if ratio is None: ratio = round(float(row.workload or 0) / 100.0, 2) role = str(row.role or "unknown") project_count = project_count_map.get(uid, 0) lines.append( f"#{uid} {str(row.name or f'User#{uid}')}（角色:{role}，负载:{float(ratio):.2f}，关联项目:{project_count}）" ) elapsed = int((time.perf_counter() - started) * 1000) header = f"项目「{scoped_project_name}」成员清单如下：" if scoped_project_name else "当前团队成员清单如下：" return { "answer": header + "\n" + "\n".join(lines), "sources": [ { "source": "db", "label": "User database", "score": 0.99, "metadata": { "table": "users", "count": len(members), "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.99, "missing_fields": [], } if "信号" in text or "信号源" in text: scoped_project_id, scoped_project_name, scoped_keyword = await self._resolve_project_scope_for_question( question=text, project_id=project_id, ) if "项目" in text and scoped_project_id is None and scoped_keyword: elapsed = int((time.perf_counter() - started) * 1000) return { "answer": f"未检索到名称或描述包含“{scoped_keyword}”的项目，无法列出信号清单。", "sources": [ { "source": "db", "label": "Project database", "score": 0.94, "metadata": {"table": "projects", "query_keyword": scoped_keyword, "match_count": 0}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.94, "missing_fields": ["project_id"], } stmt = select(Signal).where(Signal.tenant_id == self.tenant_id) if scoped_project_id is not None: stmt = stmt.where(Signal.matched_project_id == int(scoped_project_id)) signals = ( await self.db.execute(stmt.order_by(Signal.updated_at.desc(), Signal.id.desc()).limit(30)) ).scalars().all() elapsed = int((time.perf_counter() - started) * 1000) if not signals: answer = ( f"项目「{scoped_project_name}」当前没有可列举的信号。" if scoped_project_name else "当前没有可列举的信号。" ) return { "answer": answer, "sources": [ { "source": "db", "label": "Project database", "score": 0.95, "metadata": { "table": "signals", "count": 0, "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.95, "missing_fields": [], } lines = [] for row in signals: title = str(row.title or row.summary or row.content or f"Signal#{int(row.id)}").replace("\n", " ").strip() status = str(row.status or "unknown") lines.append(f"#{int(row.id)} {title[:90]}（{status}）") header = f"项目「{scoped_project_name}」信号清单如下：" if scoped_project_name else "当前信号清单如下：" return { "answer": header + "\n" + "\n".join(lines), "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": { "table": "signals", "count": len(signals), "project_id": scoped_project_id, "project_name": scoped_project_name, }, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.99, "missing_fields": [], } if "项目" in text: stmt = select(Project).where(Project.tenant_id == self.tenant_id) if project_id is not None: stmt = stmt.where(Project.id == int(project_id)) projects = ( await self.db.execute(stmt.order_by(Project.updated_at.desc(), Project.id.desc()).limit(20)) ).scalars().all() if not projects: elapsed = int((time.perf_counter() - started) * 1000) return { "answer": "当前没有可列举的项目。", "sources": [ { "source": "db", "label": "Project database", "score": 0.95, "metadata": {"table": "projects", "count": 0}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.95, "missing_fields": [], } lines = [f"#{int(p.id)} {str(p.title or p.name or '未命名项目')}（{str(p.status or 'unknown')}）" for p in projects] elapsed = int((time.perf_counter() - started) * 1000) return { "answer": "当前项目列表如下：\n" + "\n".join(lines), "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": {"table": "projects", "count": len(projects)}, } ], "db_query_ms": elapsed, "response_time_ms": elapsed, "query_type": QUERY_TYPE_LIST, "confidence": 0.99, "missing_fields": [], } return None async def _ensure_session_state(self, *, session_id: Optional[int]) -> Optional[QASessionState]: if session_id is None: return None row = ( await self.db.execute( select(QASessionState).where( and_( QASessionState.tenant_id == self.tenant_id, QASessionState.session_id == int(session_id), ) ) ) ).scalar_one_or_none() if row is None: row = QASessionState( tenant_id=self.tenant_id, session_id=int(session_id), last_subject=None, last_entity_ids=[], last_intent=None, ambiguity="unknown", rewrite_history=[], ) self.db.add(row) await self.db.flush() return row def _rewrite_question_with_state( self, *, question: str, conversation: list[dict[str, str]], session_state: Optional[QASessionState], ) -> tuple[str, dict[str, Any]]: expanded = self._expand_followup_question(question, conversation) current = str(expanded or question or "").strip() if not current: return current, {"mode": "empty"} lower = current.lower() has_referential = any(k in lower for k in FOLLOWUP_REFERENTIAL_KEYWORDS) if not has_referential: return current, {"mode": "direct"} state_subject = "" state_entity_ids: list[int] = [] if session_state is not None: state_subject = str(session_state.last_subject or "").strip() if isinstance(session_state.last_entity_ids, list): for value in session_state.last_entity_ids: try: state_entity_ids.append(int(value)) except Exception: continue if state_subject: rewritten = f"围绕“{state_subject}”回答：{current}" return rewritten[:1200], {"mode": "state_subject", "subject": state_subject} if state_entity_ids: anchor = ",".join(str(x) for x in state_entity_ids[:6]) rewritten = f"围绕会话实体ID[{anchor}]回答：{current}" return rewritten[:1200], {"mode": "state_entities", "entity_ids": state_entity_ids[:6]} return current, {"mode": "referential_without_subject"} def _build_clarification_question( self, *, original_question: str, effective_question: str, session_state: Optional[QASessionState], ) -> Optional[str]: text = str(original_question or "").strip().lower() if not text: return None if self._is_correction_query(text): if self._has_explicit_target_scope(text): return None return ( "收到，你在纠正上一条回答。请直接说你要的目标范围，例如：" "`我问的是 AI 引擎驱动方式，第一层是否调用 openclaw？`" ) has_referential = any(k in text for k in FOLLOWUP_REFERENTIAL_KEYWORDS) if not has_referential: return None state_subject = str(getattr(session_state, "last_subject", "") or "").strip() state_entity_ids = getattr(session_state, "last_entity_ids", None) has_state_entities = isinstance(state_entity_ids, list) and any(str(x).strip() for x in state_entity_ids) if state_subject or has_state_entities: return None if str(effective_question or "").strip() != text and "补充问题" in str(effective_question): return None return "你这句缺少明确主语。请先指明对象，例如：`分别是哪些项目？` 或 `请列出这些项目的名称和ID`。" async def _update_session_state_after_turn( self, *, session_state: Optional[QASessionState], question: str, effective_question: str, answer: str, query_type: str, ambiguity: str, rewrite_meta: dict[str, Any], sources: list[dict[str, Any]], ) -> None: if session_state is None: return subject = self._extract_state_subject(question=question, answer=answer, sources=sources) if subject: session_state.last_subject = subject[:255] session_state.last_intent = "question" session_state.ambiguity = str(ambiguity or "clear")[:20] session_state.last_entity_ids = self._collect_entity_ids_from_sources(sources) history = session_state.rewrite_history if isinstance(session_state.rewrite_history, list) else [] history.append( { "at": now_cn_aware().isoformat(), "question": question[:500], "effective_question": effective_question[:500], "query_type": str(query_type or QUERY_TYPE_UNKNOWN), "ambiguity": str(ambiguity or "clear"), "rewrite_meta": rewrite_meta if isinstance(rewrite_meta, dict) else {}, } ) session_state.rewrite_history = history[-20:] await self.db.flush() def _collect_entity_ids_from_sources(self, sources: list[dict[str, Any]]) -> list[int]: ids: list[int] = [] for source in sources: if not isinstance(source, dict): continue meta = source.get("metadata") if isinstance(source.get("metadata"), dict) else {} for key in ( "project_id", "task_id", "signal_id", "user_id", "entity_id", "subject_entity_id", "object_entity_id", ): value = meta.get(key) try: if value is not None: ids.append(int(value)) except Exception: continue matched = meta.get("matched_project_ids") if isinstance(matched, list): for value in matched: try: ids.append(int(value)) except Exception: continue uniq: list[int] = [] for value in ids: if value not in uniq: uniq.append(value) return uniq[:20] def _extract_state_subject( self, *, question: str, answer: str, sources: list[dict[str, Any]], ) -> str: for source in sources: if not isinstance(source, dict): continue meta = source.get("metadata") if isinstance(source.get("metadata"), dict) else {} project_keyword = str(meta.get("query_keyword") or "").strip() if project_keyword: return project_keyword[:255] subject_name = str(meta.get("subject_name") or "").strip() object_name = str(meta.get("object_name") or "").strip() if subject_name and object_name: return f"{subject_name}->{object_name}"[:255] text = str(question or "").strip() keyword = self._extract_project_lookup_keyword(text) if len(keyword) >= 2: return keyword[:255] answer_text = str(answer or "").strip() m = re.search(r"项目[「\"]?([^」\"\n]{2,40})", answer_text) if m: return str(m.group(1)).strip()[:255] return "" async def get_session_state_replay(self, *, session_id: int, user_id: int) -> dict[str, Any]: session = ( await self.db.execute( select(QASession).where( and_( QASession.tenant_id == self.tenant_id, QASession.id == int(session_id), QASession.user_id == int(user_id), ) ) ) ).scalar_one_or_none() if session is None: raise ValueError("qa session 不存在") state = await self._ensure_session_state(session_id=int(session_id)) history = state.rewrite_history if isinstance(state.rewrite_history, list) else [] return { "session_id": int(session_id), "last_subject": str(state.last_subject or "") if state else "", "last_entity_ids": state.last_entity_ids if state else [], "last_intent": str(state.last_intent or "") if state else "", "ambiguity": str(state.ambiguity or "") if state else "", "rewrite_history": history[-20:], "updated_at": state.updated_at.isoformat() if state and state.updated_at else None, } async def _try_glm_project_semantic_answer( self, *, clean_q: str, project_id: Optional[int], ) -> Optional[dict[str, Any]]: question = str(clean_q or "").strip() if not question: return None if self._is_people_query(question): return None if not any(k in question for k in ("项目", "系统", "进度", "状态", "有没有", "是否", "存在", "立项", "任务")): return None db_started = time.perf_counter() stmt = select(Project).where(Project.tenant_id == self.tenant_id) if project_id is not None: stmt = stmt.where(Project.id == int(project_id)) projects = (await self.db.execute(stmt.order_by(Project.updated_at.desc()).limit(120))).scalars().all() if not projects: db_query_ms = int((time.perf_counter() - db_started) * 1000) return { "answer": "当前项目库为空，无法检索到匹配项目。", "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": {"table": "projects", "match_count": 0, "retrieval_mode": "glm_semantic"}, } ], "db_query_ms": db_query_ms, "response_time_ms": db_query_ms, } project_ids = [int(p.id) for p in projects] task_rows = [] if project_ids: task_rows = ( await self.db.execute( select(Task).where( and_( Task.tenant_id == self.tenant_id, Task.project_id.in_(project_ids), ) ) ) ).scalars().all() db_query_ms = int((time.perf_counter() - db_started) * 1000) done_statuses = {"done", "completed", "已完成", "完成", "closed", "关闭"} doing_statuses = {"in_progress", "doing", "进行中", "开发中", "处理中", "processing"} pending_statuses = {"todo", "pending", "待处理", "未开始", "open"} task_stats: dict[int, dict[str, Any]] = {} for pid in project_ids: task_stats[pid] = { "task_total": 0, "done_count": 0, "doing_count": 0, "pending_count": 0, "latest_task_updated_at": None, } for row in task_rows: pid = int(row.project_id) if pid not in task_stats: continue stat = task_stats[pid] stat["task_total"] += 1 status = str(row.status or "").strip().lower() if status in done_statuses: stat["done_count"] += 1 elif status in doing_statuses: stat["doing_count"] += 1 elif status in pending_statuses: stat["pending_count"] += 1 else: stat["pending_count"] += 1 row_time = row.updated_at or row.created_at latest = stat.get("latest_task_updated_at") if row_time and (latest is None or row_time > latest): stat["latest_task_updated_at"] = row_time project_index: list[dict[str, Any]] = [] for p in projects: pid = int(p.id) stats = task_stats.get(pid) or {} project_index.append( { "id": pid, "title": str(p.title or p.name or "")[:120], "name": str(p.name or "")[:120], "status": str(p.status or "unknown"), "description": str(p.description or "")[:220], "task_total": int(stats.get("task_total") or 0), "done_count": int(stats.get("done_count") or 0), "doing_count": int(stats.get("doing_count") or 0), "pending_count": int(stats.get("pending_count") or 0), "latest_task_updated_at": ( stats.get("latest_task_updated_at").isoformat() if stats.get("latest_task_updated_at") is not None else None ), } ) prompt = PROMPT_RAG_DB_PROJECT_QA.format( question=question[:600], project_index=json.dumps(project_index, ensure_ascii=False), ).strip() llm_started = time.perf_counter() try: raw = await self.llm.generate( prompt, db=self.db, tenant_id=self.tenant_id, function_name="rag_db_project_semantic_retrieval", temperature=0.0, max_tokens=520, use_prompt_profile=False, ) parsed = self._parse_json_object(raw) if not parsed: return None handled = bool(parsed.get("handled")) answer = str(parsed.get("answer") or "").strip() confidence = self._to_float(parsed.get("confidence"), 0.0) matched_ids_raw = parsed.get("matched_project_ids") matched_ids: list[int] = [] if isinstance(matched_ids_raw, list): for x in matched_ids_raw: try: matched_ids.append(int(x)) except Exception: continue matched_ids = matched_ids[:8] if not handled or not answer: return None response_ms = int((time.perf_counter() - llm_started) * 1000) + db_query_ms return { "answer": answer, "sources": [ { "source": "db", "label": "Project database", "score": round(max(0.0, min(1.0, confidence or 0.9)), 4), "metadata": { "table": "projects", "matched_project_ids": matched_ids, "project_index_count": len(project_index), "retrieval_mode": "glm_semantic", }, } ], "db_query_ms": db_query_ms, "response_time_ms": response_ms, } except Exception: return None async def _persist_fast_answer( self, *, question: str, asked_by: Optional[int], asked_via: str, project_id: Optional[int], fast_result: dict[str, Any], session_id: Optional[int] = None, external_session_id: Optional[int] = None, turn_index: Optional[int] = None, query_type: str = QUERY_TYPE_EXPLAIN, missing_fields: Optional[list[str]] = None, state_row: Optional[QASessionState] = None, effective_question: Optional[str] = None, rewrite_meta: Optional[dict[str, Any]] = None, thinking_mode: str = THINKING_MODE_NORMAL, ) -> dict[str, Any]: started = time.perf_counter() answer_text = str(fast_result.get("answer") or "未命中结果") sources = list(fast_result.get("sources") or []) active_session_id = session_id active_turn_index = turn_index if active_session_id is None or active_turn_index is None: active_session_id, active_turn_index = await self._touch_session_for_new_record( asked_by=asked_by, asked_via=asked_via, project_id=project_id, question=question, ) active_turn_index = int(active_turn_index or 1) active_session_id_int = int(active_session_id) if active_session_id is not None else None confidence = self._to_float(fast_result.get("confidence"), 0.96) if missing_fields is None: missing_fields = list(fast_result.get("missing_fields") or []) final_query_type = str(fast_result.get("query_type") or query_type or QUERY_TYPE_EXPLAIN) qa = QARecord( tenant_id=self.tenant_id, question=question, answer=answer_text, sources=sources, intent="question", domains_used=["db"], response_time_ms=int(fast_result.get("response_time_ms") or 0), asked_by=asked_by, asked_via=(asked_via or "pmflow")[:20], project_id=project_id, session_id=active_session_id_int, turn_index=active_turn_index, ) self.db.add(qa) if state_row is None and active_session_id_int is not None: state_row = await self._ensure_session_state(session_id=active_session_id_int) await self._update_session_state_after_turn( session_state=state_row, question=question, effective_question=str(effective_question or question), answer=answer_text, query_type=final_query_type, ambiguity="clear", rewrite_meta=rewrite_meta or {}, sources=sources, ) await self.db.commit() await self.db.refresh(qa) cache_session_id = active_session_id_int if active_session_id_int is not None else ( int(external_session_id) if external_session_id is not None else None ) await self.context_cache.append_session_messages( tenant_id=self.tenant_id, session_id=cache_session_id, question=question, answer=answer_text, ) elapsed = int((time.perf_counter() - started) * 1000) return self._build_response_payload( answer=answer_text, sources=sources, domains_used=["db"], response_time_ms=int(fast_result.get("response_time_ms") or elapsed), db_query_ms=int(fast_result.get("db_query_ms") or 0), vector_search_ms=0, db_timeout=False, vector_timeout=False, qa_id=int(qa.id), session_id=active_session_id_int, turn_index=active_turn_index, relevance_max=0.99, query_type=final_query_type, confidence=confidence, missing_fields=list(missing_fields or []), thinking_mode=thinking_mode, ) async def _try_fast_project_progress_answer( self, *, clean_q: str, project_id: Optional[int], ) -> Optional[dict[str, Any]]: question = str(clean_q or "").strip() if not question: return None if not PROJECT_PROGRESS_RE.search(question): return None if "项目" not in question and "系统" not in question: return None primary_keyword = self._extract_project_lookup_keyword(question) query_terms = self._extract_query_terms(question, preferred=primary_keyword) if not query_terms: return None started = time.perf_counter() matched: list[Project] = [] used_keyword = query_terms[0] for term in query_terms: rows = await self._search_projects_by_keyword(keyword=term, project_id=project_id, limit=3) if rows: matched = rows used_keyword = term break if not matched: db_query_ms = int((time.perf_counter() - started) * 1000) return { "answer": f"未检索到名称或描述包含“{used_keyword}”的项目，无法判断进度。", "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": {"table": "projects", "query_keyword": used_keyword, "match_count": 0}, } ], "db_query_ms": db_query_ms, "response_time_ms": db_query_ms, } target = matched[0] task_rows = ( await self.db.execute( select(Task).where( and_( Task.tenant_id == self.tenant_id, Task.project_id == int(target.id), ) ) ) ).scalars().all() db_query_ms = int((time.perf_counter() - started) * 1000) total = len(task_rows) done_statuses = {"done", "completed", "已完成", "完成", "closed", "关闭"} doing_statuses = {"in_progress", "doing", "进行中", "开发中", "处理中", "processing"} pending_statuses = {"todo", "pending", "待处理", "未开始", "open"} done = 0 doing = 0 pending = 0 latest_update: Optional[datetime] = None for row in task_rows: status = str(row.status or "").strip().lower() if status in done_statuses: done += 1 elif status in doing_statuses: doing += 1 elif status in pending_statuses: pending += 1 else: pending += 1 row_time = row.updated_at or row.created_at if row_time and (latest_update is None or row_time > latest_update): latest_update = row_time project_name = str(target.title or target.name or f"项目#{int(target.id)}") project_status = str(target.status or "unknown") if total == 0: answer = ( f"项目「{project_name}」当前状态为 {project_status}。" "目前未检索到该项目的任务拆解记录，因此只能确认项目状态，无法量化阶段进度。" ) else: completion = round((done / total) * 100, 1) latest_text = latest_update.strftime("%Y-%m-%d %H:%M") if latest_update else "未知" answer = ( f"项目「{project_name}」当前状态为 {project_status}。" f"任务总数 {total}，已完成 {done}，进行中 {doing}，待处理 {pending}，完成率 {completion}%。" f"最近任务更新时间：{latest_text}。" ) if len(matched) > 1: others = "、".join([str(x.title or x.name or f'#{x.id}') for x in matched[1:]]) answer += f" 另有 {len(matched) - 1} 个同名候选项目：{others}。" return { "answer": answer, "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": { "table": "projects", "project_id": int(target.id), "query_keyword": used_keyword, "match_count": len(matched), }, }, { "source": "db", "label": "Project database", "score": 0.98, "metadata": { "table": "tasks", "project_id": int(target.id), "task_total": total, "done_count": done, "doing_count": doing, "pending_count": pending, }, }, ], "db_query_ms": db_query_ms, "response_time_ms": db_query_ms, } async def _try_fast_project_existence_answer( self, *, clean_q: str, project_id: Optional[int], ) -> Optional[dict[str, Any]]: question = str(clean_q or "").strip() if not question: return None if "项目" not in question and "系统" not in question: return None if ( "吗" not in question and "?" not in question and "？" not in question and "是否有" not in question and "有没有" not in question and "有无" not in question and not PROJECT_EXISTENCE_RE.search(question) ): return None primary_keyword = self._extract_project_lookup_keyword(question) query_terms = self._extract_query_terms(question, preferred=primary_keyword) if not query_terms: return None started = time.perf_counter() matched: list[Project] = [] used_keyword = query_terms[0] for term in query_terms: rows = await self._search_projects_by_keyword(keyword=term, project_id=project_id, limit=5) if rows: matched = rows used_keyword = term break db_query_ms = int((time.perf_counter() - started) * 1000) if matched: lines = [ f"#{int(p.id)} {str(p.title or p.name or '未命名项目')}（状态:{str(p.status or 'unknown')}）" for p in matched ] answer = f"有，当前匹配到 {len(matched)} 个相关项目：\n" + "\n".join(lines) else: answer = f"暂未检索到名称或描述包含“{used_keyword}”的项目。" return { "answer": answer, "sources": [ { "source": "db", "label": "Project database", "score": 0.99, "metadata": { "table": "projects", "query_keyword": used_keyword, "match_count": len(matched), }, } ], "db_query_ms": db_query_ms, } async def _search_projects_by_keyword( self, *, keyword: str, project_id: Optional[int], limit: int, ) -> list[Project]: term = str(keyword or "").strip() if not term: return [] stmt = select(Project).where(Project.tenant_id == self.tenant_id) if project_id is not None: stmt = stmt.where(Project.id == int(project_id)) like = f"%{term}%" stmt = stmt.where( or_( Project.name.ilike(like), Project.title.ilike(like), Project.description.ilike(like), ) ) return (await self.db.execute(stmt.limit(max(1, int(limit))))).scalars().all() @staticmethod def _extract_project_lookup_keyword(question: str) -> str: text = str(question or "").strip() if not text: return "" text = text.replace("？", "?") direct_patterns = [ r"(?:是否有|有没有|有无)([\u4e00-\u9fffA-Za-z0-9_-]{1,24})(?:项目|系统)?", r"(?:是否存在|存在)([\u4e00-\u9fffA-Za-z0-9_-]{1,24})(?:项目|系统)?", ] for pattern in direct_patterns: m = re.search(pattern, text) if m: value = str(m.group(1) or "").strip(" ，。,.!?？：:;；“”\"'()（）") value = QUESTION_NOISE_RE.sub("", value).strip() if len(value) >= 2: return value[:24] anchored_matches = list(re.finditer(r"([\u4e00-\u9fffA-Za-z0-9_-]{1,24})(?:项目|系统)", text)) for m in reversed(anchored_matches): value = str(m.group(1) or "").strip(" ，。,.!?？：:;；“”\"'()（）") value = QUESTION_NOISE_RE.sub("", value).strip() if len(value) >= 2: return value[:24] patterns = [ r"有(?:没有)?(.{1,24}?)(?:项目|系统).*\?", r"项目.*?有(?:没有)?(.{1,24}?)(?:吗|\?)", r"是否有(.{1,24}?)(?:项目|系统)", r"(.{1,24}?)(?:项目|系统)(?:有吗|有没有|是否有)", ] for pattern in patterns: m = re.search(pattern, text) if not m: continue kw = str(m.group(1) or "").strip(" ，。,.!?？：:;；“”\"'()（）") kw = QUESTION_NOISE_RE.sub("", kw).strip() if kw: return kw[:24] fallback = re.sub(r"[?？]", " ", text) fallback = re.sub( r"(项目|系统|目前|当前|现在|我们|有没有|是否有|有吗|有|吗|什么|进度|层面|阶段|是在|在|是|请问|一下|多少)", " ", fallback, ) fallback = re.sub(r"\s+", " ", fallback).strip(" ，。,.!?？：:;；") return fallback[:24] @staticmethod def _extract_query_terms(question: str, *, preferred: str = "") -> list[str]: base = str(question or "").strip() terms: list[str] = [] pref = str(preferred or "").strip() if len(pref) >= 2: terms.append(pref[:24]) clean = re.sub(r"[?？,，。!！:：;；()（）\"“”']", " ", base) clean = QUESTION_NOISE_RE.sub(" ", clean) clean = re.sub(r"(项目|系统|进度|状态|阶段|完成率|层面)", " ", clean) clean = re.sub(r"\s+", " ", clean).strip() token_candidates: list[str] = [] for token in clean.split(" "): t = token.strip() if len(t) >= 2: token_candidates.append(t[:24]) for m in re.finditer(r"[\u4e00-\u9fff]{2,10}", clean): token_candidates.append(m.group(0)[:24]) for t in sorted(token_candidates, key=len, reverse=True): if t and t not in terms: terms.append(t) return terms[:6] @staticmethod def _is_people_query(question: str) -> bool: text = str(question or "").strip().lower() if not text: return False return any(k in text for k in PEOPLE_KEYWORDS) # ------------------------------------------------------------------ # Manifest / Entity resolution / Agentic retrieval # ------------------------------------------------------------------ async def get_project_manifest(self, *, project_id: Optional[int] = None) -> dict[str, Any]: system_snapshot = await self.context_cache.get_system_snapshot( db=self.db, tenant_id=self.tenant_id, ) symbol_snapshot = await self._get_symbol_snapshot_cached() project_info: dict[str, Any] = {} if project_id is not None: row = ( await self.db.execute( select(Project).where( and_( Project.tenant_id == self.tenant_id, Project.id == int(project_id), ) ) ) ).scalar_one_or_none() if row is not None: project_info = { "id": int(row.id), "title": str(row.title or row.name or f"项目#{int(row.id)}"), "status": str(row.status or "unknown"), } kg_overview = {"entity_total": 0, "relation_total": 0, "evidence_total": 0} try: kg_overview = { "entity_total": int( (await self.db.execute(select(func.count(KGEntity.id)).where(KGEntity.tenant_id == self.tenant_id))) .scalar_one() or 0 ), "relation_total": int( ( await self.db.execute(select(func.count(KGRelation.id)).where(KGRelation.tenant_id == self.tenant_id)) ).scalar_one() or 0 ), "evidence_total": int( ( await self.db.execute(select(func.count(KGEvidence.id)).where(KGEvidence.tenant_id == self.tenant_id)) ).scalar_one() or 0 ), } except Exception: kg_overview = {"entity_total": 0, "relation_total": 0, "evidence_total": 0} platform = ( system_snapshot.get("platform_profile") if isinstance(system_snapshot.get("platform_profile"), dict) else {} ) backend = platform.get("backend") if isinstance(platform.get("backend"), dict) else {} module_counts = backend.get("module_counts") if isinstance(backend.get("module_counts"), dict) else {} feature_catalog = platform.get("feature_catalog") if isinstance(platform.get("feature_catalog"), list) else [] feature_summary = { "feature_total": len(feature_catalog), "top_features": [ { "name": str(item.get("name") or ""), "route": str(item.get("route") or ""), } for item in feature_catalog[:10] if isinstance(item, dict) ], } return { "tenant_id": self.tenant_id, "project": project_info, "module_counts": module_counts, "feature_summary": feature_summary, "symbol_summary": symbol_snapshot, "kg_overview": kg_overview, "manifest_text": self._compose_manifest_text( project_info=project_info, module_counts=module_counts, feature_summary=feature_summary, symbol_snapshot=symbol_snapshot, kg_overview=kg_overview, ), "generated_at": now_cn_aware().isoformat(), } async def entity_resolution( self, *, question: str, project_id: Optional[int] = None, limit: int = 10, ) -> dict[str, Any]: normalized = self._normalize_entity_text(question) alias_terms = self._collect_alias_terms(normalized) query_terms = self._extract_query_terms(question, preferred=self._extract_project_lookup_keyword(question)) for term in alias_terms: if term not in query_terms: query_terms.append(term) candidates: list[dict[str, Any]] = [] if query_terms: try: max_terms = query_terms[:6] conditions = [] for term in max_terms: like = f"%{term}%" conditions.append(KGEntity.display_name.ilike(like)) conditions.append(KGEntity.entity_key.ilike(like)) rows = ( await self.db.execute( select(KGEntity) .where( and_( KGEntity.tenant_id == self.tenant_id, or_(*conditions), ) ) .order_by(KGEntity.last_seen_at.desc(), KGEntity.id.desc()) .limit(max(20, int(limit) * 4)) ) ).scalars().all() for row in rows: payload = self._safe_dict(row.payload) row_project_id = payload.get("project_id") if project_id is not None and row_project_id is not None and int(row_project_id) != int(project_id): continue score = 0.0 lowered_name = str(row.display_name or "").lower() lowered_key = str(row.entity_key or "").lower() for term in max_terms: term_lower = term.lower() if term_lower in lowered_name: score += 0.34 if term_lower in lowered_key: score += 0.22 if score <= 0: continue candidates.append( { "entity_id": int(row.id), "entity_type": str(row.entity_type or ""), "entity_key": str(row.entity_key or ""), "display_name": str(row.display_name or ""), "score": round(min(score, 0.99), 4), } ) except Exception: candidates = [] candidates.sort(key=lambda x: float(x.get("score") or 0.0), reverse=True) return { "normalized_question": normalized, "canonical_terms": alias_terms, "query_terms": query_terms[:8], "matched_entities": candidates[: max(1, int(limit))], } async def get_feature_catalog( self, *, query: Optional[str] = None, limit: int = 20, ) -> dict[str, Any]: system_snapshot = await self.context_cache.get_system_snapshot( db=self.db, tenant_id=self.tenant_id, ) catalog = self._feature_catalog_from_snapshot(system_snapshot) if query and str(query).strip(): rows = self._match_feature_catalog_entries( question=str(query), system_snapshot=system_snapshot, limit=max(1, min(int(limit), 100)), ) return { "query": str(query), "count": len(rows), "items": rows, } max_rows = max(1, min(int(limit), 200)) return { "query": str(query or ""), "count": min(len(catalog), max_rows), "items": catalog[:max_rows], } def _compose_manifest_text( self, *, project_info: dict[str, Any], module_counts: dict[str, Any], feature_summary: dict[str, Any], symbol_snapshot: dict[str, Any], kg_overview: dict[str, Any], ) -> str: lines: list[str] = [] if project_info: lines.append( f"当前项目：#{int(project_info.get('id') or 0)} {str(project_info.get('title') or 'unknown')} " f"(status={str(project_info.get('status') or 'unknown')})" ) lines.append( "后端模块规模：" f"api={int(module_counts.get('api', 0))}," f"services={int(module_counts.get('services', 0))}," f"models={int(module_counts.get('models', 0))}," f"schemas={int(module_counts.get('schemas', 0))}" ) lines.append(f"产品功能：total={int(feature_summary.get('feature_total', 0))}") top_features = feature_summary.get("top_features") if isinstance(top_features, list) and top_features: feature_brief = "；".join( f"{str(item.get('name') or '')}({str(item.get('route') or '')})" for item in top_features[:6] if isinstance(item, dict) and str(item.get("name") or "").strip() ) if feature_brief: lines.append(f"核心入口：{feature_brief}") lines.append( "符号索引：" f"files={int(symbol_snapshot.get('indexed_files', 0))}," f"symbols={int(symbol_snapshot.get('symbol_total', 0))}," f"python={int(symbol_snapshot.get('python_symbols', 0))}," f"typescript={int(symbol_snapshot.get('typescript_symbols', 0))}" ) lines.append( "知识图谱：" f"entities={int(kg_overview.get('entity_total', 0))}," f"relations={int(kg_overview.get('relation_total', 0))}," f"evidences={int(kg_overview.get('evidence_total', 0))}" ) top_modules = symbol_snapshot.get("top_modules") if isinstance(top_modules, list) and top_modules: brief = "；".join( f"{str(item.get('module') or '')}:{int(item.get('symbols') or 0)}" for item in top_modules[:5] if isinstance(item, dict) ) if brief: lines.append(f"热点模块：{brief}") return "\n".join(lines)[:2200] @staticmethod def _normalize_entity_text(text: str) -> str: normalized = str(text or "").strip().lower() normalized = normalized.replace("—", "-").replace("_", " ") normalized = re.sub(r"[^\w\u4e00-\u9fff\- ]+", " ", normalized) normalized = re.sub(r"\s+", " ", normalized).strip() return normalized def _collect_alias_terms(self, normalized_question: str) -> list[str]: aliases: list[str] = [] for token, canonical in ENTITY_ALIAS_MAP.items(): if token in normalized_question and canonical not in aliases: aliases.append(canonical) return aliases[:8] @staticmethod def _feature_catalog_from_snapshot(system_snapshot: Optional[dict[str, Any]]) -> list[dict[str, Any]]: if not isinstance(system_snapshot, dict): return [] platform = system_snapshot.get("platform_profile") if not isinstance(platform, dict): return [] catalog = platform.get("feature_catalog") if not isinstance(catalog, list): return [] return [item for item in catalog if isinstance(item, dict)] def _match_feature_catalog_entries( self, *, question: str, system_snapshot: Optional[dict[str, Any]], limit: int = 4, ) -> list[dict[str, Any]]: text = str(question or "").strip().lower() if not text: return [] catalog = self._feature_catalog_from_snapshot(system_snapshot) if not catalog: return [] query_terms = [x.lower() for x in self._extract_query_terms(question)[:8] if str(x).strip()] matched: list[dict[str, Any]] = [] for item in catalog: name = str(item.get("name") or "").strip() route = str(item.get("route") or "").strip() component = str(item.get("page_component") or "").strip() aliases_raw = item.get("aliases") alias_values = aliases_raw if isinstance(aliases_raw, list) else [] alias_pool = [name, route, component, *[str(x) for x in alias_values]] alias_pool = [a.strip().lower() for a in alias_pool if str(a).strip()] if not alias_pool: continue score = 0.0 for alias in alias_pool: if alias and alias in text: score += 0.45 for term in query_terms: if term and any(term in alias or alias in term for alias in alias_pool): score += 0.18 if score <= 0: continue matched.append( { **item, "match_score": round(min(score, 0.99), 4), } ) matched.sort(key=lambda x: float(x.get("match_score") or 0.0), reverse=True) return matched[: max(1, int(limit))] def _retrieve_feature_catalog_context( self, *, question: str, system_snapshot: Optional[dict[str, Any]], max_items: int = 4, ) -> list[RetrievedItem]: rows = self._match_feature_catalog_entries( question=question, system_snapshot=system_snapshot, limit=max_items, ) out: list[RetrievedItem] = [] for row in rows: name = str(row.get("name") or "unknown_feature") route = str(row.get("route") or "") summary = str(row.get("summary") or "") page_file = str(row.get("page_file") or "") page_component = str(row.get("page_component") or "") apis = row.get("apis") if isinstance(row.get("apis"), list) else [] api_text = "、".join(str(x) for x in apis[:4] if str(x).strip()) content = f"功能: {name}\n用途: {summary}\n入口: {route}\n页面: {page_file}" if api_text: content += f"\n相关接口: {api_text}" score = min(0.99, max(0.62, float(row.get("match_score") or 0.62))) out.append( RetrievedItem( source="feature_catalog", content=content[:1200], score=score, source_label="Feature Catalog", metadata={ "feature_name": name, "route": route, "page_file": page_file, "page_component": page_component, "apis": [str(x) for x in apis[:8] if str(x).strip()], "match_score": round(score, 4), }, ) ) return out def _collect_feature_evidence( self, *, question: str, retrieved: list[RetrievedItem], system_snapshot: Optional[dict[str, Any]], ) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for item in retrieved: if str(item.source or "").strip().lower() != "feature_catalog": continue meta = item.metadata if isinstance(item.metadata, dict) else {} route = str(meta.get("route") or "").strip() page_file = str(meta.get("page_file") or "").strip() feature_name = str(meta.get("feature_name") or "").strip() apis = meta.get("apis") if isinstance(meta.get("apis"), list) else [] clean_apis = [str(x).strip() for x in apis if str(x).strip()] if not feature_name and not route and not page_file: continue rows.append( { "feature_name": feature_name, "route": route, "page_file": page_file, "apis": clean_apis[:6], } ) if rows: return rows[:4] # 若检索结果里没有 feature_catalog，回退到 snapshot 匹配结果 fallback = self._match_feature_catalog_entries( question=question, system_snapshot=system_snapshot, limit=3, ) for row in fallback: apis = row.get("apis") if isinstance(row.get("apis"), list) else [] rows.append( { "feature_name": str(row.get("name") or ""), "route": str(row.get("route") or ""), "page_file": str(row.get("page_file") or ""), "apis": [str(x).strip() for x in apis[:6] if str(x).strip()], } ) return rows[:3] def _enforce_feature_evidence_in_answer( self, *, question: str, answer_text: str, retrieved: list[RetrievedItem], system_snapshot: Optional[dict[str, Any]], ) -> str: text = str(question or "").strip().lower() has_feature_token = any(k in text for k in FEATURE_QUERY_KEYWORDS) ask_feature = has_feature_token and any(k in text for k in FEATURE_PURPOSE_HINTS) if not ask_feature: return answer_text evidence_rows = self._collect_feature_evidence( question=question, retrieved=retrieved, system_snapshot=system_snapshot, ) if not evidence_rows: return answer_text # 已包含明确入口与接口信息时不重复追加 lowered_answer = str(answer_text or "").lower() if ("/" in lowered_answer and "接口" in lowered_answer) or "证据补充" in lowered_answer: return answer_text lines: list[str] = [str(answer_text or "").strip(), "", "证据补充（Feature Catalog）："] for idx, row in enumerate(evidence_rows, start=1): feature_name = str(row.get("feature_name") or "未知功能") route = str(row.get("route") or "") page_file = str(row.get("page_file") or "") apis = row.get("apis") if isinstance(row.get("apis"), list) else [] api_text = "、".join(str(x) for x in apis[:3] if str(x).strip()) line = f"{idx}. {feature_name}" if route: line += f" | 路由: {route}" if page_file: line += f" | 页面: {page_file}" if api_text: line += f" | API: {api_text}" lines.append(line) return "\n".join(lines).strip()[:3200] def _symbol_cache_key(self) -> str: return f"{self.tenant_id}:backend_symbol_snapshot" async def _get_symbol_snapshot_cached(self, *, ttl_seconds: int = 300) -> dict[str, Any]: now = time.time() key = self._symbol_cache_key() cached = self._symbol_snapshot_cache.get(key) if cached and cached[0] > now: return dict(cached[1]) try: service = SymbolIndexService(self.db, tenant_id=self.tenant_id) built = await service.get_overview(top_n_files=10, auto_sync_if_empty=True) except Exception: built = self._build_symbol_index_snapshot() self._symbol_snapshot_cache[key] = (now + max(60, int(ttl_seconds)), dict(built)) return built def _build_symbol_index_snapshot( self, *, max_files: int = 240, max_symbols: int = 4000, ) -> dict[str, Any]: backend_root = Path(__file__).resolve().parents[2] app_root = backend_root / "app" if not app_root.exists(): return { "root": str(app_root), "indexed_files": 0, "symbol_total": 0, "python_symbols": 0, "typescript_symbols": 0, "top_modules": [], "generated_at": now_cn_aware().isoformat(), } candidates = [ p for p in app_root.rglob("*") if p.is_file() and p.suffix.lower() in {".py", ".ts", ".tsx", ".js", ".jsx"} ] candidates.sort(key=lambda p: str(p)) file_paths = candidates[: max(1, int(max_files))] python_symbols = 0 ts_symbols = 0 total_symbols = 0 module_counter: dict[str, int] = {} for path in file_paths: rel = str(path.relative_to(backend_root)) module = rel.split("/", 2)[:2] module_name = "/".join(module) if module else rel try: text = path.read_text(encoding="utf-8", errors="ignore") except Exception: continue symbols_in_file = 0 if path.suffix.lower() == ".py": extracted = self._extract_python_symbols(text) symbols_in_file = len(extracted) python_symbols += symbols_in_file else: extracted = self._extract_text_symbols(text) symbols_in_file = len(extracted) ts_symbols += symbols_in_file if symbols_in_file <= 0: continue module_counter[module_name] = int(module_counter.get(module_name, 0)) + symbols_in_file total_symbols += symbols_in_file if total_symbols >= max_symbols: break top_modules = [ {"module": name, "symbols": int(count)} for name, count in sorted(module_counter.items(), key=lambda item: item[1], reverse=True)[:10] ] return { "root": str(app_root), "indexed_files": len(file_paths), "symbol_total": int(total_symbols), "python_symbols": int(python_symbols), "typescript_symbols": int(ts_symbols), "top_modules": top_modules, "generated_at": now_cn_aware().isoformat(), } @staticmethod def _extract_python_symbols(text: str) -> list[str]: raw = str(text or "") if not raw.strip(): return [] try: tree = ast.parse(raw) except Exception: return [] out: list[str] = [] for node in tree.body: if isinstance(node, ast.ClassDef): out.append(f"class:{node.name}") elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): out.append(f"function:{node.name}") return out @staticmethod def _extract_text_symbols(text: str) -> list[str]: raw = str(text or "") if not raw.strip(): return [] out: list[str] = [] for match in TS_SYMBOL_RE.finditer(raw): name = str(match.group(1) or "").strip() if name: out.append(name) return out async def _run_single_retrieval_pass( self, *, question: str, domains: list[str], project_id: Optional[int], source_chat_id: Optional[str], resolved_entities: Optional[dict[str, Any]] = None, vector_top_k: Optional[int] = None, ) -> tuple[list[RetrievedItem], dict[str, Any]]: async def _run_db_retrieval() -> tuple[list[RetrievedItem], int, bool]: if "db" not in domains: return ([], 0, False) started_at = time.perf_counter() try: rows = await asyncio.wait_for( self._retrieve_structured_db(question, project_id=project_id), timeout=DB_QUERY_TIMEOUT_SECONDS, ) return (rows, int((time.perf_counter() - started_at) * 1000), False) except asyncio.TimeoutError: return ([], int((time.perf_counter() - started_at) * 1000), True) async def _run_vector_retrieval() -> tuple[list[RetrievedItem], int, bool]: if not any(x in domains for x in {"doc", "feishu", "pmflow", "url"}): return ([], 0, False) started_at = time.perf_counter() try: rows = await asyncio.wait_for( self._retrieve_vector_context( question, domains=domains, project_id=project_id, source_chat_id=source_chat_id, top_k=vector_top_k, ), timeout=VECTOR_SEARCH_TIMEOUT_SECONDS, ) return (rows, int((time.perf_counter() - started_at) * 1000), False) except asyncio.TimeoutError: return ([], int((time.perf_counter() - started_at) * 1000), True) async def _run_graph_retrieval() -> tuple[list[RetrievedItem], int, bool]: if "graph" not in domains: return ([], 0, False) started_at = time.perf_counter() try: rows = await asyncio.wait_for( self._retrieve_graph_context( question, project_id=project_id, resolved_entities=resolved_entities, ), timeout=GRAPH_QUERY_TIMEOUT_SECONDS, ) return (rows, int((time.perf_counter() - started_at) * 1000), False) except asyncio.TimeoutError: return ([], int((time.perf_counter() - started_at) * 1000), True) (db_rows, db_ms, db_timeout), (vec_rows, vec_ms, vec_timeout), (graph_rows, graph_ms, graph_timeout) = ( await asyncio.gather( _run_db_retrieval(), _run_vector_retrieval(), _run_graph_retrieval(), ) ) merged = [*db_rows, *vec_rows, *graph_rows] merged.sort(key=lambda x: x.score, reverse=True) return ( merged, { "db_query_ms": int(db_ms), "vector_search_ms": int(vec_ms), "graph_query_ms": int(graph_ms), "db_timeout": bool(db_timeout), "vector_timeout": bool(vec_timeout), "graph_timeout": bool(graph_timeout), }, ) async def _build_second_pass_query( self, *, question: str, project_id: Optional[int], ) -> tuple[str, dict[str, Any]]: resolved = await self.entity_resolution(question=question, project_id=project_id, limit=5) manifest = await self.get_project_manifest(project_id=project_id) hints: list[str] = [] for term in resolved.get("canonical_terms") or []: if str(term).strip(): hints.append(str(term).strip()) for row in resolved.get("matched_entities") or []: if not isinstance(row, dict): continue name = str(row.get("display_name") or "").strip() if name: hints.append(name[:30]) symbol_names: list[str] = [] try: symbol_service = SymbolIndexService(self.db, tenant_id=self.tenant_id) symbol_hits = await symbol_service.search_symbols(query=question, limit=8) for row in symbol_hits.get("items") or []: if not isinstance(row, dict): continue name = str(row.get("qualified_name") or row.get("symbol_name") or "").strip() if name: symbol_names.append(name[:60]) except Exception: symbol_names = [] for name in symbol_names[:6]: hints.append(name) feature_names: list[str] = [] feature_routes: list[str] = [] try: system_snapshot = await self.context_cache.get_system_snapshot( db=self.db, tenant_id=self.tenant_id, ) feature_hits = self._match_feature_catalog_entries( question=question, system_snapshot=system_snapshot, limit=6, ) for row in feature_hits: name = str(row.get("name") or "").strip() route = str(row.get("route") or "").strip() if name: feature_names.append(name[:50]) if route: feature_routes.append(route[:80]) except Exception: feature_names = [] feature_routes = [] for name in feature_names[:4]: hints.append(name) for route in feature_routes[:3]: hints.append(route) symbol_top = manifest.get("symbol_summary") if isinstance(manifest.get("symbol_summary"), dict) else {} modules = symbol_top.get("top_modules") if isinstance(symbol_top.get("top_modules"), list) else [] module_hint = "；".join( str(item.get("module") or "") for item in modules[:3] if isinstance(item, dict) and str(item.get("module") or "").strip() ) clean_hints = [] for item in hints: if item and item not in clean_hints: clean_hints.append(item) if not clean_hints and not module_hint: return question[:1200], {"mode": "no_hint"} revised = str(question or "").strip() if clean_hints: revised = f"{revised}。实体线索：{', '.join(clean_hints[:6])}" if module_hint: revised = f"{revised}。候选模块：{module_hint}" if symbol_names: revised = f"{revised}。候选符号：{', '.join(symbol_names[:4])}" if feature_names: revised = f"{revised}。候选功能：{', '.join(feature_names[:3])}" if feature_routes: revised = f"{revised}。候选入口：{', '.join(feature_routes[:2])}" return revised[:1200], { "mode": "second_pass_query", "hint_count": len(clean_hints), "module_hint": module_hint, "symbol_hit_count": len(symbol_names), "feature_hit_count": len(feature_names), } @staticmethod def _merge_retrieved_items( base: list[RetrievedItem], extra: list[RetrievedItem], *, limit: Optional[int] = None, ) -> list[RetrievedItem]: seen: set[str] = set() merged: list[RetrievedItem] = [] for item in [*base, *extra]: key = f"{item.source}|{item.source_label}|{item.content[:120]}" if key in seen: continue seen.add(key) merged.append(item) merged.sort(key=lambda x: x.score, reverse=True) cap = max(DEFAULT_TOP_K * 2, 8) if limit is not None: cap = max(4, min(int(limit), 80)) return merged[:cap] @staticmethod def _should_second_pass(retrieved: list[RetrievedItem]) -> bool: if not retrieved: return True top_score = float(retrieved[0].score or 0.0) if top_score < 0.55: return True return len(retrieved) < 3 @staticmethod def _merge_search_metrics(base: dict[str, Any], delta: dict[str, Any]) -> dict[str, Any]: return { "db_query_ms": int(base.get("db_query_ms", 0)) + int(delta.get("db_query_ms", 0)), "vector_search_ms": int(base.get("vector_search_ms", 0)) + int(delta.get("vector_search_ms", 0)), "graph_query_ms": int(base.get("graph_query_ms", 0)) + int(delta.get("graph_query_ms", 0)), "db_timeout": bool(base.get("db_timeout")) or bool(delta.get("db_timeout")), "vector_timeout": bool(base.get("vector_timeout")) or bool(delta.get("vector_timeout")), "graph_timeout": bool(base.get("graph_timeout")) or bool(delta.get("graph_timeout")), } @staticmethod def _summarize_retrieval_hints(rows: list[RetrievedItem], *, max_items: int = 3) -> str: hints: list[str] = [] for row in rows[: max(1, int(max_items))]: label = str(row.source_label or "").strip() if label: hints.append(label[:60]) continue content = str(row.content or "").strip() if content: hints.append(content.replace("\n", " ")[:60]) uniq: list[str] = [] for item in hints: if item and item not in uniq: uniq.append(item) return "；".join(uniq[:3]) async def _multi_step_search( self, *, question: str, domains: list[str], project_id: Optional[int], source_chat_id: Optional[str], session_state: Optional[QASessionState], max_steps_override: Optional[int] = None, vector_top_k: Optional[int] = None, ) -> tuple[list[RetrievedItem], dict[str, Any]]: resolved = await self.entity_resolution(question=question, project_id=project_id, limit=6) manifest = await self.get_project_manifest(project_id=project_id) system_snapshot = await self.context_cache.get_system_snapshot( db=self.db, tenant_id=self.tenant_id, ) symbol_hits: dict[str, Any] = {"count": 0, "items": []} try: symbol_service = SymbolIndexService(self.db, tenant_id=self.tenant_id) symbol_hits = await symbol_service.search_symbols(query=question, limit=6) except Exception: symbol_hits = {"count": 0, "items": []} feature_hits = self._match_feature_catalog_entries( question=question, system_snapshot=system_snapshot, limit=6, ) max_steps = max(1, min(int(max_steps_override or AGENT_MAX_RETRIEVAL_STEPS), 8)) tool_trace: list[dict[str, Any]] = [ { "step": 0, "tool": "entity_resolution", "matched_entities": len(resolved.get("matched_entities") or []), "canonical_terms": resolved.get("canonical_terms") if isinstance(resolved.get("canonical_terms"), list) else [], }, { "step": 0, "tool": "get_project_manifest", "symbol_total": int( ( manifest.get("symbol_summary") if isinstance(manifest.get("symbol_summary"), dict) else {} ).get("symbol_total", 0) ), "kg_entity_total": int( ( manifest.get("kg_overview") if isinstance(manifest.get("kg_overview"), dict) else {} ).get("entity_total", 0) ), }, { "step": 0, "tool": "search_symbols", "hit_count": int(symbol_hits.get("count") or 0), }, { "step": 0, "tool": "feature_catalog_lookup", "hit_count": len(feature_hits), "feature_names": [str(x.get("name") or "") for x in feature_hits[:4]], }, ] if session_state is not None and isinstance(session_state.last_entity_ids, list): anchor_ids = [int(x) for x in session_state.last_entity_ids if str(x).isdigit()][:6] else: anchor_ids = [] current_question = str(question or "").strip()[:1200] current_domains: list[str] = [] for item in domains: if item not in current_domains: current_domains.append(item) merged_rows: list[RetrievedItem] = [] merged_metrics: dict[str, Any] = { "db_query_ms": 0, "vector_search_ms": 0, "graph_query_ms": 0, "db_timeout": False, "vector_timeout": False, "graph_timeout": False, } merge_limit = max(DEFAULT_TOP_K * 2, 8) if vector_top_k is not None: merge_limit = max(merge_limit, max(2, int(vector_top_k)) * 3) for step in range(1, max_steps + 1): resolved_payload = { **resolved, "anchor_entity_ids": anchor_ids, } step_rows, step_metrics = await self._run_single_retrieval_pass( question=current_question, domains=current_domains, project_id=project_id, source_chat_id=source_chat_id, resolved_entities=resolved_payload, vector_top_k=vector_top_k, ) feature_rows = self._retrieve_feature_catalog_context( question=current_question, system_snapshot=system_snapshot, max_items=4, ) if feature_rows: step_rows = self._merge_retrieved_items(feature_rows, step_rows, limit=merge_limit) merged_rows = self._merge_retrieved_items(merged_rows, step_rows, limit=merge_limit) merged_metrics = self._merge_search_metrics(merged_metrics, step_metrics) top_score = round(float(step_rows[0].score), 4) if step_rows else 0.0 tool_trace.append( { "step": step, "tool": "parallel_retrieval", "query": current_question[:240], "domains": current_domains, "items": len(step_rows), "feature_items": len(feature_rows), "top_score": top_score, "merged_items": len(merged_rows), } ) if step >= max_steps: tool_trace.append( { "step": step, "tool": "planner_stop", "reason": "max_steps_reached", "max_steps": max_steps, } ) break if not self._should_second_pass(merged_rows): tool_trace.append( { "step": step, "tool": "planner_stop", "reason": "enough_evidence", "top_score": round(float(merged_rows[0].score), 4) if merged_rows else 0.0, } ) break next_question, rewrite_meta = await self._build_second_pass_query( question=current_question, project_id=project_id, ) row_hint = self._summarize_retrieval_hints(step_rows) if row_hint and row_hint not in next_question: next_question = f"{next_question}。已命中线索：{row_hint}" next_question = next_question[:1200] if next_question.strip() == current_question.strip(): tool_trace.append( { "step": step, "tool": "planner_stop", "reason": "query_not_changed", "rewrite_meta": rewrite_meta, } ) break next_domains: list[str] = [] for item in [*current_domains, "graph", "db", "doc", "pmflow", "feishu"]: if item not in next_domains: next_domains.append(item) tool_trace.append( { "step": step + 1, "tool": "query_refine", "rewrite_meta": rewrite_meta, "next_query": next_question[:240], "next_domains": next_domains, "anchor_entity_ids": anchor_ids, } ) current_question = next_question current_domains = next_domains result_cap = max(DEFAULT_TOP_K * 2, 8) if vector_top_k is not None: result_cap = max(result_cap, max(2, int(vector_top_k)) * 2) return merged_rows[:result_cap], { **merged_metrics, "domains_used": current_domains, "tool_trace": tool_trace, } async def multi_step_search( self, *, question: str, project_id: Optional[int] = None, source_chat_id: Optional[str] = None, session_state: Optional[QASessionState] = None, ) -> dict[str, Any]: domains = self._route_domains(question) rows, metrics = await self._multi_step_search( question=question, domains=domains, project_id=project_id, source_chat_id=source_chat_id, session_state=session_state, ) return { "question": question, "domains": metrics.get("domains_used") or domains, "items": self._dedupe_sources(rows), "retrieval_metrics": { "db_query_ms": int(metrics.get("db_query_ms") or 0), "vector_search_ms": int(metrics.get("vector_search_ms") or 0), "graph_query_ms": int(metrics.get("graph_query_ms") or 0), "db_timeout": bool(metrics.get("db_timeout")), "vector_timeout": bool(metrics.get("vector_timeout")), "graph_timeout": bool(metrics.get("graph_timeout")), }, "tool_trace": metrics.get("tool_trace") if isinstance(metrics.get("tool_trace"), list) else [], } # ------------------------------------------------------------------ # Retrieval # ------------------------------------------------------------------ def _route_domains(self, question: str) -> list[str]: text = question.lower() domains: list[str] = [] is_referential = any(k in text for k in FOLLOWUP_REFERENTIAL_KEYWORDS) if self._is_people_query(text): domains.append("db") if any(k in text for k in PROJECT_KEYWORDS): domains.append("db") domains.append("graph") if is_referential: domains.append("graph") if any(k in text for k in CODE_KEYWORDS): domains.append("graph") domains.append("doc") if any(k in text for k in FEATURE_QUERY_KEYWORDS): domains.append("doc") domains.append("graph") domains.append("pmflow") if any(k in text for k in DOC_KEYWORDS): domains.append("doc") if any(k in text for k in HISTORY_KEYWORDS): domains.append("feishu") if any(k in text for k in PMFLOW_KEYWORDS): domains.append("pmflow") if self._is_engine_process_query(text): domains.append("graph") domains.append("doc") domains.append("pmflow") domains.append("db") if URL_RE.search(text) or "根据" in text and "http" in text: domains.append("url") if not domains: # 模糊问题：并行检索所有向量域 domains.extend(["graph", "doc", "feishu", "pmflow", "url"]) # 去重并保持顺序 uniq: list[str] = [] for item in domains: if item not in uniq: uniq.append(item) return uniq async def _retrieve_structured_db(self, question: str, *, project_id: Optional[int]) -> list[RetrievedItem]: text = question.lower() rows: list[RetrievedItem] = [] ask_people = self._is_people_query(text) if ask_people: scoped_project_id: Optional[int] = None scoped_project_name: Optional[str] = None if "项目" in text or project_id is not None: scoped_project_id, scoped_project_name, _ = await self._resolve_project_scope_for_question( question=text, project_id=project_id, ) user_stmt = select(User).where( and_( User.tenant_id == self.tenant_id, User.is_active.is_(True), ) ) if scoped_project_id is not None: member_subq = select(UserProject.user_id).where( and_( UserProject.tenant_id == self.tenant_id, UserProject.project_id == int(scoped_project_id), ) ) user_stmt = user_stmt.where(User.id.in_(member_subq)) members = ( await self.db.execute( user_stmt.order_by(User.updated_at.desc(), User.id.desc()).limit(20) ) ).scalars().all() if members: preview = "；".join( f"#{int(row.id)} {str(row.name or f'User#{int(row.id)}')}（角色:{str(row.role or 'unknown')}）" for row in members[:10] ) rows.append( RetrievedItem( source="db", content=( f"成员清单（{len(members)}人）：{preview}" if scoped_project_name is None else f"项目「{scoped_project_name}」成员清单（{len(members)}人）：{preview}" ), score=0.95, source_label="User database", metadata={ "table": "users", "count": len(members), "project_id": scoped_project_id, "project_name": scoped_project_name, }, ) ) if "项目" in text and not ask_people: stmt = select(Project).where(Project.tenant_id == self.tenant_id) if project_id: stmt = stmt.where(Project.id == project_id) projects = (await self.db.execute(stmt.limit(8))).scalars().all() if projects: summary = "；".join( f"#{p.id} {p.title or p.name}（状态:{p.status}）" for p in projects ) rows.append( RetrievedItem( source="db", content=f"项目列表：{summary}", score=0.93, source_label="Project database", metadata={"table": "projects", "count": len(projects)}, ) ) if "信号" in text or "待处理" in text: signal_stmt = select(Signal).where(Signal.tenant_id == self.tenant_id) if project_id: signal_stmt = signal_stmt.where( or_( Signal.matched_project_id == project_id, Signal.content.ilike(f"%#{project_id}%"), ) ) pending = ( await self.db.execute(signal_stmt.order_by(Signal.created_at.desc()).limit(10)) ).scalars().all() if pending: preview = "；".join((s.summary or s.content or "")[:80] for s in pending[:5]) rows.append( RetrievedItem( source="db", content=f"最近信号（{len(pending)}条）：{preview}", score=0.9, source_label="Project database", metadata={"table": "signals", "count": len(pending)}, ) ) if "任务" in text or "进度" in text: task_stmt = select(Task).where(Task.tenant_id == self.tenant_id) if project_id: task_stmt = task_stmt.where(Task.project_id == project_id) tasks = (await self.db.execute(task_stmt.order_by(Task.updated_at.desc()).limit(12))).scalars().all() if tasks: open_tasks = [t for t in tasks if str(t.status or "").lower() not in {"done", "completed", "已完成"}] owners = [str(t.assignee_id) for t in open_tasks if t.assignee_id is not None] rows.append( RetrievedItem( source="db", content=( f"任务总数 {len(tasks)}，未完成 {len(open_tasks)}。" f"当前负责人ID（去重）: {', '.join(sorted(set(owners))[:8]) or '暂无'}" ), score=0.91, source_label="Project database", metadata={"table": "tasks", "open_count": len(open_tasks)}, ) ) if "宪章" in text or "charter" in text: charter_stmt = select(TruthNode).where(TruthNode.tenant_id == self.tenant_id) if project_id: charter_stmt = charter_stmt.where(TruthNode.project_id == project_id) charters = (await self.db.execute(charter_stmt.order_by(TruthNode.updated_at.desc()).limit(6))).scalars().all() if charters: rows.append( RetrievedItem( source="db", content="\n".join( f"宪章#{c.id}: {(c.title or '无标题')} / {str(c.content or '')[:200]}" for c in charters ), score=0.92, source_label="Project database", metadata={"table": "truth_nodes", "count": len(charters)}, ) ) if "工作量" in text or "负载" in text: workload_stmt = select(UserWorkload).where(UserWorkload.tenant_id == self.tenant_id) workloads = (await self.db.execute(workload_stmt.order_by(UserWorkload.updated_at.desc()).limit(10))).scalars().all() if workloads: content = "；".join( f"user#{w.user_id} 已分配 {float(w.current_assigned_hours or 0):.1f}h / " f"最大 {float(w.max_available_hours_per_week or 0):.1f}h，负载比 {float(w.workload_ratio or 0):.2f}" for w in workloads ) rows.append( RetrievedItem( source="db", content=f"团队工作量：{content}", score=0.88, source_label="Project database", metadata={"table": "user_workloads", "count": len(workloads)}, ) ) return rows[:DEFAULT_TOP_K] async def _retrieve_graph_context( self, question: str, *, project_id: Optional[int], resolved_entities: Optional[dict[str, Any]] = None, ) -> list[RetrievedItem]: try: resolution = resolved_entities or {} matched_entities = ( resolution.get("matched_entities") if isinstance(resolution.get("matched_entities"), list) else [] ) anchor_ids_raw = ( resolution.get("anchor_entity_ids") if isinstance(resolution.get("anchor_entity_ids"), list) else [] ) anchor_ids: list[int] = [] for value in anchor_ids_raw: try: anchor_ids.append(int(value)) except Exception: continue seed_entities: list[KGEntity] = [] if matched_entities: ids: list[int] = [] for row in matched_entities: if not isinstance(row, dict): continue try: ids.append(int(row.get("entity_id"))) except Exception: continue if ids: seed_entities = ( await self.db.execute( select(KGEntity).where( and_( KGEntity.tenant_id == self.tenant_id, KGEntity.id.in_(ids), ) ) ) ).scalars().all() if not seed_entities: terms = self._extract_query_terms(question, preferred=self._extract_project_lookup_keyword(question)) if terms: conds = [] for term in terms[:6]: like = f"%{term}%" conds.append(KGEntity.display_name.ilike(like)) conds.append(KGEntity.entity_key.ilike(like)) seed_entities = ( await self.db.execute( select(KGEntity) .where( and_( KGEntity.tenant_id == self.tenant_id, or_(*conds), ) ) .order_by(KGEntity.last_seen_at.desc(), KGEntity.id.desc()) .limit(15) ) ).scalars().all() if not seed_entities and anchor_ids: seed_entities = ( await self.db.execute( select(KGEntity).where( and_( KGEntity.tenant_id == self.tenant_id, KGEntity.id.in_(anchor_ids[:10]), ) ) ) ).scalars().all() if not seed_entities: return [] seed_ids = [int(x.id) for x in seed_entities] relation_rows = ( await self.db.execute( select(KGRelation) .where( and_( KGRelation.tenant_id == self.tenant_id, or_( KGRelation.subject_entity_id.in_(seed_ids), KGRelation.object_entity_id.in_(seed_ids), ), ) ) .order_by(KGRelation.weight.desc(), KGRelation.id.desc()) .limit(120) ) ).scalars().all() entity_ids: set[int] = set(seed_ids) for rel in relation_rows: entity_ids.add(int(rel.subject_entity_id)) entity_ids.add(int(rel.object_entity_id)) entities = ( await self.db.execute( select(KGEntity).where( and_( KGEntity.tenant_id == self.tenant_id, KGEntity.id.in_(list(entity_ids)), ) ) ) ).scalars().all() entity_map = {int(row.id): row for row in entities} relation_ids = [int(row.id) for row in relation_rows] evidence_rows: list[KGEvidence] = [] if relation_ids or entity_ids: evidence_rows = ( await self.db.execute( select(KGEvidence) .where( and_( KGEvidence.tenant_id == self.tenant_id, or_( KGEvidence.relation_id.in_(relation_ids or [-1]), KGEvidence.entity_id.in_(list(entity_ids) or [-1]), ), ) ) .order_by(KGEvidence.occurred_at.desc(), KGEvidence.id.desc()) .limit(220) ) ).scalars().all() evidence_by_relation: dict[int, str] = {} evidence_by_entity: dict[int, str] = {} for ev in evidence_rows: excerpt = str(ev.content_excerpt or "").strip() if not excerpt: continue if ev.relation_id is not None and int(ev.relation_id) not in evidence_by_relation: evidence_by_relation[int(ev.relation_id)] = excerpt[:320] if ev.entity_id is not None and int(ev.entity_id) not in evidence_by_entity: evidence_by_entity[int(ev.entity_id)] = excerpt[:320] out: list[RetrievedItem] = [] for rel in relation_rows: subject = entity_map.get(int(rel.subject_entity_id)) target = entity_map.get(int(rel.object_entity_id)) if subject is None or target is None: continue subject_payload = self._safe_dict(subject.payload) target_payload = self._safe_dict(target.payload) if project_id is not None: subject_pid = subject_payload.get("project_id") target_pid = target_payload.get("project_id") if subject_pid is not None and int(subject_pid) != int(project_id): if target_pid is not None and int(target_pid) != int(project_id): continue evidence_text = evidence_by_relation.get(int(rel.id)) if not evidence_text: evidence_text = evidence_by_entity.get(int(subject.id)) or evidence_by_entity.get(int(target.id)) or "" content = ( f"图路径: {str(subject.display_name or subject.entity_key)} --[{str(rel.relation_type)}]--> " f"{str(target.display_name or target.entity_key)}\n" f"证据: {evidence_text or '无显式证据摘录'}" ) weight = self._to_float(rel.weight, 1.0) score = min(0.99, max(0.45, 0.62 + min(weight, 1.5) * 0.18)) out.append( RetrievedItem( source="graph", content=content[:1200], score=score, source_label="Knowledge Graph", metadata={ "relation_id": int(rel.id), "relation_type": str(rel.relation_type or ""), "subject_entity_id": int(subject.id), "object_entity_id": int(target.id), "subject_name": str(subject.display_name or subject.entity_key), "object_name": str(target.display_name or target.entity_key), "path": f"{subject.id}->{str(rel.relation_type)}->{target.id}", }, ) ) if not out: for seed in seed_entities[:DEFAULT_TOP_K]: payload = self._safe_dict(seed.payload) if project_id is not None and payload.get("project_id") is not None: if int(payload.get("project_id")) != int(project_id): continue excerpt = evidence_by_entity.get(int(seed.id)) or "" out.append( RetrievedItem( source="graph", content=f"实体: {str(seed.display_name or seed.entity_key)}\n证据: {excerpt or '无显式证据摘录'}", score=0.58, source_label="Knowledge Graph", metadata={ "entity_id": int(seed.id), "entity_type": str(seed.entity_type or ""), "entity_key": str(seed.entity_key or ""), }, ) ) out.sort(key=lambda x: x.score, reverse=True) return out[: max(DEFAULT_TOP_K, 6)] except Exception: return [] async def _retrieve_vector_context( self, question: str, *, domains: list[str], project_id: Optional[int], source_chat_id: Optional[str], top_k: Optional[int] = None, ) -> list[RetrievedItem]: question_embedding = await self.llm.get_embedding_async( question, db=self.db, tenant_id=self.tenant_id, ) if not question_embedding: return [] source_types = self._map_domains_to_source_types(domains) if not source_types: return [] limit = max(1, min(int(top_k or DEFAULT_TOP_K), 40)) if self._is_postgresql_session(): pg_rows = await self._retrieve_vector_context_pg( question_embedding=question_embedding, source_types=source_types, project_id=project_id, source_chat_id=source_chat_id, limit=limit, ) if pg_rows is not None: return pg_rows return await self._retrieve_vector_context_legacy( question_embedding=question_embedding, source_types=source_types, project_id=project_id, source_chat_id=source_chat_id, limit=limit, ) def _is_postgresql_session(self) -> bool: try: bind = self.db.get_bind() except Exception: return False dialect_name = str(getattr(getattr(bind, "dialect", None), "name", "")).lower() return dialect_name.startswith("postgres") @staticmethod def _vector_literal(values: list[float]) -> str: normalized: list[str] = [] for raw in values: try: normalized.append(f"{float(raw):.8f}") except Exception: normalized.append("0.0") return "[" + ",".join(normalized) + "]" async def _retrieve_vector_context_pg( self, *, question_embedding: list[float], source_types: list[str], project_id: Optional[int], source_chat_id: Optional[str], limit: int, ) -> Optional[list[RetrievedItem]]: query_vec = self._vector_literal(question_embedding) scoped_project_id = int(project_id) if project_id is not None else 0 scoped_source_chat_id = str(source_chat_id or "").strip() stmt = text( """ SELECT id, source_type, content, project_id, "metadata" AS chunk_metadata, 1 - (embedding <=> CAST(:query_vec AS vector)) AS similarity FROM knowledge_base_chunks WHERE tenant_id = :tenant_id AND source_type IN :source_types AND embedding IS NOT NULL AND ( :project_scope = FALSE OR (source_type IN ('feishu', 'pmflow') AND project_id = :project_id) OR (source_type NOT IN ('feishu', 'pmflow') AND (project_id = :project_id OR project_id IS NULL)) ) AND ( :source_chat_id = '' OR source_type <> 'feishu' OR COALESCE("metadata"->>'chat_id', '') = '' OR "metadata"->>'chat_id' = :source_chat_id ) ORDER BY embedding <=> CAST(:query_vec AS vector) LIMIT :limit """ ).bindparams(bindparam("source_types", expanding=True)) try: rows = ( await self.db.execute( stmt, { "tenant_id": self.tenant_id, "source_types": list(source_types), "project_scope": project_id is not None, "project_id": scoped_project_id, "source_chat_id": scoped_source_chat_id, "query_vec": query_vec, "limit": int(limit), }, ) ).mappings().all() except Exception: return None scored: list[RetrievedItem] = [] for row in rows: source_type = str(row.get("source_type") or "doc") meta = self._safe_dict(row.get("chunk_metadata")).copy() chunk_project_id_raw = row.get("project_id") chunk_project_id = int(chunk_project_id_raw) if chunk_project_id_raw is not None else None if chunk_project_id is not None: meta.setdefault("project_id", chunk_project_id) if source_type == "pmflow": target_url = str(meta.get("target_url") or "").strip() if not target_url: target_url = self._build_pmflow_target_url( record_type=str(meta.get("record_type") or ""), record_id=meta.get("record_id"), project_id=meta.get("project_id"), ) if target_url: meta["target_url"] = target_url score = self._to_float(row.get("similarity"), 0.0) if score <= 0: continue label = self._source_label(source_type, meta) scored.append( RetrievedItem( source=source_type, content=str(row.get("content") or "")[:1200], score=score, source_label=label, metadata=meta, ) ) scored.sort(key=lambda x: x.score, reverse=True) return scored[:limit] async def _retrieve_vector_context_legacy( self, *, question_embedding: list[float], source_types: list[str], project_id: Optional[int], source_chat_id: Optional[str], limit: int, ) -> list[RetrievedItem]: stmt = select(KnowledgeBaseChunk).where( and_( KnowledgeBaseChunk.tenant_id == self.tenant_id, KnowledgeBaseChunk.source_type.in_(source_types), ) ) scan_limit = max(120, min(1200, limit * 20)) chunks = (await self.db.execute(stmt.limit(scan_limit))).scalars().all() scored: list[RetrievedItem] = [] for chunk in chunks: meta = self._safe_dict(chunk.chunk_metadata).copy() chunk_project_id = int(chunk.project_id) if chunk.project_id is not None else None if chunk_project_id is not None: meta.setdefault("project_id", chunk_project_id) if project_id is not None: if chunk.source_type in {"feishu", "pmflow"}: if chunk_project_id != int(project_id): continue elif chunk_project_id is not None and chunk_project_id != int(project_id): continue if chunk.source_type == "feishu" and source_chat_id: chat_id = str(meta.get("chat_id") or "").strip() if chat_id and chat_id != str(source_chat_id): continue if chunk.source_type == "pmflow": target_url = str(meta.get("target_url") or "").strip() if not target_url: target_url = self._build_pmflow_target_url( record_type=str(meta.get("record_type") or ""), record_id=meta.get("record_id"), project_id=meta.get("project_id"), ) if target_url: meta["target_url"] = target_url score = self._cosine_similarity(question_embedding, self._embedding_to_list(chunk.embedding)) if score <= 0: continue label = self._source_label(chunk.source_type, meta) scored.append( RetrievedItem( source=str(chunk.source_type or "doc"), content=str(chunk.content or "")[:1200], score=score, source_label=label, metadata=meta, ) ) scored.sort(key=lambda x: x.score, reverse=True) return scored[:limit] # ------------------------------------------------------------------ # Knowledge indexing # ------------------------------------------------------------------ async def create_knowledge_doc( self, *, title: str, file_type: str, raw_content: str, uploaded_by: Optional[int], source_type: str = "file", file_url: Optional[str] = None, project_id: Optional[int] = None, ) -> dict[str, Any]: doc = await self.create_knowledge_doc_placeholder( title=title, file_type=file_type, uploaded_by=uploaded_by, source_type=source_type, file_url=file_url, ) await self._index_doc_content( doc=doc, raw_content=raw_content, project_id=project_id, source_type=source_type, ) return self._serialize_doc(doc) async def create_knowledge_doc_placeholder( self, *, title: str, file_type: str, uploaded_by: Optional[int], source_type: str = "file", file_url: Optional[str] = None, ) -> KnowledgeBaseDoc: doc = KnowledgeBaseDoc( tenant_id=self.tenant_id, title=(title or "未命名文档")[:255], file_type=(file_type or "txt")[:32], source_type=(source_type or "file")[:32], file_url=(file_url or "")[:1024] or None, uploaded_by=uploaded_by, status="processing", last_fetched_at=now_cn_naive() if source_type == "url" else None, ) self.db.add(doc) await self.db.commit() await self.db.refresh(doc) return doc async def process_knowledge_doc_content( self, *, doc_id: int, raw_content: str, project_id: Optional[int] = None, ) -> dict[str, Any]: doc = ( await self.db.execute( select(KnowledgeBaseDoc).where( and_( KnowledgeBaseDoc.id == int(doc_id), KnowledgeBaseDoc.tenant_id == self.tenant_id, ) ) ) ).scalar_one_or_none() if not doc: raise ValueError("doc_not_found") await self._index_doc_content( doc=doc, raw_content=raw_content, project_id=project_id, source_type=str(doc.source_type or "file"), ) return self._serialize_doc(doc) async def _index_doc_content( self, *, doc: KnowledgeBaseDoc, raw_content: str, project_id: Optional[int], source_type: str, ) -> None: clean_content = str(raw_content or "").strip() if not clean_content: doc.status = "failed" doc.error_message = "文档内容为空，无法向量化" await self.db.commit() return try: await self.db.execute( delete(KnowledgeBaseChunk).where( and_( KnowledgeBaseChunk.tenant_id == self.tenant_id, KnowledgeBaseChunk.doc_id == int(doc.id), ) ) ) chunk_items = self._chunk_text_with_metadata(clean_content, chunk_size=500, overlap=50) for idx, (part, extra_meta) in enumerate(chunk_items): emb = await self.llm.get_embedding_async(part, db=self.db, tenant_id=self.tenant_id) chunk_metadata = { "title": doc.title, "file_type": doc.file_type, "source_type": source_type, "file_url": doc.file_url, } chunk_metadata.update(extra_meta) self.db.add( KnowledgeBaseChunk( tenant_id=self.tenant_id, doc_id=int(doc.id), source_type="url" if source_type == "url" else "doc", project_id=project_id, content=part, embedding=emb if emb else None, chunk_index=idx, chunk_metadata=chunk_metadata, ) ) doc.status = "complete" doc.error_message = None if source_type == "url": doc.last_fetched_at = now_cn_naive() await self.db.commit() except Exception as exc: doc.status = "failed" doc.error_message = str(exc)[:500] await self.db.commit() raise async def list_knowledge_docs(self) -> list[dict[str, Any]]: docs = ( await self.db.execute( select(KnowledgeBaseDoc) .where(KnowledgeBaseDoc.tenant_id == self.tenant_id) .order_by(KnowledgeBaseDoc.created_at.desc()) .limit(200) ) ).scalars().all() return [ self._serialize_doc(d) for d in docs ] async def get_knowledge_doc(self, *, doc_id: int) -> dict[str, Any]: doc = ( await self.db.execute( select(KnowledgeBaseDoc).where( and_( KnowledgeBaseDoc.id == int(doc_id), KnowledgeBaseDoc.tenant_id == self.tenant_id, ) ) ) ).scalar_one_or_none() if not doc: raise ValueError("doc_not_found") return self._serialize_doc(doc) async def delete_knowledge_doc(self, *, doc_id: int) -> bool: doc = ( await self.db.execute( select(KnowledgeBaseDoc).where( and_( KnowledgeBaseDoc.id == int(doc_id), KnowledgeBaseDoc.tenant_id == self.tenant_id, ) ) ) ).scalar_one_or_none() if not doc: return False await self.db.execute( delete(KnowledgeBaseChunk).where( and_( KnowledgeBaseChunk.doc_id == int(doc_id), KnowledgeBaseChunk.tenant_id == self.tenant_id, ) ) ) await self.db.delete(doc) await self.db.commit() return True async def ingest_feishu_message( self, *, content: str, sender: str, chat_id: str, message_id: Optional[str], captured_at: Optional[datetime] = None, project_id: Optional[int] = None, ) -> None: text = str(content or "").strip() if not text: return chunks = self._chunk_text(text, chunk_size=320, overlap=40) for idx, part in enumerate(chunks): emb = await self.llm.get_embedding_async(part, db=self.db, tenant_id=self.tenant_id) self.db.add( KnowledgeBaseChunk( tenant_id=self.tenant_id, doc_id=None, source_type="feishu", project_id=project_id, content=part, embedding=emb if emb else None, chunk_index=idx, chunk_metadata={ "sender": sender, "chat_id": chat_id, "message_id": message_id, "captured_at": (captured_at or now_cn_aware()).isoformat(), }, ) ) await self.db.commit() async def mark_doc_processing(self, *, doc_id: int) -> dict[str, Any]: doc = ( await self.db.execute( select(KnowledgeBaseDoc).where( and_( KnowledgeBaseDoc.id == int(doc_id), KnowledgeBaseDoc.tenant_id == self.tenant_id, ) ) ) ).scalar_one_or_none() if not doc: raise ValueError("doc_not_found") doc.status = "processing" doc.error_message = None await self.db.commit() await self.db.refresh(doc) return self._serialize_doc(doc) @staticmethod async def ingest_feishu_message_async( *, tenant_id: int, content: str, sender: str, chat_id: str, message_id: Optional[str], captured_at: Optional[datetime] = None, project_id: Optional[int] = None, ) -> None: async with AsyncSessionLocal() as session: service = AdaptiveRAGService(session, tenant_id=tenant_id) try: await service.ingest_feishu_message( content=content, sender=sender, chat_id=chat_id, message_id=message_id, captured_at=captured_at, project_id=project_id, ) except Exception: await session.rollback() @staticmethod async def process_knowledge_doc_content_async( *, tenant_id: int, doc_id: int, raw_content: str, project_id: Optional[int] = None, ) -> None: async with AsyncSessionLocal() as session: service = AdaptiveRAGService(session, tenant_id=tenant_id) try: await service.process_knowledge_doc_content( doc_id=doc_id, raw_content=raw_content, project_id=project_id, ) except Exception: await session.rollback() async def refresh_url_doc(self, *, doc_id: int, project_id: Optional[int] = None) -> dict[str, Any]: doc = ( await self.db.execute( select(KnowledgeBaseDoc).where( and_( KnowledgeBaseDoc.id == int(doc_id), KnowledgeBaseDoc.tenant_id == self.tenant_id, ) ) ) ).scalar_one_or_none() if not doc: raise ValueError("doc_not_found") if str(doc.source_type or "").lower() != "url": raise ValueError("doc_is_not_url_source") if not str(doc.file_url or "").strip(): raise ValueError("doc_url_missing") text = await self._fetch_url_text(str(doc.file_url)) await self._index_doc_content( doc=doc, raw_content=text, project_id=project_id, source_type="url", ) await self.db.refresh(doc) return self._serialize_doc(doc) @staticmethod async def refresh_url_doc_async( *, tenant_id: int, doc_id: int, project_id: Optional[int] = None, ) -> None: async with AsyncSessionLocal() as session: service = AdaptiveRAGService(session, tenant_id=tenant_id) try: await service.refresh_url_doc(doc_id=doc_id, project_id=project_id) except Exception: await session.rollback() async def upsert_pmflow_record( self, *, record_type: str, record_id: int, project_id: Optional[int], title: str, content: str, ) -> None: safe_type = str(record_type or "").strip().lower()[:32] safe_id = int(record_id) if not safe_type or safe_id <= 0: return stmt = select(KnowledgeBaseChunk).where( and_( KnowledgeBaseChunk.tenant_id == self.tenant_id, KnowledgeBaseChunk.source_type == "pmflow", ) ) if project_id is not None: stmt = stmt.where(KnowledgeBaseChunk.project_id == int(project_id)) existing = (await self.db.execute(stmt.limit(2000))).scalars().all() for row in existing: meta = self._safe_dict(row.chunk_metadata) if str(meta.get("record_type") or "").strip().lower() != safe_type: continue if str(meta.get("record_id") or "").strip() != str(safe_id): continue await self.db.delete(row) text = f"{title}\n\n{content}".strip() chunks = self._chunk_text(text, chunk_size=500, overlap=50) for idx, part in enumerate(chunks): emb = await self.llm.get_embedding_async(part, db=self.db, tenant_id=self.tenant_id) target_url = self._build_pmflow_target_url( record_type=safe_type, record_id=safe_id, project_id=project_id, ) self.db.add( KnowledgeBaseChunk( tenant_id=self.tenant_id, doc_id=None, source_type="pmflow", project_id=project_id, content=part, embedding=emb if emb else None, chunk_index=idx, chunk_metadata={ "record_type": safe_type, "record_id": safe_id, "project_id": int(project_id) if project_id is not None else None, "target_url": target_url, "title": title[:255], "updated_at": now_cn_aware().isoformat(), }, ) ) await self.db.commit() @staticmethod async def upsert_pmflow_record_async( *, tenant_id: int, record_type: str, record_id: int, project_id: Optional[int], title: str, content: str, ) -> None: async with AsyncSessionLocal() as session: service = AdaptiveRAGService(session, tenant_id=tenant_id) try: await service.upsert_pmflow_record( record_type=record_type, record_id=record_id, project_id=project_id, title=title, content=content, ) except Exception: await session.rollback() @staticmethod def _build_intent_correction_record( *, user_utterance: str, wrong_interpretation: str, correct_intent: str, canonical_action: str, clarification_template: str, risk_level: str, project_id: Optional[int], ) -> tuple[int, str, str, dict[str, Any]]: utterance = " ".join(str(user_utterance or "").split()).strip()[:500] wrong = " ".join(str(wrong_interpretation or "").split()).strip()[:300] correct = " ".join(str(correct_intent or "").split()).strip().lower()[:60] action = " ".join(str(canonical_action or "").split()).strip().lower()[:80] clarification = " ".join(str(clarification_template or "").split()).strip()[:500] risk = " ".join(str(risk_level or "medium").split()).strip().lower()[:20] or "medium" fingerprint = hashlib.sha1( "||".join([utterance, wrong, correct, action, clarification, risk]).encode("utf-8") ).hexdigest() record_id = int(fingerprint[:12], 16) title = f"意图纠错/{correct or 'unknown'}" content = ( f"用户原话: {utterance}\n" f"错误理解: {wrong}\n" f"正确意图: {correct}\n" f"标准动作: {action}\n" f"澄清模板: {clarification}\n" f"风险等级: {risk}" )[:4000] metadata = { "record_type": "intent_correction", "record_id": str(record_id), "project_id": int(project_id) if project_id is not None else None, "user_utterance": utterance, "wrong_interpretation": wrong, "correct_intent": correct, "canonical_action": action, "clarification_template": clarification, "risk_level": risk, "target_url": None, "title": title, "updated_at": now_cn_aware().isoformat(), } return record_id, title, content, metadata async def upsert_intent_correction_sample( self, *, user_utterance: str, wrong_interpretation: str, correct_intent: str, canonical_action: str, clarification_template: str, risk_level: str, project_id: Optional[int] = None, ) -> dict[str, Any]: record_id, title, content, metadata = self._build_intent_correction_record( user_utterance=user_utterance, wrong_interpretation=wrong_interpretation, correct_intent=correct_intent, canonical_action=canonical_action, clarification_template=clarification_template, risk_level=risk_level, project_id=project_id, ) if not metadata["user_utterance"] or not metadata["correct_intent"]: raise ValueError("intent correction sample missing required fields") stmt = select(KnowledgeBaseChunk).where( and_( KnowledgeBaseChunk.tenant_id == self.tenant_id, KnowledgeBaseChunk.source_type == "pmflow", ) ) existing = (await self.db.execute(stmt.limit(2000))).scalars().all() for row in existing: meta = self._safe_dict(row.chunk_metadata) if str(meta.get("record_type") or "").strip().lower() != "intent_correction": continue if str(meta.get("record_id") or "").strip() != str(record_id): continue await self.db.delete(row) embedding = await self.llm.get_embedding_async(content, db=self.db, tenant_id=self.tenant_id) self.db.add( KnowledgeBaseChunk( tenant_id=self.tenant_id, doc_id=None, source_type="pmflow", project_id=int(project_id) if project_id is not None else None, content=content, embedding=embedding if embedding else None, chunk_index=0, chunk_metadata=metadata, ) ) await self.db.commit() return metadata async def retrieve_intent_correction_samples( self, *, utterance: str, top_k: int = 3, project_id: Optional[int] = None, ) -> list[dict[str, Any]]: text = " ".join(str(utterance or "").split()).strip() if len(text) < 2: return [] question_embedding = await self.llm.get_embedding_async( text, db=self.db, tenant_id=self.tenant_id, ) if not question_embedding: return [] rows = ( await self.db.execute( select(KnowledgeBaseChunk).where( and_( KnowledgeBaseChunk.tenant_id == self.tenant_id, KnowledgeBaseChunk.source_type == "pmflow", ) ).limit(500) ) ).scalars().all() scored: list[dict[str, Any]] = [] for row in rows: meta = self._safe_dict(row.chunk_metadata) if str(meta.get("record_type") or "").strip().lower() != "intent_correction": continue try: chunk_project_id = int(meta.get("project_id")) if meta.get("project_id") is not None else None except Exception: chunk_project_id = None if project_id is not None and chunk_project_id is not None and chunk_project_id != int(project_id): continue score = self._cosine_similarity(question_embedding, self._embedding_to_list(row.embedding)) if score <= 0: continue scored.append( { "score": round(float(score), 4), "record_id": str(meta.get("record_id") or ""), "user_utterance": str(meta.get("user_utterance") or ""), "wrong_interpretation": str(meta.get("wrong_interpretation") or ""), "correct_intent": str(meta.get("correct_intent") or ""), "canonical_action": str(meta.get("canonical_action") or ""), "clarification_template": str(meta.get("clarification_template") or ""), "risk_level": str(meta.get("risk_level") or "medium"), "project_id": chunk_project_id, } ) scored.sort(key=lambda x: float(x.get("score") or 0.0), reverse=True) return scored[: max(1, min(int(top_k or 3), 10))] # ------------------------------------------------------------------ # QA session helpers # ------------------------------------------------------------------ @staticmethod def _to_naive_utc(value: Optional[datetime]) -> Optional[datetime]: if value is None: return None if value.tzinfo is None: return value return value.astimezone(CN_TZ).replace(tzinfo=None) @staticmethod def _normalize_session_title_raw(text: str, *, max_len: int = 30) -> str: compact = " ".join(str(text or "").replace("\n", " ").split()).strip() if not compact: return "" return compact[:max_len] @classmethod def _build_auto_session_title(cls, question: str) -> Optional[str]: normalized = cls._normalize_session_title_raw(question, max_len=30) if not normalized: return None compact = re.sub(r"\s+", "", normalized) if len(compact) < SESSION_TITLE_MIN_LEN: return None return normalized @staticmethod def _is_default_session_title(title: Optional[str]) -> bool: clean = str(title or "").strip() return (not clean) or clean == SESSION_DEFAULT_TITLE async def _touch_session_for_new_record( self, *, asked_by: Optional[int], asked_via: str, project_id: Optional[int], question: str, preferred_session_id: Optional[int] = None, force_new_session: bool = False, ) -> tuple[Optional[int], int]: if asked_by is None: return (None, 1) now_utc = now_cn_naive() chosen: Optional[QASession] = None preferred_id = int(preferred_session_id) if preferred_session_id else 0 if preferred_id > 0: chosen = ( await self.db.execute( select(QASession).where( and_( QASession.tenant_id == self.tenant_id, QASession.user_id == int(asked_by), QASession.id == preferred_id, ) ) ) ).scalar_one_or_none() # 无显式 session_id 时，仅在非强制新会话模式下复用同渠道（asked_via）且时间窗口内的最近会话。 if chosen is None and not force_new_session: latest_record = ( await self.db.execute( select(QARecord.session_id, QARecord.created_at) .where( and_( QARecord.tenant_id == self.tenant_id, QARecord.asked_by == int(asked_by), QARecord.asked_via == (asked_via or "pmflow")[:20], QARecord.session_id.is_not(None), ) ) .order_by(QARecord.created_at.desc(), QARecord.id.desc()) .limit(1) ) ).first() if latest_record and latest_record.session_id is not None: last_at = self._to_naive_utc(latest_record.created_at) if last_at is not None: gap_seconds = abs((now_utc - last_at).total_seconds()) is_same_day = now_utc.date() == last_at.date() if is_same_day and gap_seconds <= SESSION_GAP_MINUTES * 60: chosen = ( await self.db.execute( select(QASession).where( and_( QASession.tenant_id == self.tenant_id, QASession.user_id == int(asked_by), QASession.id == int(latest_record.session_id), ) ) ) ).scalar_one_or_none() if chosen is None: title = self._build_auto_session_title(question) or SESSION_DEFAULT_TITLE chosen = QASession( tenant_id=self.tenant_id, user_id=int(asked_by), title=title[:120], project_id=project_id, started_at=now_utc, last_message_at=now_utc, message_count=0, ) self.db.add(chosen) await self.db.flush() else: if chosen.project_id is None and project_id is not None: chosen.project_id = int(project_id) chosen.last_message_at = now_utc chosen.message_count = int(chosen.message_count or 0) + 1 if self._is_default_session_title(chosen.title): generated = self._build_auto_session_title(question) if generated: chosen.title = generated[:120] await self.db.flush() return (int(chosen.id), int(chosen.message_count)) # ------------------------------------------------------------------ # QA feedback/list/export # ------------------------------------------------------------------ async def _latest_feedback_map(self, record_ids: list[int]) -> dict[int, QAFeedback]: ids = [int(x) for x in record_ids if x is not None] if not ids: return {} feedback_rows = ( await self.db.execute( select(QAFeedback).where( and_( QAFeedback.tenant_id == self.tenant_id, QAFeedback.qa_id.in_(ids), ) ) ) ).scalars().all() latest_feedback: dict[int, QAFeedback] = {} for fb in feedback_rows: key = int(fb.qa_id) curr = latest_feedback.get(key) fb_at = self._to_naive_utc(fb.created_at) or datetime.min curr_at = self._to_naive_utc(curr.created_at) if curr else None if curr is None or curr_at is None or fb_at >= curr_at: latest_feedback[key] = fb return latest_feedback async def list_qa_sessions( self, *, user_id: int, date_from: Optional[datetime], date_to: Optional[datetime], page: int, page_size: int, ) -> dict[str, Any]: page = max(1, int(page)) page_size = max(1, min(100, int(page_size))) date_from_naive = self._to_naive_utc(date_from) date_to_naive = self._to_naive_utc(date_to) base_stmt = select(QASession).where( and_( QASession.tenant_id == self.tenant_id, QASession.user_id == int(user_id), ) ) if date_from_naive is not None: base_stmt = base_stmt.where(QASession.last_message_at >= date_from_naive) if date_to_naive is not None: base_stmt = base_stmt.where(QASession.last_message_at <= date_to_naive) total = int((await self.db.execute(select(func.count()).select_from(base_stmt.subquery()))).scalar() or 0) rows = ( await self.db.execute( base_stmt.order_by(QASession.last_message_at.desc(), QASession.id.desc()) .offset((page - 1) * page_size) .limit(page_size) ) ).scalars().all() if not rows: return { "items": [], "page": page, "page_size": page_size, "total": total, "total_pages": 0, "count": 0, } session_ids = [int(x.id) for x in rows] record_rows = ( await self.db.execute( select(QARecord).where( and_( QARecord.tenant_id == self.tenant_id, QARecord.session_id.in_(session_ids), ) ) ) ).scalars().all() records_by_session: dict[int, list[QARecord]] = {} for row in record_rows: sid = int(row.session_id or 0) if sid <= 0: continue records_by_session.setdefault(sid, []).append(row) feedback_map = await self._latest_feedback_map([int(x.id) for x in record_rows]) items: list[dict[str, Any]] = [] for row in rows: sid = int(row.id) session_records = sorted( records_by_session.get(sid, []), key=lambda r: (int(r.turn_index or 0), self._to_naive_utc(r.created_at) or datetime.min), ) positive = 0 negative = 0 corrected = 0 last_asked_via = "pmflow" for rec in session_records: last_asked_via = str(rec.asked_via or last_asked_via) fb = feedback_map.get(int(rec.id)) if not fb: continue rating = str(fb.rating or "").strip().lower() if rating == "positive": positive += 1 elif rating == "negative": negative += 1 if str(fb.correct_answer or "").strip(): corrected += 1 message_count = int(row.message_count or len(session_records)) unrated = max(message_count - positive - negative, 0) first_question = session_records[0].question if session_records else "" items.append( { "id": sid, "title": str(row.title or SESSION_DEFAULT_TITLE), "project_id": row.project_id, "started_at": row.started_at.isoformat() if row.started_at else None, "last_message_at": row.last_message_at.isoformat() if row.last_message_at else None, "message_count": message_count, "turn_count": message_count, "rating_summary": { "positive": positive, "negative": negative, "unrated": unrated, "corrected": corrected, }, "last_asked_via": last_asked_via, "preview_question": str(first_question or "")[:120], } ) total_pages = math.ceil(total / page_size) if total > 0 else 0 return { "items": items, "page": page, "page_size": page_size, "total": total, "total_pages": total_pages, "count": len(items), } async def get_qa_session_detail(self, *, session_id: int, user_id: int) -> dict[str, Any]: session_row = ( await self.db.execute( select(QASession).where( and_( QASession.id == int(session_id), QASession.tenant_id == self.tenant_id, QASession.user_id == int(user_id), ) ) ) ).scalar_one_or_none() if not session_row: raise ValueError("qa session 不存在") records = ( await self.db.execute( select(QARecord).where( and_( QARecord.tenant_id == self.tenant_id, QARecord.session_id == int(session_id), ) ).order_by(QARecord.turn_index.asc(), QARecord.created_at.asc()) ) ).scalars().all() feedback_map = await self._latest_feedback_map([int(x.id) for x in records]) messages: list[dict[str, Any]] = [] positive = 0 negative = 0 corrected = 0 for row in records: fb = feedback_map.get(int(row.id)) ack_meta = self._extract_ack_metadata_from_sources(row.sources, answer=row.answer) if fb: rating = str(fb.rating or "").strip().lower() if rating == "positive": positive += 1 elif rating == "negative": negative += 1 if str(fb.correct_answer or "").strip(): corrected += 1 messages.append( { "id": int(row.id), "question": row.question, "answer": row.answer, "sources": row.sources or [], "intent": row.intent, "domains_used": row.domains_used or [], "response_time_ms": row.response_time_ms, "asked_by": row.asked_by, "asked_via": row.asked_via, "project_id": row.project_id, "session_id": row.session_id, "turn_index": int(row.turn_index or 1), "created_at": row.created_at.isoformat() if row.created_at else None, "feedback": { "id": int(fb.id), "rating": fb.rating, "helpfulness": fb.helpfulness, "error_types": fb.error_types or [], "correct_answer": fb.correct_answer, } if fb else None, **ack_meta, } ) message_count = int(session_row.message_count or len(records)) unrated = max(message_count - positive - negative, 0) return { "session": { "id": int(session_row.id), "title": str(session_row.title or SESSION_DEFAULT_TITLE), "project_id": session_row.project_id, "started_at": session_row.started_at.isoformat() if session_row.started_at else None, "last_message_at": session_row.last_message_at.isoformat() if session_row.last_message_at else None, "message_count": message_count, "turn_count": message_count, "rating_summary": { "positive": positive, "negative": negative, "unrated": unrated, "corrected": corrected, }, }, "messages": messages, } async def rename_qa_session(self, *, session_id: int, user_id: int, title: str) -> dict[str, Any]: session_row = ( await self.db.execute( select(QASession).where( and_( QASession.id == int(session_id), QASession.tenant_id == self.tenant_id, QASession.user_id == int(user_id), ) ) ) ).scalar_one_or_none() if not session_row: raise ValueError("qa session 不存在") cleaned = self._normalize_session_title_raw(title, max_len=120) if not cleaned: raise ValueError("会话标题不能为空") session_row.title = cleaned await self.db.commit() return {"id": int(session_row.id), "title": cleaned} async def delete_qa_session(self, *, session_id: int, user_id: int) -> bool: session_row = ( await self.db.execute( select(QASession).where( and_( QASession.id == int(session_id), QASession.tenant_id == self.tenant_id, QASession.user_id == int(user_id), ) ) ) ).scalar_one_or_none() if not session_row: return False qa_ids = ( await self.db.execute( select(QARecord.id).where( and_( QARecord.tenant_id == self.tenant_id, QARecord.session_id == int(session_id), ) ) ) ).scalars().all() id_list = [int(x) for x in qa_ids] if id_list: await self.db.execute( delete(QAFeedback).where( and_( QAFeedback.tenant_id == self.tenant_id, QAFeedback.qa_id.in_(id_list), ) ) ) await self.db.execute( delete(QARecord).where( and_( QARecord.tenant_id == self.tenant_id, QARecord.session_id == int(session_id), ) ) ) await self.db.delete(session_row) await self.db.commit() return True async def submit_feedback( self, *, qa_id: int, rating: str, submitted_by: Optional[int], helpfulness: Optional[str] = None, error_types: Optional[list[str]] = None, correct_answer: Optional[str] = None, ) -> dict[str, Any]: qa = ( await self.db.execute( select(QARecord).where( and_(QARecord.id == int(qa_id), QARecord.tenant_id == self.tenant_id) ) ) ).scalar_one_or_none() if not qa: raise ValueError("qa record 不存在") feedback = QAFeedback( tenant_id=self.tenant_id, qa_id=int(qa_id), rating=(rating or "").strip().lower()[:16], helpfulness=(helpfulness or "").strip()[:32] or None, error_types=[str(x).strip() for x in (error_types or []) if str(x).strip()] or None, correct_answer=(correct_answer or "").strip()[:4000] or None, submitted_by=submitted_by, ) self.db.add(feedback) await self.db.commit() await self.db.refresh(feedback) if str(feedback.rating or "").strip().lower() == "negative" and ( str(feedback.correct_answer or "").strip() or [str(x).strip() for x in (feedback.error_types or []) if str(x).strip()] ): try: await self.upsert_intent_correction_sample( user_utterance=str(qa.question or ""), wrong_interpretation=str(qa.answer or ""), correct_intent="query", canonical_action="clarify_before_answer", clarification_template=str(feedback.correct_answer or "").strip() or "收到纠正后先澄清范围再回答。", risk_level="medium", project_id=int(qa.project_id) if qa.project_id is not None else None, ) except Exception: await self.db.rollback() return { "id": int(feedback.id), "qa_id": int(feedback.qa_id), "rating": feedback.rating, "helpfulness": feedback.helpfulness, "error_types": feedback.error_types or [], "correct_answer": feedback.correct_answer, } async def list_qa_records( self, *, project_id: Optional[int], date_from: Optional[datetime], date_to: Optional[datetime], rating: Optional[str], page: int, page_size: int, ) -> dict[str, Any]: page = max(1, int(page)) page_size = max(1, min(100, int(page_size))) date_from_naive = self._to_naive_utc(date_from) date_to_naive = self._to_naive_utc(date_to) stmt = select(QARecord).where(QARecord.tenant_id == self.tenant_id) if project_id is not None: stmt = stmt.where(QARecord.project_id == int(project_id)) if date_from_naive is not None: stmt = stmt.where(QARecord.created_at >= date_from_naive) if date_to_naive is not None: stmt = stmt.where(QARecord.created_at <= date_to_naive) records = ( await self.db.execute( stmt.order_by(QARecord.created_at.desc()) .offset((page - 1) * page_size) .limit(page_size) ) ).scalars().all() ids = [int(r.id) for r in records] feedback_rows = [] if ids: fb_stmt = select(QAFeedback).where( and_( QAFeedback.tenant_id == self.tenant_id, QAFeedback.qa_id.in_(ids), ) ) if rating: fb_stmt = fb_stmt.where(QAFeedback.rating == rating) feedback_rows = (await self.db.execute(fb_stmt)).scalars().all() latest_feedback: dict[int, QAFeedback] = {} for fb in feedback_rows: key = int(fb.qa_id) curr = latest_feedback.get(key) if curr is None or (fb.created_at and curr.created_at and fb.created_at > curr.created_at): latest_feedback[key] = fb items: list[dict[str, Any]] = [] for row in records: fb = latest_feedback.get(int(row.id)) if rating and not fb: continue ack_meta = self._extract_ack_metadata_from_sources(row.sources, answer=row.answer) items.append( { "id": int(row.id), "question": row.question, "answer": row.answer, "sources": row.sources or [], "intent": row.intent, "domains_used": row.domains_used or [], "response_time_ms": row.response_time_ms, "asked_by": row.asked_by, "asked_via": row.asked_via, "project_id": row.project_id, "session_id": row.session_id, "turn_index": int(row.turn_index or 1), "created_at": row.created_at.isoformat() if row.created_at else None, "feedback": { "id": int(fb.id), "rating": fb.rating, "helpfulness": fb.helpfulness, "error_types": fb.error_types or [], "correct_answer": fb.correct_answer, } if fb else None, **ack_meta, } ) return { "items": items, "page": page, "page_size": page_size, "count": len(items), } async def get_qa_stats( self, *, project_id: Optional[int], date_from: Optional[datetime], date_to: Optional[datetime], ) -> dict[str, Any]: date_from_naive = self._to_naive_utc(date_from) date_to_naive = self._to_naive_utc(date_to) record_stmt = select(QARecord).where(QARecord.tenant_id == self.tenant_id) if project_id is not None: record_stmt = record_stmt.where(QARecord.project_id == int(project_id)) if date_from_naive is not None: record_stmt = record_stmt.where(QARecord.created_at >= date_from_naive) if date_to_naive is not None: record_stmt = record_stmt.where(QARecord.created_at <= date_to_naive) records = (await self.db.execute(record_stmt)).scalars().all() if not records: return { "total_sessions": 0, "total_qa": 0, "rated_count": 0, "rated_ratio": 0.0, "average_helpfulness": 0.0, "exportable_pairs": 0, } record_ids = [int(r.id) for r in records] feedback_rows = ( await self.db.execute( select(QAFeedback).where( and_( QAFeedback.tenant_id == self.tenant_id, QAFeedback.qa_id.in_(record_ids), ) ) ) ).scalars().all() latest_feedback: dict[int, QAFeedback] = {} for fb in feedback_rows: key = int(fb.qa_id) curr = latest_feedback.get(key) fb_at = fb.created_at or datetime.min curr_at = curr.created_at if curr and curr.created_at else datetime.min if curr is None or fb_at >= curr_at: latest_feedback[key] = fb rated_count = len(latest_feedback) rated_ratio = round(rated_count / max(len(records), 1), 4) helpful_scores: list[float] = [] helpful_map = { "fully": 5.0, "could_be_better": 4.0, "partially": 3.0, } for fb in latest_feedback.values(): rating = str(fb.rating or "").strip().lower() if rating == "positive": score = helpful_map.get(str(fb.helpfulness or "").strip().lower(), 4.0) elif rating == "negative": score = 1.0 else: continue helpful_scores.append(score) avg_helpfulness = round(sum(helpful_scores) / len(helpful_scores), 2) if helpful_scores else 0.0 exportable_pairs = 0 for fb in latest_feedback.values(): rating = str(fb.rating or "").strip().lower() if rating == "positive" and str(fb.helpfulness or "").strip(): exportable_pairs += 1 continue if rating == "negative": exportable_pairs += 1 return { "total_sessions": len({int(r.session_id) for r in records if r.session_id is not None}), "total_qa": len(records), "rated_count": rated_count, "rated_ratio": rated_ratio, "average_helpfulness": avg_helpfulness, "exportable_pairs": exportable_pairs, } async def export_qa_jsonl( self, *, include_unrated: bool, project_id: Optional[int], rating: Optional[str], date_from: Optional[datetime], date_to: Optional[datetime], domain: Optional[str], ) -> str: date_from_naive = self._to_naive_utc(date_from) date_to_naive = self._to_naive_utc(date_to) record_stmt = select(QARecord).where(QARecord.tenant_id == self.tenant_id) if project_id is not None: record_stmt = record_stmt.where(QARecord.project_id == int(project_id)) if date_from_naive is not None: record_stmt = record_stmt.where(QARecord.created_at >= date_from_naive) if date_to_naive is not None: record_stmt = record_stmt.where(QARecord.created_at <= date_to_naive) if domain: record_stmt = record_stmt.where(QARecord.domains_used.is_not(None)) records = (await self.db.execute(record_stmt.order_by(QARecord.created_at.asc()))).scalars().all() if not records: return "" feedback_map = await self._latest_feedback_map([int(r.id) for r in records]) session_ids = sorted({int(r.session_id) for r in records if r.session_id is not None}) session_map: dict[int, QASession] = {} if session_ids: rows = ( await self.db.execute( select(QASession).where( and_( QASession.tenant_id == self.tenant_id, QASession.id.in_(session_ids), ) ) ) ).scalars().all() session_map = {int(x.id): x for x in rows} lines: list[str] = [] normalized_rating = str(rating or "").strip().lower() or None for record in records: feedback = feedback_map.get(int(record.id)) if normalized_rating == "unrated": if feedback: continue else: if not feedback and not include_unrated: continue if normalized_rating in {"positive", "negative"}: if not feedback or str(feedback.rating or "").strip().lower() != normalized_rating: continue if domain: domains = [str(x).lower() for x in (record.domains_used or [])] if domain.lower() not in domains: continue session = session_map.get(int(record.session_id)) if record.session_id is not None else None row = { "session_id": int(record.session_id) if record.session_id is not None else None, "session_title": str(session.title or SESSION_DEFAULT_TITLE) if session else SESSION_DEFAULT_TITLE, "turn_index": int(record.turn_index or 1), "question": record.question, "answer": record.answer, "sources": record.sources or [], "domains_used": record.domains_used or [], "asked_via": record.asked_via, "project_id": record.project_id, "created_at": record.created_at.isoformat() if record.created_at else None, # 兼容历史导出字段 "prompt": record.question, "response": record.answer, "rating": str(feedback.rating) if feedback else "unrated", } if feedback: if feedback.helpfulness: row["helpfulness"] = feedback.helpfulness errors = [str(x) for x in (feedback.error_types or []) if str(x).strip()] if errors: row["error_type"] = errors[0] row["error_types"] = errors if feedback.correct_answer: row["correct_answer"] = feedback.correct_answer lines.append(json.dumps(row, ensure_ascii=False)) return "\n".join(lines) # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ @staticmethod def _contains_any(text: str, keywords: tuple[str, ...]) -> bool: lowered = str(text or "").strip().lower() if not lowered: return False return any(keyword in lowered for keyword in keywords) @classmethod def _is_correction_query(cls, question: str) -> bool: text = str(question or "").strip().lower() if not text: return False if BRIEF_NEGATIVE_FEEDBACK_RE.match(text): return True if cls._contains_any(text, CORRECTION_CUE_KEYWORDS): return True if re.search(r"(不是|别|不要).{0,8}(系统架构|架构)", text): return True return False @classmethod def _has_explicit_target_scope(cls, question: str) -> bool: text = str(question or "").strip().lower() if not text: return False explicit_groups: tuple[tuple[str, ...], ...] = ( ENGINE_PROCESS_KEYWORDS, FEATURE_QUERY_KEYWORDS, ARCHITECTURE_KEYWORDS, PROJECT_KEYWORDS, PEOPLE_KEYWORDS, DOC_KEYWORDS, ) for group in explicit_groups: if cls._contains_any(text, group): return True if any(token in text for token in ("openclaw", "llm", "model", "engine", "api")): return True return False @classmethod def _is_engine_process_query(cls, question: str) -> bool: return cls._contains_any(question, ENGINE_PROCESS_KEYWORDS) @classmethod def _is_explicit_deep_request(cls, question: str) -> bool: return cls._contains_any(question, DEEP_EXPLICIT_KEYWORDS) @classmethod def _forced_deep_reason( cls, *, question: str, effective_question: str, conversation: list[dict[str, str]], ) -> str: if cls._is_correction_query(question) or cls._is_correction_query(effective_question): if conversation: return "correction_turn" if cls._is_engine_process_query(question) or cls._is_engine_process_query(effective_question): return "engine_process_query" return "" @classmethod def _should_force_deep_for_channel( cls, *, asked_via: str, question: str, effective_question: str, reason: str, ) -> bool: if not str(reason or "").strip(): return False channel = str(asked_via or "").strip().lower() if channel != "feishu": return True if cls._is_explicit_deep_request(question) or cls._is_explicit_deep_request(effective_question): return True if str(reason or "").strip() == "engine_process_query": combined = f"{question} {effective_question}".strip() return cls._has_explicit_target_scope(combined) return False @classmethod def _is_snapshot_only_sources(cls, sources: list[dict[str, Any]]) -> bool: if not isinstance(sources, list) or not sources: return True allowed_labels = {"System Snapshot", "User Snapshot", "Session State"} for row in sources: if not isinstance(row, dict): continue source = str(row.get("source") or "").strip().lower() label = str(row.get("label") or "").strip() if source != "system": return False if label and label not in allowed_labels: return False return True @classmethod def _should_auto_deep_retry( cls, *, question: str, effective_question: str, sources: list[dict[str, Any]], relevance_max: float, ) -> str: process_sensitive = cls._is_engine_process_query(question) or cls._is_engine_process_query(effective_question) correction_sensitive = cls._is_correction_query(question) or cls._is_correction_query(effective_question) if not process_sensitive and not correction_sensitive: return "" snapshot_only = cls._is_snapshot_only_sources(sources) if snapshot_only: return "snapshot_only" if float(relevance_max or 0.0) < 0.62: return "low_relevance" return "" @classmethod def _allow_auto_deep_retry_for_channel( cls, *, asked_via: str, question: str, effective_question: str, reason: str, ) -> bool: if not str(reason or "").strip(): return False channel = str(asked_via or "").strip().lower() if channel != "feishu": return True if cls._is_explicit_deep_request(question) or cls._is_explicit_deep_request(effective_question): return True return False @classmethod def _should_block_answer_with_guard( cls, *, question: str, effective_question: str, sources: list[dict[str, Any]], relevance_max: float, ) -> bool: if not (cls._is_engine_process_query(question) or cls._is_engine_process_query(effective_question)): return False if not cls._is_snapshot_only_sources(sources): return False return float(relevance_max or 0.0) < 0.68 @staticmethod def _build_evidence_guard_answer(question: str) -> str: raw = str(question or "").strip() target = raw[:80] if raw else "该问题" return ( f"我不能对“{target}”直接给结论：当前只命中系统快照，缺少调用链证据。" "请补充目标对象（如 openclaw 路由规则、函数名、入口层级），我会按调用链返回可核验答案。" ) @staticmethod def _is_snapshot_direct_question(question: str) -> bool: text = str(question or "").strip().lower() if not text: return False if AdaptiveRAGService._is_correction_query(text): return False if AdaptiveRAGService._is_engine_process_query(text): return False has_feature_token = any(k in text for k in FEATURE_QUERY_KEYWORDS) ask_feature_purpose = has_feature_token and any(k in text for k in FEATURE_PURPOSE_HINTS) return ( any(k in text for k in IDENTITY_KEYWORDS) or any(k in text for k in ARCHITECTURE_KEYWORDS) or ask_feature_purpose ) @staticmethod def _expand_followup_question(question: str, conversation: list[dict[str, str]]) -> str: current = str(question or "").strip() if not current: return current lowered = current.lower() # 短句不等于追问：若已包含明确业务对象，则按独立问题处理，避免串错上下文。 has_explicit_subject = any( marker in lowered for marker in ("项目", "信号", "信号源", "任务", "系统", "架构", "数据库", "redis", "接口", "用户") ) is_short_followup = len(current) <= 16 and not has_explicit_subject has_referential = any(k in lowered for k in FOLLOWUP_REFERENTIAL_KEYWORDS) if not (is_short_followup or has_referential): return current anchor_question = "" for turn in reversed(conversation[-5:]): candidate = str(turn.get("question") or "").strip() if not candidate: continue if candidate == current: continue if any( marker in candidate for marker in ("项目", "系统", "任务", "信号", "接口", "数据库", "redis", "架构", "模块") ): anchor_question = candidate break if not anchor_question: return current is_enumeration = any(k in lowered for k in FOLLOWUP_ENUMERATION_KEYWORDS) if is_enumeration and ("信号" in lowered or "信号" in anchor_question): return f"{anchor_question}。请列出该项目关联的信号标题和ID。" if is_enumeration and ("任务" in lowered or "任务" in anchor_question): return f"{anchor_question}。请列出对应任务的标题和状态。" if is_enumeration and "项目" in anchor_question: return f"{anchor_question}。请列出对应项目的名称和ID。" return f"{anchor_question}。补充问题：{current}" def _build_snapshot_direct_answer( self, question: str, *, system_snapshot: Optional[dict[str, Any]], user_snapshot: Optional[dict[str, Any]], ) -> Optional[str]: text = str(question or "").strip().lower() if not text: return None if self._is_correction_query(text): return None if self._is_engine_process_query(text): return None ask_identity = any(k in text for k in IDENTITY_KEYWORDS) ask_architecture = any(k in text for k in ARCHITECTURE_KEYWORDS) has_feature_token = any(k in text for k in FEATURE_QUERY_KEYWORDS) ask_feature = has_feature_token and any(k in text for k in FEATURE_PURPOSE_HINTS) if not ask_identity and not ask_architecture and not ask_feature: return None system_data = system_snapshot if isinstance(system_snapshot, dict) else {} assistant = system_data.get("assistant_profile") if isinstance(system_data.get("assistant_profile"), dict) else {} platform = system_data.get("platform_profile") if isinstance(system_data.get("platform_profile"), dict) else {} backend = platform.get("backend") if isinstance(platform.get("backend"), dict) else {} infra = platform.get("infrastructure") if isinstance(platform.get("infrastructure"), dict) else {} module_counts = backend.get("module_counts") if isinstance(backend.get("module_counts"), dict) else {} lines: list[str] = [] if ask_identity: name = str(assistant.get("name") or "PMFlow AI 管家") identity = str(assistant.get("identity") or "PMFlow 内置 AI 助手") lines.append(f"我是{name}，{identity}。") responsibilities = assistant.get("responsibilities") if isinstance(assistant.get("responsibilities"), list) else [] if responsibilities: joined = "；".join(str(x) for x in responsibilities[:5] if str(x).strip()) if joined: lines.append(f"职责范围：{joined}。") if ask_architecture: runtime = str(platform.get("runtime") or "unknown") backend_framework = str(backend.get("framework") or "FastAPI") orm_name = str(backend.get("orm") or "SQLAlchemy 2.0") db_name = str(infra.get("database") or "PostgreSQL") cache_name = str(infra.get("cache") or "Redis") lines.append( f"当前系统采用前后端分层架构：后端 {backend_framework} + {orm_name}，数据库 {db_name}，缓存 {cache_name}，运行环境 {runtime}。" ) if module_counts: lines.append( "后端模块规模：" f"api {int(module_counts.get('api', 0))}、" f"services {int(module_counts.get('services', 0))}、" f"models {int(module_counts.get('models', 0))}、" f"schemas {int(module_counts.get('schemas', 0))}。" ) if ask_feature: feature_hits = self._match_feature_catalog_entries( question=question, system_snapshot=system_data, limit=4, ) if feature_hits: lines.append("命中的产品功能：") for idx, row in enumerate(feature_hits, start=1): name = str(row.get("name") or "未知功能") summary = str(row.get("summary") or "暂无说明") route = str(row.get("route") or "") page_file = str(row.get("page_file") or "") apis = row.get("apis") if isinstance(row.get("apis"), list) else [] api_text = "、".join(str(x) for x in apis[:3] if str(x).strip()) line = f"{idx}. {name}：{summary}" if route: line += f" 入口 {route}。" if page_file: line += f" 页面文件 {page_file}。" if api_text: line += f" 相关接口 {api_text}。" lines.append(line) else: catalog = self._feature_catalog_from_snapshot(system_data) if catalog: brief = "、".join( str(row.get("name") or "") for row in catalog[:8] if isinstance(row, dict) and str(row.get("name") or "").strip() ) if brief: lines.append(f"当前功能目录包括：{brief}。你可以继续指定其中一个功能名称，我会返回用途与入口。") else: lines.append("当前 System Snapshot 未包含 feature_catalog，建议先刷新上下文缓存后再提问。") if ask_identity and isinstance(user_snapshot, dict) and user_snapshot: projects = user_snapshot.get("projects") if isinstance(user_snapshot.get("projects"), list) else [] tasks = user_snapshot.get("assigned_tasks") if isinstance(user_snapshot.get("assigned_tasks"), list) else [] lines.append(f"你的当前上下文：关联项目 {len(projects)} 个，未完成任务 {len(tasks)} 个。") if not lines: return None lines.append("来源: System Snapshot / User Snapshot") return "\n".join(lines)[:3200] async def _generate_answer( self, question: str, retrieved: list[RetrievedItem], conversation: list[dict[str, str]], project_id: Optional[int], *, system_snapshot: Optional[dict[str, Any]] = None, user_snapshot: Optional[dict[str, Any]] = None, relevance_max: float = 0.0, thinking_mode: str = THINKING_MODE_NORMAL, context_block_limit: Optional[int] = None, answer_max_tokens: Optional[int] = None, ) -> str: active_thinking_mode = self._normalize_thinking_mode(thinking_mode) deep_thinking = active_thinking_mode == THINKING_MODE_DEEP context_blocks: list[str] = [] used_tokens = 0 max_blocks = max(4, min(int(context_block_limit or DEFAULT_TOP_K * 2), 32)) manifest = await self.get_project_manifest(project_id=project_id) manifest_text = str(manifest.get("manifest_text") or "").strip() if manifest_text: manifest_block = f"[M] 项目画像\n{manifest_text}\n" manifest_tokens = max(len(manifest_block) // 4, 1) if used_tokens + manifest_tokens <= MAX_CONTEXT_TOKENS: context_blocks.append(manifest_block) used_tokens += manifest_tokens for idx, item in enumerate(retrieved[:max_blocks]): block = f"[{idx + 1}] 来源: {item.source_label}\n相关度: {item.score:.3f}\n内容: {item.content}\n" block_tokens = max(len(block) // 4, 1) if used_tokens + block_tokens > MAX_CONTEXT_TOKENS: break used_tokens += block_tokens context_blocks.append(block) history_lines = [] for row in conversation[-10:]: history_lines.append(f"Q: {row.get('question', '')}\nA: {row.get('answer', '')}") system_snapshot_text = json.dumps(system_snapshot or {}, ensure_ascii=False)[:12000] user_snapshot_text = json.dumps(user_snapshot or {}, ensure_ascii=False)[:6000] prompt = PROMPT_RAG_ANSWER.format( project_id=project_id if project_id is not None else "unknown", question=question, conversation=chr(10).join(history_lines) if history_lines else "无", context_blocks=chr(10).join(context_blocks) if context_blocks else "无", system_snapshot=system_snapshot_text or "{}", user_snapshot=user_snapshot_text or "{}", relevance_max=f"{max(0.0, float(relevance_max or 0.0)):.3f}", ).strip() if deep_thinking: prompt = ( f"{prompt}\n\n" "补充要求（深入思考模式）：\n" "1. 先给结论，再给证据链；\n" "2. 明确区分已验证事实与推断；\n" "3. 若存在不确定项，给出待确认清单。" ) effective_max_tokens = int( answer_max_tokens if answer_max_tokens is not None else (DEEP_ANSWER_MAX_TOKENS if deep_thinking else 900) ) effective_max_tokens = max(256, min(effective_max_tokens, 3000)) effective_temperature = 0.15 if deep_thinking else 0.2 function_name = "adaptive_rag_answer_deep" if deep_thinking else "adaptive_rag_answer_normal" answer = await self.llm.generate( prompt, db=self.db, tenant_id=self.tenant_id, function_name=function_name, temperature=effective_temperature, max_tokens=effective_max_tokens, use_prompt_profile=False, ) text = str(answer or "").strip() if not text: return "我未能从当前上下文生成有效答案，请补充问题细节后重试。" enforced = self._enforce_feature_evidence_in_answer( question=question, answer_text=text, retrieved=retrieved, system_snapshot=system_snapshot, ) return str(enforced or text)[:3200] async def _load_recent_conversation( self, *, asked_by: Optional[int], asked_via: str, project_id: Optional[int], session_id: Optional[int], external_session_id: Optional[int], limit: int, ) -> list[dict[str, str]]: active_session_id = session_id if session_id is not None else external_session_id if active_session_id is not None: cached_messages = await self.context_cache.get_session_messages( tenant_id=self.tenant_id, session_id=active_session_id, ) if cached_messages: cached_turns = self.context_cache.messages_to_turns(cached_messages) if cached_turns: return cached_turns[-limit:] if asked_by is None: return [] stmt = select(QARecord).where( and_( QARecord.tenant_id == self.tenant_id, QARecord.asked_by == int(asked_by), QARecord.asked_via == (asked_via or "pmflow")[:20], ) ) if session_id is not None: stmt = stmt.where(QARecord.session_id == int(session_id)) if project_id is not None: stmt = stmt.where(QARecord.project_id == int(project_id)) rows = (await self.db.execute(stmt.order_by(QARecord.created_at.desc()).limit(limit))).scalars().all() return [{"question": str(x.question or ""), "answer": str(x.answer or "")} for x in reversed(rows)] def _map_domains_to_source_types(self, domains: list[str]) -> list[str]: mapped: list[str] = [] if "doc" in domains: mapped.append("doc") if "feishu" in domains: mapped.append("feishu") if "pmflow" in domains: mapped.append("pmflow") if "url" in domains: mapped.append("url") return mapped def _source_label(self, source_type: str, metadata: dict[str, Any]) -> str: if source_type == "doc": title = metadata.get("title") or "unknown" page_number = metadata.get("page_number") if page_number is not None: return f"Document[{title}] p.{page_number}" return f"Document[{title}]" if source_type == "url": return f"URL[{metadata.get('file_url') or metadata.get('url') or 'external'}]" if source_type == "feishu": return "Feishu history" if source_type == "pmflow": record_type = str(metadata.get("record_type") or "").strip() record_id = metadata.get("record_id") if record_type and record_id is not None: return f"PMFlow[{record_type}#{record_id}]" return "PMFlow records" return "Knowledge base" async def _noop_results(self) -> list[RetrievedItem]: return [] async def _fetch_url_text(self, url: str) -> str: target = str(url or "").strip() if not target: raise ValueError("url_empty") async with httpx.AsyncClient(timeout=20.0, trust_env=False, follow_redirects=True) as client: resp = await client.get(target, headers={"User-Agent": "PMFlow-RAG/1.0"}) resp.raise_for_status() content_type = str(resp.headers.get("content-type") or "").lower() body = resp.text if "html" in content_type: return self._html_to_text(body) return body[:300000] @staticmethod def _html_to_text(html: str) -> str: text = str(html or "") text = re.sub(r".*?", " ", text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r".*?", " ", text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r" | ", " ", text) text = re.sub(r"&", "&", text) text = re.sub(r"<", "<", text) text = re.sub(r">", ">", text) text = re.sub(r"\s+", " ", text).strip() return text[:300000] def _chunk_text(self, text: str, *, chunk_size: int = 500, overlap: int = 50) -> list[str]: clean = str(text or "").strip() if not clean: return [] if len(clean) <= chunk_size: return [clean] chunks: list[str] = [] start = 0 step = max(1, chunk_size - overlap) while start < len(clean): end = min(len(clean), start + chunk_size) part = clean[start:end].strip() if part: chunks.append(part) if end >= len(clean): break start += step return chunks def _chunk_text_with_metadata( self, text: str, *, chunk_size: int = 500, overlap: int = 50, ) -> list[tuple[str, dict[str, Any]]]: segments = self._split_text_by_page_markers(text) if not segments: return [(part, {}) for part in self._chunk_text(text, chunk_size=chunk_size, overlap=overlap)] out: list[tuple[str, dict[str, Any]]] = [] for page_number, segment in segments: for part in self._chunk_text(segment, chunk_size=chunk_size, overlap=overlap): meta: dict[str, Any] = {} if page_number is not None: meta["page_number"] = int(page_number) out.append((part, meta)) return out def _split_text_by_page_markers(self, text: str) -> list[tuple[Optional[int], str]]: clean = str(text or "") matches = list(PAGE_MARKER_RE.finditer(clean)) if not matches: return [] segments: list[tuple[Optional[int], str]] = [] prefix = clean[: matches[0].start()].strip() if prefix: segments.append((None, prefix)) for idx, marker in enumerate(matches): try: page_number = int(marker.group(1)) except Exception: page_number = None start = marker.end() end = matches[idx + 1].start() if idx + 1 < len(matches) else len(clean) segment = clean[start:end].strip() if segment: segments.append((page_number, segment)) return segments @staticmethod def _to_float(value: Any, default: float) -> float: try: return float(value) except Exception: return float(default) @staticmethod def _safe_dict(value: Any) -> dict[str, Any]: if isinstance(value, dict): return value if isinstance(value, str): try: parsed = json.loads(value) return parsed if isinstance(parsed, dict) else {} except Exception: return {} return {} @staticmethod def _parse_json_object(raw: str) -> dict[str, Any]: text = str(raw or "").strip() if text.startswith("```"): parts = text.split("```") if len(parts) >= 2: text = parts[1].strip() if text.startswith("json"): text = text[4:].strip() parsed: dict[str, Any] = {} try: value = json.loads(text) if isinstance(value, dict): parsed = value except Exception: pass return parsed @staticmethod def _embedding_to_list(raw: Any) -> list[float]: if raw is None: return [] if isinstance(raw, list): return [float(x) for x in raw] if isinstance(raw, str): try: value = json.loads(raw) if isinstance(value, list): return [float(x) for x in value] except Exception: return [] return [] @staticmethod def _cosine_similarity(v1: list[float], v2: list[float]) -> float: if not v1 or not v2: return 0.0 size = min(len(v1), len(v2)) if size == 0: return 0.0 a = v1[:size] b = v2[:size] dot = sum(x * y for x, y in zip(a, b)) norm_a = math.sqrt(sum(x * x for x in a)) norm_b = math.sqrt(sum(y * y for y in b)) if norm_a == 0 or norm_b == 0: return 0.0 sim = dot / (norm_a * norm_b) return (sim + 1) / 2 def _dedupe_sources(self, retrieved: list[RetrievedItem]) -> list[dict[str, Any]]: seen: set[str] = set() out: list[dict[str, Any]] = [] for item in retrieved: meta_identity = ( item.metadata.get("message_id") or item.metadata.get("file_url") or item.metadata.get("target_url") or item.metadata.get("path") or item.metadata.get("relation_id") or item.metadata.get("entity_id") or item.metadata.get("subject_entity_id") or item.metadata.get("object_entity_id") or item.content[:80] ) key = f"{item.source}|{item.source_label}|{meta_identity}" if key in seen: continue seen.add(key) out.append( { "source": item.source, "label": item.source_label, "score": round(item.score, 4), "metadata": item.metadata, } ) if len(out) >= 8: break return out @staticmethod def _build_pmflow_target_url(*, record_type: str, record_id: Any, project_id: Any) -> str: safe_type = str(record_type or "").strip().lower() try: pid = int(project_id) if project_id is not None else None except Exception: pid = None try: rid = int(record_id) if record_id is not None else None except Exception: rid = None if pid is None: return "" if safe_type == "charter" and rid is not None: return f"/projects/{pid}/charter/{rid}" if safe_type == "task" and rid is not None: return f"/projects/{pid}/board?focus=task&taskId={rid}" if safe_type == "review" and rid is not None: return f"/projects/{pid}/board?focus=review&reviewId={rid}" return f"/projects/{pid}/board" @staticmethod def _serialize_doc(doc: KnowledgeBaseDoc) -> dict[str, Any]: return { "id": int(doc.id), "title": doc.title, "file_type": doc.file_type, "source_type": doc.source_type, "file_url": doc.file_url, "status": doc.status, "uploaded_by": doc.uploaded_by, "created_at": doc.created_at.isoformat() if doc.created_at else None, "last_fetched_at": doc.last_fetched_at.isoformat() if doc.last_fetched_at else None, "error_message": doc.error_message, } __all__ = ["AdaptiveRAGService"]