371 lines
16 KiB
Python
371 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
from .shared import *
|
|
|
|
|
|
class SemanticFactExtractMixin:
|
|
async def _create_extraction_run(
|
|
self,
|
|
*,
|
|
entity_type: str,
|
|
entity_id: int,
|
|
trigger_type: str,
|
|
input_fingerprint: str,
|
|
) -> SemanticFactExtractionRun:
|
|
run = SemanticFactExtractionRun(
|
|
tenant_id=self.tenant_id,
|
|
entity_type=_normalized_entity_type(entity_type),
|
|
entity_id=int(entity_id),
|
|
trigger_type=str(trigger_type or "").strip() or "manual_retry",
|
|
status="running",
|
|
input_fingerprint=input_fingerprint,
|
|
facts_written=0,
|
|
extract_version=SEMANTIC_FACT_EXTRACT_VERSION,
|
|
started_at=_utc_now_iso(),
|
|
)
|
|
self.db.add(run)
|
|
await self.db.flush()
|
|
return run
|
|
|
|
async def _load_project_snapshot(self, *, entity_id: int) -> Optional[dict[str, Any]]:
|
|
project = (
|
|
await self.db.execute(
|
|
select(Project).where(
|
|
and_(
|
|
Project.tenant_id == self.tenant_id,
|
|
Project.id == int(entity_id),
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if project is None:
|
|
return None
|
|
return {
|
|
"entity_type": "project",
|
|
"entity_id": int(project.id),
|
|
"display_name": str(project.title or project.name or f"项目#{int(project.id)}").strip(),
|
|
"object_status": str(project.status or "").strip(),
|
|
"updated_at": project.updated_at.isoformat() if project.updated_at else None,
|
|
"is_extractable": not _status_is_inactive(project.status),
|
|
"source_payload": {
|
|
"project_id": int(project.id),
|
|
"name": str(project.name or "").strip(),
|
|
"title": str(project.title or "").strip(),
|
|
"description": str(project.description or "").strip(),
|
|
"status": str(project.status or "").strip(),
|
|
"assigned_role": str(project.assigned_role or "").strip(),
|
|
},
|
|
}
|
|
|
|
async def _load_task_snapshot(self, *, entity_id: int) -> Optional[dict[str, Any]]:
|
|
task = (
|
|
await self.db.execute(
|
|
select(Task).where(
|
|
and_(
|
|
Task.tenant_id == self.tenant_id,
|
|
Task.id == int(entity_id),
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if task is None:
|
|
return None
|
|
return {
|
|
"entity_type": "task",
|
|
"entity_id": int(task.id),
|
|
"display_name": str(task.title or f"任务#{int(task.id)}").strip(),
|
|
"object_status": str(task.status or "").strip(),
|
|
"updated_at": task.updated_at.isoformat() if task.updated_at else None,
|
|
"is_extractable": not _status_is_inactive(task.status),
|
|
"source_payload": {
|
|
"task_id": int(task.id),
|
|
"project_id": int(task.project_id),
|
|
"truth_node_id": int(task.truth_node_id),
|
|
"assignee_id": int(task.assignee_id) if task.assignee_id is not None else None,
|
|
"title": str(task.title or "").strip(),
|
|
"description": str(task.description or "").strip(),
|
|
"status": str(task.status or "").strip(),
|
|
"priority": str(task.priority or "").strip(),
|
|
"required_skill_tag_ids": list(task.required_skill_tag_ids or []),
|
|
},
|
|
}
|
|
|
|
async def _load_signal_snapshot(self, *, entity_id: int) -> Optional[dict[str, Any]]:
|
|
signal = (
|
|
await self.db.execute(
|
|
select(Signal).where(
|
|
and_(
|
|
Signal.tenant_id == self.tenant_id,
|
|
Signal.id == int(entity_id),
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if signal is None:
|
|
return None
|
|
return {
|
|
"entity_type": "signal",
|
|
"entity_id": int(signal.id),
|
|
"display_name": str(signal.title or signal.summary or f"信号#{int(signal.id)}").strip(),
|
|
"object_status": str(signal.status or "").strip(),
|
|
"updated_at": signal.updated_at.isoformat() if signal.updated_at else None,
|
|
"is_extractable": not _status_is_inactive(signal.status),
|
|
"source_payload": {
|
|
"signal_id": int(signal.id),
|
|
"title": str(signal.title or "").strip(),
|
|
"summary": str(signal.summary or "").strip(),
|
|
"content": str(signal.content or "").strip(),
|
|
"status": str(signal.status or "").strip(),
|
|
"priority": str(signal.priority or "").strip(),
|
|
"signal_type": str(signal.signal_type or "").strip(),
|
|
"source": str(signal.source or "").strip(),
|
|
"matched_project_id": int(signal.matched_project_id) if signal.matched_project_id is not None else None,
|
|
"processed": bool(signal.processed),
|
|
},
|
|
}
|
|
|
|
async def _load_document_snapshot(self, *, entity_id: int) -> Optional[dict[str, Any]]:
|
|
doc = (
|
|
await self.db.execute(
|
|
select(KnowledgeBaseDoc).where(
|
|
and_(
|
|
KnowledgeBaseDoc.tenant_id == self.tenant_id,
|
|
KnowledgeBaseDoc.id == int(entity_id),
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if doc is None:
|
|
return None
|
|
chunk_rows = (
|
|
await self.db.execute(
|
|
select(KnowledgeBaseChunk).where(
|
|
and_(
|
|
KnowledgeBaseChunk.tenant_id == self.tenant_id,
|
|
KnowledgeBaseChunk.doc_id == int(doc.id),
|
|
)
|
|
).order_by(KnowledgeBaseChunk.chunk_index.asc()).limit(3)
|
|
)
|
|
).scalars().all()
|
|
preview_text = "\n".join(str(row.content or "").strip() for row in chunk_rows if str(row.content or "").strip())
|
|
return {
|
|
"entity_type": "document",
|
|
"entity_id": int(doc.id),
|
|
"display_name": str(doc.title or f"文档#{int(doc.id)}").strip(),
|
|
"object_status": str(doc.status or "").strip(),
|
|
"updated_at": doc.updated_at.isoformat() if doc.updated_at else None,
|
|
"is_extractable": not _status_is_inactive(doc.status),
|
|
"source_payload": {
|
|
"doc_id": int(doc.id),
|
|
"title": str(doc.title or "").strip(),
|
|
"file_type": str(doc.file_type or "").strip(),
|
|
"source_type": str(doc.source_type or "").strip(),
|
|
"file_url": str(doc.file_url or "").strip(),
|
|
"status": str(doc.status or "").strip(),
|
|
"preview_text": preview_text[:2000],
|
|
},
|
|
}
|
|
|
|
async def _load_user_snapshot(self, *, entity_id: int) -> Optional[dict[str, Any]]:
|
|
user = (
|
|
await self.db.execute(
|
|
select(User).where(
|
|
and_(
|
|
User.tenant_id == self.tenant_id,
|
|
User.id == int(entity_id),
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if user is None:
|
|
return None
|
|
return {
|
|
"entity_type": "user",
|
|
"entity_id": int(user.id),
|
|
"display_name": str(user.name or user.username or f"成员#{int(user.id)}").strip(),
|
|
"object_status": str(user.status or "").strip(),
|
|
"updated_at": user.updated_at.isoformat() if user.updated_at else None,
|
|
"is_extractable": bool(user.is_active) and not _status_is_inactive(user.status),
|
|
"source_payload": {
|
|
"user_id": int(user.id),
|
|
"username": str(user.username or "").strip(),
|
|
"name": str(user.name or "").strip(),
|
|
"role": str(user.role or "").strip(),
|
|
"status": str(user.status or "").strip(),
|
|
"is_active": bool(user.is_active),
|
|
"bio": str(user.bio or "").strip(),
|
|
"assigned_domains": list(user.assigned_domains or []),
|
|
},
|
|
}
|
|
|
|
async def _load_truth_node_snapshot(self, *, entity_type: str, entity_id: int) -> Optional[dict[str, Any]]:
|
|
truth_node_type = ENTITY_TYPE_TO_TRUTH_NODE_TYPE.get(_normalized_entity_type(entity_type))
|
|
if not truth_node_type:
|
|
return None
|
|
node = (
|
|
await self.db.execute(
|
|
select(TruthNode).where(
|
|
and_(
|
|
TruthNode.tenant_id == self.tenant_id,
|
|
TruthNode.id == int(entity_id),
|
|
TruthNode.node_type == truth_node_type,
|
|
)
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if node is None:
|
|
return None
|
|
normalized_entity_type = TRUTH_NODE_TYPE_TO_ENTITY_TYPE.get(str(node.node_type or "").strip().lower(), "")
|
|
if not normalized_entity_type:
|
|
return None
|
|
return {
|
|
"entity_type": normalized_entity_type,
|
|
"entity_id": int(node.id),
|
|
"display_name": str(node.title or f"{normalized_entity_type}#{int(node.id)}").strip(),
|
|
"object_status": str(node.status or "").strip(),
|
|
"updated_at": node.updated_at.isoformat() if node.updated_at else None,
|
|
"is_extractable": not _status_is_inactive(node.status),
|
|
"source_payload": {
|
|
"node_id": int(node.id),
|
|
"node_type": str(node.node_type or "").strip(),
|
|
"project_id": int(node.project_id) if node.project_id is not None else None,
|
|
"signal_id": int(node.signal_id) if node.signal_id is not None else None,
|
|
"title": str(node.title or "").strip(),
|
|
"status": str(node.status or "").strip(),
|
|
"version": int(node.version or 1),
|
|
"content": node.content if isinstance(node.content, dict) else {},
|
|
},
|
|
}
|
|
|
|
async def _load_entity_snapshot(
|
|
self,
|
|
*,
|
|
entity_type: str,
|
|
entity_id: int,
|
|
) -> Optional[dict[str, Any]]:
|
|
normalized_entity_type = _normalized_entity_type(entity_type)
|
|
if normalized_entity_type == "project":
|
|
return await self._load_project_snapshot(entity_id=int(entity_id))
|
|
if normalized_entity_type == "task":
|
|
return await self._load_task_snapshot(entity_id=int(entity_id))
|
|
if normalized_entity_type == "signal":
|
|
return await self._load_signal_snapshot(entity_id=int(entity_id))
|
|
if normalized_entity_type == "document":
|
|
return await self._load_document_snapshot(entity_id=int(entity_id))
|
|
if normalized_entity_type == "user":
|
|
return await self._load_user_snapshot(entity_id=int(entity_id))
|
|
if normalized_entity_type in {"requirement", "release", "review"}:
|
|
return await self._load_truth_node_snapshot(entity_type=normalized_entity_type, entity_id=int(entity_id))
|
|
raise ValueError(f"unsupported_semantic_fact_entity_type:{normalized_entity_type}")
|
|
|
|
def _render_extraction_prompt(self, *, entity_type: str, source_payload: dict[str, Any]) -> str:
|
|
normalized_entity_type = _normalized_entity_type(entity_type)
|
|
prompt_template = SEMANTIC_FACT_PROMPTS.get(normalized_entity_type)
|
|
if prompt_template is None:
|
|
raise ValueError(f"unsupported_semantic_fact_prompt:{normalized_entity_type}")
|
|
payload_text = json.dumps(source_payload, ensure_ascii=False, sort_keys=True)
|
|
if normalized_entity_type == "project":
|
|
return prompt_template.replace("<<project_payload>>", payload_text).strip()
|
|
return prompt_template.replace("<<entity_payload>>", payload_text).strip()
|
|
|
|
async def extract_entity_facts(
|
|
self,
|
|
*,
|
|
entity_type: str,
|
|
entity_id: int,
|
|
trigger_type: str,
|
|
) -> Optional[SemanticFactExtractionRun]:
|
|
from ..llm_service import llm_service
|
|
|
|
normalized_entity_type = _normalized_entity_type(entity_type)
|
|
snapshot = await self._load_entity_snapshot(
|
|
entity_type=normalized_entity_type,
|
|
entity_id=int(entity_id),
|
|
)
|
|
if snapshot is None:
|
|
await self.deactivate_entity_facts(entity_type=normalized_entity_type, entity_id=int(entity_id))
|
|
return None
|
|
if not bool(snapshot.get("is_extractable")):
|
|
await self.deactivate_entity_facts(entity_type=normalized_entity_type, entity_id=int(entity_id))
|
|
return None
|
|
|
|
source_payload = dict(snapshot.get("source_payload") or {})
|
|
fingerprint = hashlib.sha1(
|
|
json.dumps(source_payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
).hexdigest()
|
|
run = await self._create_extraction_run(
|
|
entity_type=normalized_entity_type,
|
|
entity_id=int(entity_id),
|
|
trigger_type=trigger_type,
|
|
input_fingerprint=fingerprint,
|
|
)
|
|
try:
|
|
prompt = self._render_extraction_prompt(
|
|
entity_type=normalized_entity_type,
|
|
source_payload=source_payload,
|
|
)
|
|
extraction = await llm_service.generate_structured_object(
|
|
prompt,
|
|
db=self.db,
|
|
tenant_id=self.tenant_id,
|
|
function_name=SEMANTIC_FACT_FUNCTION_NAMES[normalized_entity_type],
|
|
temperature=0.0,
|
|
max_tokens=1200,
|
|
)
|
|
payload = extraction.payload if isinstance(extraction.payload, dict) else {}
|
|
fact_rows = list(payload.get("facts") or [])
|
|
await self.deactivate_entity_facts(
|
|
entity_type=normalized_entity_type,
|
|
entity_id=int(entity_id),
|
|
)
|
|
written = 0
|
|
for item in fact_rows:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
fact_key = str(item.get("fact_key") or "").strip()
|
|
fact_value_text = str(item.get("fact_value_text") or "").strip()
|
|
if not fact_key or not fact_value_text:
|
|
continue
|
|
await self.upsert_fact(
|
|
entity_type=normalized_entity_type,
|
|
entity_id=int(entity_id),
|
|
fact_namespace=str(item.get("fact_namespace") or "general").strip() or "general",
|
|
fact_key=fact_key,
|
|
fact_value_text=fact_value_text,
|
|
fact_value_norm=str(item.get("fact_value_norm") or "").strip() or None,
|
|
value_type=str(item.get("value_type") or "keyword").strip() or "keyword",
|
|
confidence=float(item.get("confidence") or 0.0),
|
|
evidence_source_type=normalized_entity_type,
|
|
evidence_source_id=int(entity_id),
|
|
evidence_excerpt=str(item.get("evidence_excerpt") or "").strip() or None,
|
|
extract_version=SEMANTIC_FACT_EXTRACT_VERSION,
|
|
freshness_at=str(snapshot.get("updated_at") or "").strip() or None,
|
|
)
|
|
written += 1
|
|
|
|
run.status = "completed"
|
|
run.facts_written = written
|
|
run.failure_reason = None
|
|
run.finished_at = _utc_now_iso()
|
|
await self.db.flush()
|
|
return run
|
|
except Exception as exc:
|
|
run.status = "failed"
|
|
run.failure_reason = str(exc)[:1000]
|
|
run.finished_at = _utc_now_iso()
|
|
await self.db.flush()
|
|
raise
|
|
|
|
async def extract_project_facts(self, *, project_id: int, trigger_type: str) -> SemanticFactExtractionRun:
|
|
run = await self.extract_entity_facts(
|
|
entity_type="project",
|
|
entity_id=int(project_id),
|
|
trigger_type=trigger_type,
|
|
)
|
|
if run is None:
|
|
raise ValueError(f"project_not_found:{int(project_id)}")
|
|
return run
|
|
|
|
|
|
__all__ = ["SemanticFactExtractMixin"]
|