571 lines
20 KiB
Python
571 lines
20 KiB
Python
"""Adaptive RAG API。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import io
|
|
import re
|
|
from datetime import datetime
|
|
|
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response, UploadFile
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ..core.response import success_response
|
|
from ..database import get_db
|
|
from ..dependencies.auth import get_current_user, require_roles
|
|
from ..dependencies.tenant import get_tenant_id
|
|
from ..models import User
|
|
from ..schemas.rag import (
|
|
EntityResolutionRequest,
|
|
KnowledgeDocCreateRequest,
|
|
KnowledgeURLCreateRequest,
|
|
MultiStepSearchRequest,
|
|
QAExportQuery,
|
|
QAFeedbackCreateRequest,
|
|
QAListQuery,
|
|
QASessionListQuery,
|
|
QASessionRenameRequest,
|
|
RAGAskRequest,
|
|
SymbolSearchRequest,
|
|
SymbolSyncRequest,
|
|
)
|
|
from ..services.adaptive_rag_service import AdaptiveRAGService
|
|
from ..services.symbol_index_service import SymbolIndexService
|
|
|
|
router = APIRouter(prefix="/rag", tags=["Adaptive RAG"])
|
|
|
|
|
|
def _resolve_tenant_id(user: User, fallback_tenant_id: int) -> int:
|
|
try:
|
|
return int(getattr(user, "tenant_id") or fallback_tenant_id or 1)
|
|
except Exception:
|
|
return int(fallback_tenant_id or 1)
|
|
|
|
|
|
def _safe_decode_bytes(raw: bytes) -> str:
|
|
for encoding in ("utf-8", "utf-8-sig", "gb18030", "latin-1"):
|
|
try:
|
|
return raw.decode(encoding)
|
|
except Exception:
|
|
continue
|
|
return raw.decode("utf-8", errors="ignore")
|
|
|
|
|
|
def _extract_text_from_upload(filename: str, raw: bytes) -> tuple[str, str]:
|
|
name = str(filename or "uploaded.txt")
|
|
suffix = name.rsplit(".", 1)[-1].lower() if "." in name else "txt"
|
|
|
|
if suffix in {"txt", "md", "markdown", "csv", "json", "py", "ts", "tsx", "js", "jsx"}:
|
|
return ("md" if suffix in {"md", "markdown"} else "txt", _safe_decode_bytes(raw))
|
|
|
|
if suffix == "pdf":
|
|
try:
|
|
from pypdf import PdfReader # type: ignore
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"PDF 解析依赖缺失: {exc}") from exc
|
|
reader = PdfReader(io.BytesIO(raw))
|
|
parts: list[str] = []
|
|
for idx, page in enumerate(reader.pages, start=1):
|
|
page_text = (page.extract_text() or "").strip()
|
|
if not page_text:
|
|
continue
|
|
# 写入页码标记，向量分块时会保留 page_number 元数据，便于引用来源页码。
|
|
parts.append(f"[[PAGE:{idx}]]\n{page_text}")
|
|
text = "\n\n".join(parts)
|
|
return ("pdf", text)
|
|
|
|
if suffix in {"docx", "doc"}:
|
|
try:
|
|
from docx import Document # type: ignore
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Word 解析依赖缺失: {exc}") from exc
|
|
doc = Document(io.BytesIO(raw))
|
|
text = "\n".join([p.text for p in doc.paragraphs if p.text])
|
|
return ("word", text)
|
|
|
|
raise HTTPException(status_code=400, detail=f"不支持的文件类型: .{suffix}")
|
|
|
|
|
|
@router.post("/ask")
|
|
async def rag_ask(
|
|
request: RAGAskRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
result = await service.answer_question(
|
|
question=request.question,
|
|
asked_by=int(current_user.id),
|
|
asked_via=request.asked_via,
|
|
project_id=request.project_id,
|
|
source_chat_id=request.source_chat_id,
|
|
session_id=request.session_id,
|
|
force_new_session=bool(request.new_session),
|
|
thinking_mode=request.thinking_mode,
|
|
action_mode=request.action_mode,
|
|
sync_requirement=bool(request.sync_requirement),
|
|
)
|
|
return success_response(result)
|
|
|
|
|
|
@router.get("/manifest")
|
|
async def rag_manifest(
|
|
project_id: int | None = Query(default=None),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.get_project_manifest(project_id=project_id)
|
|
return success_response(data)
|
|
|
|
|
|
@router.get("/features/catalog")
|
|
async def rag_feature_catalog(
|
|
query: str | None = Query(default=None),
|
|
limit: int = Query(default=20, ge=1, le=200),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.get_feature_catalog(query=query, limit=limit)
|
|
return success_response(data)
|
|
|
|
|
|
@router.post("/entity-resolution")
|
|
async def rag_entity_resolution(
|
|
request: EntityResolutionRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.entity_resolution(
|
|
question=request.question,
|
|
project_id=request.project_id,
|
|
limit=request.limit,
|
|
)
|
|
return success_response(data)
|
|
|
|
|
|
@router.post("/multi-step-search")
|
|
async def rag_multi_step_search(
|
|
request: MultiStepSearchRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.multi_step_search(
|
|
question=request.question,
|
|
project_id=request.project_id,
|
|
source_chat_id=request.source_chat_id,
|
|
session_state=None,
|
|
)
|
|
return success_response(data)
|
|
|
|
|
|
@router.post("/symbols/sync")
|
|
async def rag_symbols_sync(
|
|
request: SymbolSyncRequest,
|
|
current_user: User = Depends(require_roles("ADMIN", "BOSS")),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = SymbolIndexService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.sync_codebase_symbols(
|
|
base_dir=request.base_dir,
|
|
max_files=request.max_files,
|
|
force_reindex=request.force_reindex,
|
|
)
|
|
return success_response(data)
|
|
|
|
|
|
@router.post("/symbols/search")
|
|
async def rag_symbols_search(
|
|
request: SymbolSearchRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = SymbolIndexService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.search_symbols(
|
|
query=request.query,
|
|
language=request.language,
|
|
symbol_type=request.symbol_type,
|
|
limit=request.limit,
|
|
)
|
|
return success_response(data)
|
|
|
|
|
|
@router.get("/symbols/overview")
|
|
async def rag_symbols_overview(
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = SymbolIndexService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
data = await service.get_overview(top_n_files=10, auto_sync_if_empty=False)
|
|
return success_response(data)
|
|
|
|
|
|
@router.get("/symbols/{symbol_id}")
|
|
async def rag_symbol_detail(
|
|
symbol_id: int,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = SymbolIndexService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
try:
|
|
data = await service.get_symbol_detail(symbol_id=symbol_id)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
return success_response(data)
|
|
|
|
|
|
@router.get("/knowledge-base/docs")
|
|
async def list_knowledge_docs(
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
_ = current_user
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
return success_response(await service.list_knowledge_docs())
|
|
|
|
|
|
@router.post("/knowledge-base/docs")
|
|
async def create_knowledge_doc(
|
|
request: KnowledgeDocCreateRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
result = await service.create_knowledge_doc(
|
|
title=request.title,
|
|
file_type=request.file_type,
|
|
raw_content=request.content,
|
|
uploaded_by=int(current_user.id),
|
|
source_type=request.source_type,
|
|
file_url=request.file_url,
|
|
project_id=request.project_id,
|
|
)
|
|
return success_response(result)
|
|
|
|
|
|
@router.post("/knowledge-base/upload")
|
|
async def upload_knowledge_doc(
|
|
file: UploadFile = File(...),
|
|
project_id: int | None = Form(default=None),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
raw = await file.read()
|
|
if not raw:
|
|
raise HTTPException(status_code=400, detail="上传文件为空")
|
|
if len(raw) > 15 * 1024 * 1024:
|
|
raise HTTPException(status_code=400, detail="文件过大，限制 15MB")
|
|
|
|
file_type, text = _extract_text_from_upload(file.filename or "uploaded.txt", raw)
|
|
if not str(text or "").strip():
|
|
raise HTTPException(status_code=400, detail="无法提取有效文本内容")
|
|
|
|
effective_tenant_id = _resolve_tenant_id(current_user, tenant_id)
|
|
service = AdaptiveRAGService(db, tenant_id=effective_tenant_id)
|
|
doc = await service.create_knowledge_doc_placeholder(
|
|
title=(file.filename or "uploaded.txt")[:255],
|
|
file_type=file_type,
|
|
uploaded_by=int(current_user.id),
|
|
source_type="file",
|
|
file_url=None,
|
|
)
|
|
asyncio.create_task(
|
|
AdaptiveRAGService.process_knowledge_doc_content_async(
|
|
tenant_id=effective_tenant_id,
|
|
doc_id=int(doc.id),
|
|
raw_content=text,
|
|
project_id=project_id,
|
|
)
|
|
)
|
|
return success_response(service._serialize_doc(doc))
|
|
|
|
|
|
@router.post("/knowledge-base/urls")
|
|
async def create_knowledge_url(
|
|
request: KnowledgeURLCreateRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
url = str(request.url or "").strip()
|
|
if not re.match(r"^https?://", url, flags=re.IGNORECASE):
|
|
raise HTTPException(status_code=400, detail="URL 必须以 http:// 或 https:// 开头")
|
|
|
|
effective_tenant_id = _resolve_tenant_id(current_user, tenant_id)
|
|
service = AdaptiveRAGService(db, tenant_id=effective_tenant_id)
|
|
doc = await service.create_knowledge_doc_placeholder(
|
|
title=(request.title or url)[:255],
|
|
file_type="url",
|
|
uploaded_by=int(current_user.id),
|
|
source_type="url",
|
|
file_url=url,
|
|
)
|
|
asyncio.create_task(
|
|
AdaptiveRAGService.refresh_url_doc_async(
|
|
tenant_id=effective_tenant_id,
|
|
doc_id=int(doc.id),
|
|
project_id=request.project_id,
|
|
)
|
|
)
|
|
return success_response(service._serialize_doc(doc))
|
|
|
|
|
|
@router.post("/knowledge-base/docs/{doc_id}/refresh")
|
|
async def refresh_knowledge_doc(
|
|
doc_id: int,
|
|
project_id: int | None = Query(default=None),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
effective_tenant_id = _resolve_tenant_id(current_user, tenant_id)
|
|
service = AdaptiveRAGService(db, tenant_id=effective_tenant_id)
|
|
try:
|
|
doc = await service.get_knowledge_doc(doc_id=doc_id)
|
|
if str(doc.get("source_type") or "").lower() != "url":
|
|
raise HTTPException(status_code=400, detail="仅 URL 类型条目支持 refresh")
|
|
marked = await service.mark_doc_processing(doc_id=doc_id)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
|
|
asyncio.create_task(
|
|
AdaptiveRAGService.refresh_url_doc_async(
|
|
tenant_id=effective_tenant_id,
|
|
doc_id=doc_id,
|
|
project_id=project_id,
|
|
)
|
|
)
|
|
return success_response(marked)
|
|
|
|
|
|
@router.delete("/knowledge-base/docs/{doc_id}")
|
|
async def delete_knowledge_doc(
|
|
doc_id: int,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
_ = current_user
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
ok = await service.delete_knowledge_doc(doc_id=doc_id)
|
|
if not ok:
|
|
raise HTTPException(status_code=404, detail="文档不存在")
|
|
return success_response({"deleted": True, "doc_id": doc_id})
|
|
|
|
|
|
@router.post("/qa-records/{qa_id}/feedback")
|
|
async def submit_qa_feedback(
|
|
qa_id: int,
|
|
request: QAFeedbackCreateRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
try:
|
|
result = await service.submit_feedback(
|
|
qa_id=qa_id,
|
|
rating=request.rating,
|
|
helpfulness=request.helpfulness,
|
|
error_types=request.error_types,
|
|
correct_answer=request.correct_answer,
|
|
submitted_by=int(current_user.id),
|
|
)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
return success_response(result)
|
|
|
|
|
|
@router.get("/qa-records")
|
|
async def list_qa_records(
|
|
project_id: int | None = Query(default=None),
|
|
date_from: datetime | None = Query(default=None),
|
|
date_to: datetime | None = Query(default=None),
|
|
rating: str | None = Query(default=None),
|
|
page: int = Query(default=1, ge=1),
|
|
page_size: int = Query(default=20, ge=1, le=100),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
_ = current_user
|
|
query = QAListQuery(
|
|
project_id=project_id,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
rating=rating,
|
|
page=page,
|
|
page_size=page_size,
|
|
)
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
result = await service.list_qa_records(
|
|
project_id=query.project_id,
|
|
date_from=query.date_from,
|
|
date_to=query.date_to,
|
|
rating=query.rating,
|
|
page=query.page,
|
|
page_size=query.page_size,
|
|
)
|
|
return success_response(result)
|
|
|
|
|
|
@router.get("/qa-sessions")
|
|
async def list_qa_sessions(
|
|
date_from: datetime | None = Query(default=None),
|
|
date_to: datetime | None = Query(default=None),
|
|
page: int = Query(default=1, ge=1),
|
|
page_size: int = Query(default=20, ge=1, le=100),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
query = QASessionListQuery(
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
page=page,
|
|
page_size=page_size,
|
|
)
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
result = await service.list_qa_sessions(
|
|
user_id=int(current_user.id),
|
|
date_from=query.date_from,
|
|
date_to=query.date_to,
|
|
page=query.page,
|
|
page_size=query.page_size,
|
|
)
|
|
return success_response(result)
|
|
|
|
|
|
@router.get("/qa-sessions/{session_id}")
|
|
async def qa_session_detail(
|
|
session_id: int,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
try:
|
|
result = await service.get_qa_session_detail(session_id=session_id, user_id=int(current_user.id))
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
return success_response(result)
|
|
|
|
|
|
@router.get("/qa-sessions/{session_id}/state")
|
|
async def qa_session_state_replay(
|
|
session_id: int,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
try:
|
|
result = await service.get_session_state_replay(
|
|
session_id=session_id,
|
|
user_id=int(current_user.id),
|
|
)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
return success_response(result)
|
|
|
|
|
|
@router.patch("/qa-sessions/{session_id}/title")
|
|
async def rename_qa_session(
|
|
session_id: int,
|
|
request: QASessionRenameRequest,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
try:
|
|
result = await service.rename_qa_session(
|
|
session_id=session_id,
|
|
user_id=int(current_user.id),
|
|
title=request.title,
|
|
)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
return success_response(result)
|
|
|
|
|
|
@router.delete("/qa-sessions/{session_id}")
|
|
async def delete_qa_session(
|
|
session_id: int,
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
deleted = await service.delete_qa_session(session_id=session_id, user_id=int(current_user.id))
|
|
if not deleted:
|
|
raise HTTPException(status_code=404, detail="qa session 不存在")
|
|
return success_response({"deleted": True, "session_id": session_id})
|
|
|
|
|
|
@router.get("/qa-records/stats")
|
|
async def qa_records_stats(
|
|
project_id: int | None = Query(default=None),
|
|
date_from: datetime | None = Query(default=None),
|
|
date_to: datetime | None = Query(default=None),
|
|
current_user: User = Depends(get_current_user),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
result = await service.get_qa_stats(
|
|
project_id=project_id,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
)
|
|
return success_response(result)
|
|
|
|
|
|
@router.get("/qa-records/export")
|
|
async def export_qa_records_jsonl(
|
|
include_unrated: bool = Query(default=False),
|
|
project_id: int | None = Query(default=None),
|
|
rating: str | None = Query(default=None),
|
|
domain: str | None = Query(default=None),
|
|
date_from: datetime | None = Query(default=None),
|
|
date_to: datetime | None = Query(default=None),
|
|
current_user: User = Depends(require_roles("ADMIN", "BOSS")),
|
|
db: AsyncSession = Depends(get_db),
|
|
tenant_id: int = Depends(get_tenant_id),
|
|
):
|
|
_ = current_user
|
|
query = QAExportQuery(
|
|
include_unrated=include_unrated,
|
|
project_id=project_id,
|
|
rating=rating,
|
|
domain=domain,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
)
|
|
|
|
service = AdaptiveRAGService(db, tenant_id=_resolve_tenant_id(current_user, tenant_id))
|
|
payload = await service.export_qa_jsonl(
|
|
include_unrated=query.include_unrated,
|
|
project_id=query.project_id,
|
|
rating=query.rating,
|
|
domain=query.domain,
|
|
date_from=query.date_from,
|
|
date_to=query.date_to,
|
|
)
|
|
return Response(content=payload, media_type="application/x-ndjson")
|