189 lines
10 KiB
Python
189 lines
10 KiB
Python
"""add adaptive rag tables
|
|
|
|
Revision ID: add_adaptive_rag_tables
|
|
Revises: add_feishu_queue_sstc_links
|
|
Create Date: 2026-03-02
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
|
|
try:
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
except Exception: # pragma: no cover
|
|
JSONB = sa.JSON
|
|
|
|
try:
|
|
from pgvector.sqlalchemy import Vector
|
|
except Exception: # pragma: no cover
|
|
Vector = None
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision = "add_adaptive_rag_tables"
|
|
down_revision = "add_feishu_queue_sstc_links"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def _json_type(bind):
|
|
return JSONB() if bind.dialect.name == "postgresql" else sa.JSON()
|
|
|
|
|
|
def _embedding_type(bind):
|
|
if bind.dialect.name == "postgresql" and Vector is not None:
|
|
return Vector(768)
|
|
return sa.Text()
|
|
|
|
|
|
def _table_exists(bind, table_name: str) -> bool:
|
|
inspector = sa.inspect(bind)
|
|
return inspector.has_table(table_name)
|
|
|
|
|
|
def _index_exists(bind, table_name: str, index_name: str) -> bool:
|
|
inspector = sa.inspect(bind)
|
|
indexes = inspector.get_indexes(table_name)
|
|
return any(idx.get("name") == index_name for idx in indexes)
|
|
|
|
|
|
def _create_index_if_not_exists(bind, index_name: str, table_name: str, columns: list[str]):
|
|
if not _index_exists(bind, table_name, index_name):
|
|
op.create_index(index_name, table_name, columns)
|
|
|
|
|
|
def _drop_index_if_exists(bind, index_name: str, table_name: str):
|
|
if _index_exists(bind, table_name, index_name):
|
|
op.drop_index(index_name, table_name=table_name)
|
|
|
|
|
|
def upgrade():
|
|
bind = op.get_bind()
|
|
|
|
if not _table_exists(bind, "knowledge_base_docs"):
|
|
op.create_table(
|
|
"knowledge_base_docs",
|
|
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
|
sa.Column("tenant_id", sa.BigInteger(), nullable=False, server_default="1"),
|
|
sa.Column("created_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now(), onupdate=sa.func.now()),
|
|
sa.Column("title", sa.String(length=255), nullable=False),
|
|
sa.Column("file_type", sa.String(length=32), nullable=False),
|
|
sa.Column("source_type", sa.String(length=32), nullable=False, server_default="file"),
|
|
sa.Column("file_url", sa.String(length=1024), nullable=True),
|
|
sa.Column("uploaded_by", sa.BigInteger(), nullable=True),
|
|
sa.Column("status", sa.String(length=20), nullable=False, server_default="processing"),
|
|
sa.Column("last_fetched_at", sa.TIMESTAMP(), nullable=True),
|
|
sa.Column("error_message", sa.String(length=500), nullable=True),
|
|
)
|
|
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_docs_tenant_id", "knowledge_base_docs", ["tenant_id"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_docs_created_at", "knowledge_base_docs", ["created_at"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_docs_source_type", "knowledge_base_docs", ["source_type"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_docs_status", "knowledge_base_docs", ["status"])
|
|
|
|
if not _table_exists(bind, "knowledge_base_chunks"):
|
|
op.create_table(
|
|
"knowledge_base_chunks",
|
|
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
|
sa.Column("tenant_id", sa.BigInteger(), nullable=False, server_default="1"),
|
|
sa.Column("created_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now(), onupdate=sa.func.now()),
|
|
sa.Column("doc_id", sa.Integer(), sa.ForeignKey("knowledge_base_docs.id", ondelete="CASCADE"), nullable=True),
|
|
sa.Column("source_type", sa.String(length=32), nullable=False, server_default="doc"),
|
|
sa.Column("project_id", sa.BigInteger(), nullable=True),
|
|
sa.Column("content", sa.Text(), nullable=False),
|
|
sa.Column("embedding", _embedding_type(bind), nullable=True),
|
|
sa.Column("chunk_index", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("metadata", _json_type(bind), nullable=True),
|
|
)
|
|
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_chunks_tenant_id", "knowledge_base_chunks", ["tenant_id"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_chunks_created_at", "knowledge_base_chunks", ["created_at"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_chunks_doc_id", "knowledge_base_chunks", ["doc_id"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_chunks_source_type", "knowledge_base_chunks", ["source_type"])
|
|
_create_index_if_not_exists(bind, "ix_knowledge_base_chunks_project_id", "knowledge_base_chunks", ["project_id"])
|
|
|
|
if not _table_exists(bind, "qa_records"):
|
|
op.create_table(
|
|
"qa_records",
|
|
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
|
sa.Column("tenant_id", sa.BigInteger(), nullable=False, server_default="1"),
|
|
sa.Column("created_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now(), onupdate=sa.func.now()),
|
|
sa.Column("question", sa.Text(), nullable=False),
|
|
sa.Column("answer", sa.Text(), nullable=False),
|
|
sa.Column("sources", _json_type(bind), nullable=True),
|
|
sa.Column("intent", sa.String(length=20), nullable=False, server_default="question"),
|
|
sa.Column("domains_used", _json_type(bind), nullable=True),
|
|
sa.Column("response_time_ms", sa.Integer(), nullable=True),
|
|
sa.Column("asked_by", sa.BigInteger(), nullable=True),
|
|
sa.Column("asked_via", sa.String(length=20), nullable=False, server_default="pmflow"),
|
|
sa.Column("project_id", sa.BigInteger(), nullable=True),
|
|
)
|
|
|
|
_create_index_if_not_exists(bind, "ix_qa_records_tenant_id", "qa_records", ["tenant_id"])
|
|
_create_index_if_not_exists(bind, "ix_qa_records_created_at", "qa_records", ["created_at"])
|
|
_create_index_if_not_exists(bind, "ix_qa_records_asked_by", "qa_records", ["asked_by"])
|
|
_create_index_if_not_exists(bind, "ix_qa_records_asked_via", "qa_records", ["asked_via"])
|
|
_create_index_if_not_exists(bind, "ix_qa_records_project_id", "qa_records", ["project_id"])
|
|
_create_index_if_not_exists(bind, "ix_qa_records_intent", "qa_records", ["intent"])
|
|
|
|
if not _table_exists(bind, "qa_feedback"):
|
|
op.create_table(
|
|
"qa_feedback",
|
|
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
|
sa.Column("tenant_id", sa.BigInteger(), nullable=False, server_default="1"),
|
|
sa.Column("created_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.TIMESTAMP(), nullable=False, server_default=sa.func.now(), onupdate=sa.func.now()),
|
|
sa.Column("qa_id", sa.Integer(), sa.ForeignKey("qa_records.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("rating", sa.String(length=16), nullable=False),
|
|
sa.Column("helpfulness", sa.String(length=32), nullable=True),
|
|
sa.Column("error_types", _json_type(bind), nullable=True),
|
|
sa.Column("correct_answer", sa.Text(), nullable=True),
|
|
sa.Column("submitted_by", sa.BigInteger(), nullable=True),
|
|
)
|
|
|
|
_create_index_if_not_exists(bind, "ix_qa_feedback_tenant_id", "qa_feedback", ["tenant_id"])
|
|
_create_index_if_not_exists(bind, "ix_qa_feedback_created_at", "qa_feedback", ["created_at"])
|
|
_create_index_if_not_exists(bind, "ix_qa_feedback_qa_id", "qa_feedback", ["qa_id"])
|
|
_create_index_if_not_exists(bind, "ix_qa_feedback_rating", "qa_feedback", ["rating"])
|
|
_create_index_if_not_exists(bind, "ix_qa_feedback_submitted_by", "qa_feedback", ["submitted_by"])
|
|
|
|
|
|
def downgrade():
|
|
bind = op.get_bind()
|
|
|
|
if _table_exists(bind, "qa_feedback"):
|
|
_drop_index_if_exists(bind, "ix_qa_feedback_submitted_by", "qa_feedback")
|
|
_drop_index_if_exists(bind, "ix_qa_feedback_rating", "qa_feedback")
|
|
_drop_index_if_exists(bind, "ix_qa_feedback_qa_id", "qa_feedback")
|
|
_drop_index_if_exists(bind, "ix_qa_feedback_created_at", "qa_feedback")
|
|
_drop_index_if_exists(bind, "ix_qa_feedback_tenant_id", "qa_feedback")
|
|
op.drop_table("qa_feedback")
|
|
|
|
if _table_exists(bind, "qa_records"):
|
|
_drop_index_if_exists(bind, "ix_qa_records_intent", "qa_records")
|
|
_drop_index_if_exists(bind, "ix_qa_records_project_id", "qa_records")
|
|
_drop_index_if_exists(bind, "ix_qa_records_asked_via", "qa_records")
|
|
_drop_index_if_exists(bind, "ix_qa_records_asked_by", "qa_records")
|
|
_drop_index_if_exists(bind, "ix_qa_records_created_at", "qa_records")
|
|
_drop_index_if_exists(bind, "ix_qa_records_tenant_id", "qa_records")
|
|
op.drop_table("qa_records")
|
|
|
|
if _table_exists(bind, "knowledge_base_chunks"):
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_chunks_project_id", "knowledge_base_chunks")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_chunks_source_type", "knowledge_base_chunks")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_chunks_doc_id", "knowledge_base_chunks")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_chunks_created_at", "knowledge_base_chunks")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_chunks_tenant_id", "knowledge_base_chunks")
|
|
op.drop_table("knowledge_base_chunks")
|
|
|
|
if _table_exists(bind, "knowledge_base_docs"):
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_docs_status", "knowledge_base_docs")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_docs_source_type", "knowledge_base_docs")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_docs_created_at", "knowledge_base_docs")
|
|
_drop_index_if_exists(bind, "ix_knowledge_base_docs_tenant_id", "knowledge_base_docs")
|
|
op.drop_table("knowledge_base_docs")
|