560 lines
22 KiB
Python
560 lines
22 KiB
Python
"""Feishu 长连接事件接收管理器。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from datetime import datetime, timezone
|
||
import importlib.util
|
||
import json
|
||
import os
|
||
import threading
|
||
import time
|
||
from typing import Any, Optional
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||
from sqlalchemy.future import select
|
||
|
||
from ..core.logging import get_logger
|
||
from ..models import Integration
|
||
from .feishu_integration import FeishuIntegrationService
|
||
|
||
from ..utils.timezone import CN_TZ, now_cn_aware
|
||
logger = get_logger(__name__)
|
||
DEFAULT_FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS = 3
|
||
DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS = 1.0
|
||
|
||
|
||
class FeishuLongConnectionManager:
|
||
"""管理飞书长连接 SDK 生命周期与事件消费。"""
|
||
|
||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||
self._session_factory = session_factory
|
||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||
self._queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
||
self._worker_task: Optional[asyncio.Task[None]] = None
|
||
self._supervisor_task: Optional[asyncio.Task[None]] = None
|
||
self._thread: Optional[threading.Thread] = None
|
||
self._client: Any = None
|
||
self._boot_lock = asyncio.Lock()
|
||
|
||
self._running = False
|
||
self._sdk_available = self._check_sdk_available()
|
||
self._configured = False
|
||
self._event_mode = "long_connection"
|
||
self._tenant_id = 1
|
||
self._last_error: Optional[str] = None
|
||
self._last_error_code: Optional[str] = None
|
||
self._last_error_retryable = False
|
||
self._last_event_at: Optional[str] = None
|
||
self._last_auth_at: Optional[str] = None
|
||
self._last_connect_at: Optional[str] = None
|
||
self._last_disconnect_at: Optional[str] = None
|
||
self._restart_attempts = 0
|
||
self._next_retry_at = 0.0
|
||
self._stopping = False
|
||
self._max_retry_attempts = self._read_retry_max_attempts()
|
||
self._retry_base_seconds = self._read_retry_base_seconds()
|
||
|
||
async def start(self) -> None:
|
||
"""启动长连接（若未配置或未安装 SDK 则仅记录状态）。"""
|
||
if self._running:
|
||
return
|
||
|
||
self._stopping = False
|
||
self._running = True
|
||
self._loop = asyncio.get_running_loop()
|
||
self._worker_task = asyncio.create_task(
|
||
self._event_worker(),
|
||
name="feishu-long-connection-worker",
|
||
)
|
||
self._supervisor_task = asyncio.create_task(
|
||
self._connection_supervisor(),
|
||
name="feishu-long-connection-supervisor",
|
||
)
|
||
|
||
started = await self._ensure_connection(reason="startup")
|
||
if started:
|
||
logger.info("feishu_long_connection_started", tenant_id=self._tenant_id, event_mode=self._event_mode)
|
||
|
||
async def stop(self) -> None:
|
||
"""停止长连接。"""
|
||
self._stopping = True
|
||
self._running = False
|
||
|
||
current_client = self._client
|
||
if current_client is not None:
|
||
try:
|
||
await asyncio.to_thread(self._stop_client_blocking, current_client)
|
||
except Exception as exc: # pragma: no cover
|
||
logger.warning(f"feishu_long_connection_stop_failed: {exc}")
|
||
|
||
self._client = None
|
||
|
||
if self._supervisor_task:
|
||
self._supervisor_task.cancel()
|
||
try:
|
||
await self._supervisor_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
self._supervisor_task = None
|
||
|
||
if self._worker_task:
|
||
self._worker_task.cancel()
|
||
try:
|
||
await self._worker_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
self._worker_task = None
|
||
|
||
if self._thread and self._thread.is_alive():
|
||
await asyncio.to_thread(self._thread.join, 3)
|
||
self._thread = None
|
||
|
||
while not self._queue.empty():
|
||
try:
|
||
self._queue.get_nowait()
|
||
self._queue.task_done()
|
||
except asyncio.QueueEmpty:
|
||
break
|
||
|
||
logger.info("feishu_long_connection_stopped")
|
||
self._stopping = False
|
||
|
||
async def restart(self) -> None:
|
||
"""重启长连接（用于配置变更后重连）。"""
|
||
await self.stop()
|
||
await self.start()
|
||
|
||
def status(self) -> dict[str, Any]:
|
||
"""返回当前运行状态，供 API 展示。"""
|
||
return {
|
||
"running": self._running,
|
||
"tenant_id": self._tenant_id,
|
||
"event_mode": self._event_mode,
|
||
"sdk_available": self._sdk_available,
|
||
"configured": self._configured,
|
||
"worker_alive": bool(self._worker_task and not self._worker_task.done()),
|
||
"supervisor_alive": bool(self._supervisor_task and not self._supervisor_task.done()),
|
||
"thread_alive": bool(self._thread and self._thread.is_alive()),
|
||
"last_error": self._last_error,
|
||
"last_error_code": self._last_error_code,
|
||
"last_error_retryable": self._last_error_retryable,
|
||
"last_event_at": self._last_event_at,
|
||
"last_auth_at": self._last_auth_at,
|
||
"last_connect_at": self._last_connect_at,
|
||
"last_disconnect_at": self._last_disconnect_at,
|
||
"restart_attempts": self._restart_attempts,
|
||
"next_retry_at": (
|
||
datetime.fromtimestamp(self._next_retry_at, tz=CN_TZ).isoformat()
|
||
if self._next_retry_at > 0
|
||
else None
|
||
),
|
||
"retry_policy": {
|
||
"max_attempts": self._max_retry_attempts,
|
||
"base_seconds": self._retry_base_seconds,
|
||
},
|
||
}
|
||
|
||
def _check_sdk_available(self) -> bool:
|
||
# 只做可用性探测，避免在主线程提前 import lark_oapi，
|
||
# 否则 SDK 会把全局 event loop 绑定到 uvicorn loop，导致线程内 run_until_complete 报错。
|
||
return importlib.util.find_spec("lark_oapi") is not None
|
||
|
||
@staticmethod
|
||
def _read_retry_max_attempts() -> int:
|
||
raw = os.getenv(
|
||
"FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS",
|
||
str(DEFAULT_FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS),
|
||
)
|
||
try:
|
||
parsed = int(str(raw).strip())
|
||
except Exception:
|
||
parsed = DEFAULT_FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS
|
||
# 强制最多 3 次，避免无限重连雪崩。
|
||
return min(3, max(1, parsed))
|
||
|
||
@staticmethod
|
||
def _read_retry_base_seconds() -> float:
|
||
raw = os.getenv(
|
||
"FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS",
|
||
str(DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS),
|
||
)
|
||
try:
|
||
parsed = float(str(raw).strip())
|
||
except Exception:
|
||
parsed = DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS
|
||
if parsed <= 0:
|
||
parsed = DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS
|
||
return parsed
|
||
|
||
@staticmethod
|
||
def _stop_client_blocking(client: Any) -> None:
|
||
stop_fn = getattr(client, "stop", None)
|
||
if callable(stop_fn):
|
||
stop_fn()
|
||
|
||
async def _load_active_config(self) -> Optional[dict[str, Any]]:
|
||
async with self._session_factory() as db:
|
||
result = await db.execute(
|
||
select(Integration).where(
|
||
Integration.type == "feishu",
|
||
Integration.is_active.is_(True),
|
||
).order_by(Integration.updated_at.desc(), Integration.id.desc())
|
||
)
|
||
integration = result.scalars().first()
|
||
if not integration:
|
||
return None
|
||
|
||
config = integration.config
|
||
parsed: dict[str, Any]
|
||
if isinstance(config, str):
|
||
try:
|
||
parsed = json.loads(config) if config else {}
|
||
except Exception:
|
||
parsed = {}
|
||
else:
|
||
parsed = config or {}
|
||
parsed["tenant_id"] = int(getattr(integration, "tenant_id", 1) or 1)
|
||
return parsed
|
||
|
||
async def _ensure_connection(self, *, reason: str) -> bool:
|
||
async with self._boot_lock:
|
||
if not self._running:
|
||
return False
|
||
if self._thread and self._thread.is_alive():
|
||
return True
|
||
if self._client is not None:
|
||
try:
|
||
await asyncio.to_thread(self._stop_client_blocking, self._client)
|
||
except Exception as exc: # pragma: no cover
|
||
logger.warning(f"feishu_long_connection_stale_client_stop_failed: {exc}")
|
||
finally:
|
||
self._client = None
|
||
|
||
config = await self._load_active_config()
|
||
self._event_mode = "long_connection"
|
||
self._tenant_id = int((config or {}).get("tenant_id") or 1)
|
||
app_id = str((config or {}).get("app_id") or "").strip()
|
||
app_secret = str((config or {}).get("app_secret") or "").strip()
|
||
self._configured = bool(app_id and app_secret)
|
||
|
||
if not self._configured:
|
||
self._clear_error()
|
||
logger.info("feishu_long_connection_not_configured")
|
||
return False
|
||
|
||
if not self._sdk_available:
|
||
self._set_error("lark_oapi not installed", code="sdk_missing", retryable=False)
|
||
logger.warning("feishu_long_connection_sdk_missing: pip install lark-oapi")
|
||
return False
|
||
|
||
try:
|
||
await self._preflight_auth(tenant_id=self._tenant_id)
|
||
self._last_auth_at = now_cn_aware().isoformat()
|
||
except Exception as exc:
|
||
code, retryable = self._classify_error(exc)
|
||
self._set_error(str(exc), code=code, retryable=retryable)
|
||
if retryable:
|
||
self._schedule_retry()
|
||
logger.warning(
|
||
"feishu_long_connection_preflight_failed",
|
||
tenant_id=self._tenant_id,
|
||
error_code=code,
|
||
retryable=retryable,
|
||
reason=reason,
|
||
error=str(exc),
|
||
)
|
||
return False
|
||
|
||
self._clear_error()
|
||
self._last_connect_at = now_cn_aware().isoformat()
|
||
self._last_disconnect_at = None
|
||
self._restart_attempts = 0
|
||
self._next_retry_at = 0.0
|
||
self._start_ws_thread(app_id, app_secret)
|
||
return True
|
||
|
||
async def _preflight_auth(self, *, tenant_id: int) -> None:
|
||
async with self._session_factory() as db:
|
||
service = FeishuIntegrationService(db)
|
||
await service.load_config(tenant_id=tenant_id)
|
||
await service._get_tenant_access_token()
|
||
|
||
async def _connection_supervisor(self) -> None:
|
||
while self._running:
|
||
try:
|
||
await asyncio.sleep(3)
|
||
except asyncio.CancelledError:
|
||
break
|
||
|
||
if not self._running or self._stopping:
|
||
continue
|
||
if not self._configured or self._event_mode != "long_connection":
|
||
continue
|
||
if not self._sdk_available:
|
||
continue
|
||
if self._thread and self._thread.is_alive():
|
||
continue
|
||
if self._last_error_code and not self._last_error_retryable:
|
||
continue
|
||
|
||
now = time.time()
|
||
if self._next_retry_at > now:
|
||
continue
|
||
|
||
await self._ensure_connection(reason="supervisor")
|
||
|
||
@staticmethod
|
||
def _classify_error(exc: Exception) -> tuple[str, bool]:
|
||
text = str(exc or "").strip().lower()
|
||
if "401" in text or "403" in text or "unauthorized" in text or "invalid app" in text:
|
||
return "auth_failed", False
|
||
if any(marker in text for marker in ("timed out", "timeout", "econnreset", "connection reset", "network", "dns", "temporary failure")):
|
||
return "network_error", True
|
||
if "certificate" in text or "ssl" in text or "tls" in text:
|
||
return "network_error", True
|
||
return "client_error", True
|
||
|
||
def _set_error(self, message: str, *, code: str, retryable: bool) -> None:
|
||
self._last_error = str(message or "").strip() or None
|
||
self._last_error_code = code
|
||
self._last_error_retryable = retryable
|
||
|
||
def _clear_error(self) -> None:
|
||
self._last_error = None
|
||
self._last_error_code = None
|
||
self._last_error_retryable = False
|
||
|
||
def _schedule_retry(self) -> None:
|
||
if self._restart_attempts >= self._max_retry_attempts:
|
||
self._last_error_retryable = False
|
||
self._next_retry_at = 0.0
|
||
logger.error(
|
||
"feishu_long_connection_retry_exhausted",
|
||
tenant_id=self._tenant_id,
|
||
attempts=self._restart_attempts,
|
||
max_attempts=self._max_retry_attempts,
|
||
last_error_code=self._last_error_code,
|
||
last_error=self._last_error,
|
||
)
|
||
return
|
||
self._restart_attempts += 1
|
||
backoff = self._retry_base_seconds * (2 ** max(0, self._restart_attempts - 1))
|
||
self._next_retry_at = time.time() + backoff
|
||
|
||
def _start_ws_thread(self, app_id: str, app_secret: str) -> None:
|
||
self._thread = threading.Thread(
|
||
target=self._run_ws_client,
|
||
args=(app_id, app_secret),
|
||
name="feishu-long-connection",
|
||
daemon=True,
|
||
)
|
||
self._thread.start()
|
||
|
||
def _run_ws_client(self, app_id: str, app_secret: str) -> None:
|
||
"""在独立线程中运行 SDK 客户端（SDK start 是阻塞调用）。"""
|
||
try:
|
||
import lark_oapi as lark # type: ignore
|
||
|
||
builder = lark.EventDispatcherHandler.builder("", "")
|
||
register_receive = getattr(builder, "register_p2_im_message_receive_v1", None)
|
||
if not callable(register_receive):
|
||
raise RuntimeError("当前 lark_oapi 版本不支持 register_p2_im_message_receive_v1")
|
||
builder = register_receive(self._on_message_event)
|
||
|
||
# 明确注册高频非业务事件，避免 SDK 抛 processor not found 干扰可观测性。
|
||
register_read = getattr(builder, "register_p2_im_message_message_read_v1", None)
|
||
if callable(register_read):
|
||
builder = register_read(self._on_ignored_event)
|
||
register_reaction_create = getattr(builder, "register_p2_im_message_reaction_created_v1", None)
|
||
if callable(register_reaction_create):
|
||
builder = register_reaction_create(self._on_ignored_event)
|
||
register_reaction_delete = getattr(builder, "register_p2_im_message_reaction_deleted_v1", None)
|
||
if callable(register_reaction_delete):
|
||
builder = register_reaction_delete(self._on_ignored_event)
|
||
register_recalled = getattr(builder, "register_p2_im_message_recalled_v1", None)
|
||
if callable(register_recalled):
|
||
builder = register_recalled(self._on_ignored_event)
|
||
|
||
event_handler = builder.build()
|
||
self._client = lark.ws.Client(
|
||
app_id,
|
||
app_secret,
|
||
event_handler=event_handler,
|
||
log_level=lark.LogLevel.INFO,
|
||
)
|
||
self._client.start()
|
||
except Exception as exc: # pragma: no cover
|
||
code, retryable = self._classify_error(exc)
|
||
self._set_error(str(exc), code=code, retryable=retryable)
|
||
if retryable and self._running and not self._stopping:
|
||
self._schedule_retry()
|
||
logger.error(
|
||
"feishu_long_connection_client_failed",
|
||
tenant_id=self._tenant_id,
|
||
error_code=code,
|
||
retryable=retryable,
|
||
error=str(exc),
|
||
)
|
||
finally:
|
||
if self._client is not None:
|
||
try:
|
||
self._stop_client_blocking(self._client)
|
||
except Exception as exc: # pragma: no cover
|
||
logger.warning(f"feishu_long_connection_client_cleanup_failed: {exc}")
|
||
self._client = None
|
||
if self._running and not self._stopping:
|
||
self._last_disconnect_at = now_cn_aware().isoformat()
|
||
if not self._last_error_code:
|
||
self._set_error("feishu long connection thread exited", code="thread_exited", retryable=True)
|
||
self._schedule_retry()
|
||
logger.warning("feishu_long_connection_thread_exited", tenant_id=self._tenant_id)
|
||
self._thread = None
|
||
|
||
def _on_message_event(self, data: Any) -> None:
|
||
"""SDK 回调：将事件安全投递到主事件循环。"""
|
||
if not self._running or not self._loop:
|
||
return
|
||
|
||
try:
|
||
perf_t0 = time.perf_counter()
|
||
payload = self._to_dict(data)
|
||
message_id = ""
|
||
event_type = ""
|
||
try:
|
||
event = payload.get("event", {}) if isinstance(payload, dict) else {}
|
||
header = payload.get("header", {}) if isinstance(payload, dict) else {}
|
||
message = event.get("message", {}) if isinstance(event, dict) else {}
|
||
message_id = str(message.get("message_id") or "").strip()
|
||
event_type = str(
|
||
(header.get("event_type") if isinstance(header, dict) else "")
|
||
or payload.get("event_type")
|
||
or payload.get("type")
|
||
or ""
|
||
).strip()
|
||
except Exception:
|
||
pass
|
||
if isinstance(payload, dict):
|
||
payload["_perf_t0"] = float(perf_t0)
|
||
print(
|
||
f"[PERF] message_received ts={now_cn_aware().isoformat()} T0={perf_t0:.6f} "
|
||
f"message_id={message_id or 'unknown'} "
|
||
f"event_type={event_type or 'unknown'}"
|
||
)
|
||
logger.info(
|
||
"feishu_long_connection_event_received: "
|
||
f"event_type={event_type or 'unknown'}, message_id={message_id or 'unknown'}"
|
||
)
|
||
self._loop.call_soon_threadsafe(self._queue.put_nowait, payload)
|
||
except Exception as exc: # pragma: no cover
|
||
self._set_error(str(exc), code="enqueue_error", retryable=True)
|
||
logger.warning(f"feishu_long_connection_enqueue_failed: {exc}")
|
||
|
||
def _on_ignored_event(self, data: Any) -> None:
|
||
"""显式忽略非业务事件，避免 SDK 报 processor not found。"""
|
||
if not self._running:
|
||
return
|
||
payload = self._to_dict(data)
|
||
event_type = ""
|
||
try:
|
||
header = payload.get("header", {}) if isinstance(payload, dict) else {}
|
||
event_type = str(
|
||
(header.get("event_type") if isinstance(header, dict) else "")
|
||
or payload.get("event_type")
|
||
or payload.get("type")
|
||
or ""
|
||
).strip()
|
||
except Exception:
|
||
pass
|
||
logger.info(f"feishu_long_connection_event_ignored: event_type={event_type or 'unknown'}")
|
||
|
||
async def _event_worker(self) -> None:
|
||
while self._running:
|
||
try:
|
||
payload = await self._queue.get()
|
||
except asyncio.CancelledError:
|
||
break
|
||
|
||
try:
|
||
await self._process_payload(payload)
|
||
self._last_event_at = now_cn_aware().isoformat()
|
||
except Exception as exc:
|
||
self._set_error(str(exc), code="event_process_failed", retryable=True)
|
||
logger.warning(f"feishu_long_connection_process_failed: {exc}")
|
||
finally:
|
||
self._queue.task_done()
|
||
|
||
async def _process_payload(self, payload: dict[str, Any]) -> None:
|
||
event_data = self._extract_event_data(payload)
|
||
if not event_data:
|
||
logger.info("feishu_long_connection_event_ignored: empty_event")
|
||
return
|
||
|
||
event_type = str(event_data.get("type") or "").strip()
|
||
if event_type and event_type not in {"im.message.receive_v1", "message.receive_v1"}:
|
||
logger.info(f"feishu_long_connection_event_skipped: event_type={event_type}")
|
||
return
|
||
|
||
async with self._session_factory() as db:
|
||
service = FeishuIntegrationService(db)
|
||
await service.load_config(tenant_id=self._tenant_id)
|
||
await service.handle_message_event(event_data)
|
||
|
||
def _extract_event_data(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
event_data = payload.get("event")
|
||
if not isinstance(event_data, dict):
|
||
event_data = {}
|
||
|
||
header = payload.get("header")
|
||
event_type: Optional[str] = None
|
||
if isinstance(header, dict):
|
||
event_type = header.get("event_type")
|
||
if not event_type:
|
||
event_type = payload.get("event_type") or payload.get("type")
|
||
|
||
if event_type:
|
||
event_data = {**event_data, "type": event_type}
|
||
|
||
perf_t0 = payload.get("_perf_t0")
|
||
if isinstance(perf_t0, (int, float)):
|
||
event_data = {**event_data, "_perf_t0": float(perf_t0)}
|
||
|
||
if event_data:
|
||
event_data = {**event_data, "_ingress_source": "long_connection"}
|
||
|
||
return event_data
|
||
|
||
def _to_dict(self, value: Any) -> Any:
|
||
if value is None or isinstance(value, (str, int, float, bool)):
|
||
return value
|
||
if isinstance(value, list):
|
||
return [self._to_dict(item) for item in value]
|
||
if isinstance(value, dict):
|
||
return {k: self._to_dict(v) for k, v in value.items()}
|
||
to_dict = getattr(value, "to_dict", None)
|
||
if callable(to_dict):
|
||
return self._to_dict(to_dict())
|
||
if hasattr(value, "__dict__"):
|
||
return {
|
||
k: self._to_dict(v)
|
||
for k, v in value.__dict__.items()
|
||
if not k.startswith("_")
|
||
}
|
||
return str(value)
|
||
|
||
|
||
_manager: Optional[FeishuLongConnectionManager] = None
|
||
|
||
|
||
def init_feishu_long_connection_manager(
|
||
session_factory: async_sessionmaker[AsyncSession],
|
||
) -> FeishuLongConnectionManager:
|
||
global _manager
|
||
if _manager is None:
|
||
_manager = FeishuLongConnectionManager(session_factory)
|
||
return _manager
|
||
|
||
|
||
def get_feishu_long_connection_manager() -> Optional[FeishuLongConnectionManager]:
|
||
return _manager
|