"""Feishu 长连接事件接收管理器。""" from __future__ import annotations import asyncio from datetime import datetime, timezone import importlib.util import json import os import threading import time from typing import Any, Optional from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from sqlalchemy.future import select from ..core.logging import get_logger from ..models import Integration from .feishu_integration import FeishuIntegrationService from ..utils.timezone import CN_TZ, now_cn_aware logger = get_logger(__name__) DEFAULT_FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS = 3 DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS = 1.0 class FeishuLongConnectionManager: """管理飞书长连接 SDK 生命周期与事件消费。""" def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: self._session_factory = session_factory self._loop: Optional[asyncio.AbstractEventLoop] = None self._queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue() self._worker_task: Optional[asyncio.Task[None]] = None self._supervisor_task: Optional[asyncio.Task[None]] = None self._thread: Optional[threading.Thread] = None self._client: Any = None self._boot_lock = asyncio.Lock() self._running = False self._sdk_available = self._check_sdk_available() self._configured = False self._event_mode = "long_connection" self._tenant_id = 1 self._last_error: Optional[str] = None self._last_error_code: Optional[str] = None self._last_error_retryable = False self._last_event_at: Optional[str] = None self._last_auth_at: Optional[str] = None self._last_connect_at: Optional[str] = None self._last_disconnect_at: Optional[str] = None self._restart_attempts = 0 self._next_retry_at = 0.0 self._stopping = False self._max_retry_attempts = self._read_retry_max_attempts() self._retry_base_seconds = self._read_retry_base_seconds() async def start(self) -> None: """启动长连接（若未配置或未安装 SDK 则仅记录状态）。""" if self._running: return self._stopping = False self._running = True self._loop = asyncio.get_running_loop() self._worker_task = asyncio.create_task( self._event_worker(), name="feishu-long-connection-worker", ) self._supervisor_task = asyncio.create_task( self._connection_supervisor(), name="feishu-long-connection-supervisor", ) started = await self._ensure_connection(reason="startup") if started: logger.info("feishu_long_connection_started", tenant_id=self._tenant_id, event_mode=self._event_mode) async def stop(self) -> None: """停止长连接。""" self._stopping = True self._running = False current_client = self._client if current_client is not None: try: await asyncio.to_thread(self._stop_client_blocking, current_client) except Exception as exc: # pragma: no cover logger.warning(f"feishu_long_connection_stop_failed: {exc}") self._client = None if self._supervisor_task: self._supervisor_task.cancel() try: await self._supervisor_task except asyncio.CancelledError: pass self._supervisor_task = None if self._worker_task: self._worker_task.cancel() try: await self._worker_task except asyncio.CancelledError: pass self._worker_task = None if self._thread and self._thread.is_alive(): await asyncio.to_thread(self._thread.join, 3) self._thread = None while not self._queue.empty(): try: self._queue.get_nowait() self._queue.task_done() except asyncio.QueueEmpty: break logger.info("feishu_long_connection_stopped") self._stopping = False async def restart(self) -> None: """重启长连接（用于配置变更后重连）。""" await self.stop() await self.start() def status(self) -> dict[str, Any]: """返回当前运行状态，供 API 展示。""" return { "running": self._running, "tenant_id": self._tenant_id, "event_mode": self._event_mode, "sdk_available": self._sdk_available, "configured": self._configured, "worker_alive": bool(self._worker_task and not self._worker_task.done()), "supervisor_alive": bool(self._supervisor_task and not self._supervisor_task.done()), "thread_alive": bool(self._thread and self._thread.is_alive()), "last_error": self._last_error, "last_error_code": self._last_error_code, "last_error_retryable": self._last_error_retryable, "last_event_at": self._last_event_at, "last_auth_at": self._last_auth_at, "last_connect_at": self._last_connect_at, "last_disconnect_at": self._last_disconnect_at, "restart_attempts": self._restart_attempts, "next_retry_at": ( datetime.fromtimestamp(self._next_retry_at, tz=CN_TZ).isoformat() if self._next_retry_at > 0 else None ), "retry_policy": { "max_attempts": self._max_retry_attempts, "base_seconds": self._retry_base_seconds, }, } def _check_sdk_available(self) -> bool: # 只做可用性探测，避免在主线程提前 import lark_oapi， # 否则 SDK 会把全局 event loop 绑定到 uvicorn loop，导致线程内 run_until_complete 报错。 return importlib.util.find_spec("lark_oapi") is not None @staticmethod def _read_retry_max_attempts() -> int: raw = os.getenv( "FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS", str(DEFAULT_FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS), ) try: parsed = int(str(raw).strip()) except Exception: parsed = DEFAULT_FEISHU_LONG_CONNECTION_RETRY_MAX_ATTEMPTS # 强制最多 3 次，避免无限重连雪崩。 return min(3, max(1, parsed)) @staticmethod def _read_retry_base_seconds() -> float: raw = os.getenv( "FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS", str(DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS), ) try: parsed = float(str(raw).strip()) except Exception: parsed = DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS if parsed <= 0: parsed = DEFAULT_FEISHU_LONG_CONNECTION_RETRY_BASE_SECONDS return parsed @staticmethod def _stop_client_blocking(client: Any) -> None: stop_fn = getattr(client, "stop", None) if callable(stop_fn): stop_fn() async def _load_active_config(self) -> Optional[dict[str, Any]]: async with self._session_factory() as db: result = await db.execute( select(Integration).where( Integration.type == "feishu", Integration.is_active.is_(True), ).order_by(Integration.updated_at.desc(), Integration.id.desc()) ) integration = result.scalars().first() if not integration: return None config = integration.config parsed: dict[str, Any] if isinstance(config, str): try: parsed = json.loads(config) if config else {} except Exception: parsed = {} else: parsed = config or {} parsed["tenant_id"] = int(getattr(integration, "tenant_id", 1) or 1) return parsed async def _ensure_connection(self, *, reason: str) -> bool: async with self._boot_lock: if not self._running: return False if self._thread and self._thread.is_alive(): return True if self._client is not None: try: await asyncio.to_thread(self._stop_client_blocking, self._client) except Exception as exc: # pragma: no cover logger.warning(f"feishu_long_connection_stale_client_stop_failed: {exc}") finally: self._client = None config = await self._load_active_config() self._event_mode = "long_connection" self._tenant_id = int((config or {}).get("tenant_id") or 1) app_id = str((config or {}).get("app_id") or "").strip() app_secret = str((config or {}).get("app_secret") or "").strip() self._configured = bool(app_id and app_secret) if not self._configured: self._clear_error() logger.info("feishu_long_connection_not_configured") return False if not self._sdk_available: self._set_error("lark_oapi not installed", code="sdk_missing", retryable=False) logger.warning("feishu_long_connection_sdk_missing: pip install lark-oapi") return False try: await self._preflight_auth(tenant_id=self._tenant_id) self._last_auth_at = now_cn_aware().isoformat() except Exception as exc: code, retryable = self._classify_error(exc) self._set_error(str(exc), code=code, retryable=retryable) if retryable: self._schedule_retry() logger.warning( "feishu_long_connection_preflight_failed", tenant_id=self._tenant_id, error_code=code, retryable=retryable, reason=reason, error=str(exc), ) return False self._clear_error() self._last_connect_at = now_cn_aware().isoformat() self._last_disconnect_at = None self._restart_attempts = 0 self._next_retry_at = 0.0 self._start_ws_thread(app_id, app_secret) return True async def _preflight_auth(self, *, tenant_id: int) -> None: async with self._session_factory() as db: service = FeishuIntegrationService(db) await service.load_config(tenant_id=tenant_id) await service._get_tenant_access_token() async def _connection_supervisor(self) -> None: while self._running: try: await asyncio.sleep(3) except asyncio.CancelledError: break if not self._running or self._stopping: continue if not self._configured or self._event_mode != "long_connection": continue if not self._sdk_available: continue if self._thread and self._thread.is_alive(): continue if self._last_error_code and not self._last_error_retryable: continue now = time.time() if self._next_retry_at > now: continue await self._ensure_connection(reason="supervisor") @staticmethod def _classify_error(exc: Exception) -> tuple[str, bool]: text = str(exc or "").strip().lower() if "401" in text or "403" in text or "unauthorized" in text or "invalid app" in text: return "auth_failed", False if any(marker in text for marker in ("timed out", "timeout", "econnreset", "connection reset", "network", "dns", "temporary failure")): return "network_error", True if "certificate" in text or "ssl" in text or "tls" in text: return "network_error", True return "client_error", True def _set_error(self, message: str, *, code: str, retryable: bool) -> None: self._last_error = str(message or "").strip() or None self._last_error_code = code self._last_error_retryable = retryable def _clear_error(self) -> None: self._last_error = None self._last_error_code = None self._last_error_retryable = False def _schedule_retry(self) -> None: if self._restart_attempts >= self._max_retry_attempts: self._last_error_retryable = False self._next_retry_at = 0.0 logger.error( "feishu_long_connection_retry_exhausted", tenant_id=self._tenant_id, attempts=self._restart_attempts, max_attempts=self._max_retry_attempts, last_error_code=self._last_error_code, last_error=self._last_error, ) return self._restart_attempts += 1 backoff = self._retry_base_seconds * (2 ** max(0, self._restart_attempts - 1)) self._next_retry_at = time.time() + backoff def _start_ws_thread(self, app_id: str, app_secret: str) -> None: self._thread = threading.Thread( target=self._run_ws_client, args=(app_id, app_secret), name="feishu-long-connection", daemon=True, ) self._thread.start() def _run_ws_client(self, app_id: str, app_secret: str) -> None: """在独立线程中运行 SDK 客户端（SDK start 是阻塞调用）。""" try: import lark_oapi as lark # type: ignore builder = lark.EventDispatcherHandler.builder("", "") register_receive = getattr(builder, "register_p2_im_message_receive_v1", None) if not callable(register_receive): raise RuntimeError("当前 lark_oapi 版本不支持 register_p2_im_message_receive_v1") builder = register_receive(self._on_message_event) # 明确注册高频非业务事件，避免 SDK 抛 processor not found 干扰可观测性。 register_read = getattr(builder, "register_p2_im_message_message_read_v1", None) if callable(register_read): builder = register_read(self._on_ignored_event) register_reaction_create = getattr(builder, "register_p2_im_message_reaction_created_v1", None) if callable(register_reaction_create): builder = register_reaction_create(self._on_ignored_event) register_reaction_delete = getattr(builder, "register_p2_im_message_reaction_deleted_v1", None) if callable(register_reaction_delete): builder = register_reaction_delete(self._on_ignored_event) register_recalled = getattr(builder, "register_p2_im_message_recalled_v1", None) if callable(register_recalled): builder = register_recalled(self._on_ignored_event) event_handler = builder.build() self._client = lark.ws.Client( app_id, app_secret, event_handler=event_handler, log_level=lark.LogLevel.INFO, ) self._client.start() except Exception as exc: # pragma: no cover code, retryable = self._classify_error(exc) self._set_error(str(exc), code=code, retryable=retryable) if retryable and self._running and not self._stopping: self._schedule_retry() logger.error( "feishu_long_connection_client_failed", tenant_id=self._tenant_id, error_code=code, retryable=retryable, error=str(exc), ) finally: if self._client is not None: try: self._stop_client_blocking(self._client) except Exception as exc: # pragma: no cover logger.warning(f"feishu_long_connection_client_cleanup_failed: {exc}") self._client = None if self._running and not self._stopping: self._last_disconnect_at = now_cn_aware().isoformat() if not self._last_error_code: self._set_error("feishu long connection thread exited", code="thread_exited", retryable=True) self._schedule_retry() logger.warning("feishu_long_connection_thread_exited", tenant_id=self._tenant_id) self._thread = None def _on_message_event(self, data: Any) -> None: """SDK 回调：将事件安全投递到主事件循环。""" if not self._running or not self._loop: return try: perf_t0 = time.perf_counter() payload = self._to_dict(data) message_id = "" event_type = "" try: event = payload.get("event", {}) if isinstance(payload, dict) else {} header = payload.get("header", {}) if isinstance(payload, dict) else {} message = event.get("message", {}) if isinstance(event, dict) else {} message_id = str(message.get("message_id") or "").strip() event_type = str( (header.get("event_type") if isinstance(header, dict) else "") or payload.get("event_type") or payload.get("type") or "" ).strip() except Exception: pass if isinstance(payload, dict): payload["_perf_t0"] = float(perf_t0) print( f"[PERF] message_received ts={now_cn_aware().isoformat()} T0={perf_t0:.6f} " f"message_id={message_id or 'unknown'} " f"event_type={event_type or 'unknown'}" ) logger.info( "feishu_long_connection_event_received: " f"event_type={event_type or 'unknown'}, message_id={message_id or 'unknown'}" ) self._loop.call_soon_threadsafe(self._queue.put_nowait, payload) except Exception as exc: # pragma: no cover self._set_error(str(exc), code="enqueue_error", retryable=True) logger.warning(f"feishu_long_connection_enqueue_failed: {exc}") def _on_ignored_event(self, data: Any) -> None: """显式忽略非业务事件，避免 SDK 报 processor not found。""" if not self._running: return payload = self._to_dict(data) event_type = "" try: header = payload.get("header", {}) if isinstance(payload, dict) else {} event_type = str( (header.get("event_type") if isinstance(header, dict) else "") or payload.get("event_type") or payload.get("type") or "" ).strip() except Exception: pass logger.info(f"feishu_long_connection_event_ignored: event_type={event_type or 'unknown'}") async def _event_worker(self) -> None: while self._running: try: payload = await self._queue.get() except asyncio.CancelledError: break try: await self._process_payload(payload) self._last_event_at = now_cn_aware().isoformat() except Exception as exc: self._set_error(str(exc), code="event_process_failed", retryable=True) logger.warning(f"feishu_long_connection_process_failed: {exc}") finally: self._queue.task_done() async def _process_payload(self, payload: dict[str, Any]) -> None: event_data = self._extract_event_data(payload) if not event_data: logger.info("feishu_long_connection_event_ignored: empty_event") return event_type = str(event_data.get("type") or "").strip() if event_type and event_type not in {"im.message.receive_v1", "message.receive_v1"}: logger.info(f"feishu_long_connection_event_skipped: event_type={event_type}") return async with self._session_factory() as db: service = FeishuIntegrationService(db) await service.load_config(tenant_id=self._tenant_id) await service.handle_message_event(event_data) def _extract_event_data(self, payload: dict[str, Any]) -> dict[str, Any]: event_data = payload.get("event") if not isinstance(event_data, dict): event_data = {} header = payload.get("header") event_type: Optional[str] = None if isinstance(header, dict): event_type = header.get("event_type") if not event_type: event_type = payload.get("event_type") or payload.get("type") if event_type: event_data = {**event_data, "type": event_type} perf_t0 = payload.get("_perf_t0") if isinstance(perf_t0, (int, float)): event_data = {**event_data, "_perf_t0": float(perf_t0)} if event_data: event_data = {**event_data, "_ingress_source": "long_connection"} return event_data def _to_dict(self, value: Any) -> Any: if value is None or isinstance(value, (str, int, float, bool)): return value if isinstance(value, list): return [self._to_dict(item) for item in value] if isinstance(value, dict): return {k: self._to_dict(v) for k, v in value.items()} to_dict = getattr(value, "to_dict", None) if callable(to_dict): return self._to_dict(to_dict()) if hasattr(value, "__dict__"): return { k: self._to_dict(v) for k, v in value.__dict__.items() if not k.startswith("_") } return str(value) _manager: Optional[FeishuLongConnectionManager] = None def init_feishu_long_connection_manager( session_factory: async_sessionmaker[AsyncSession], ) -> FeishuLongConnectionManager: global _manager if _manager is None: _manager = FeishuLongConnectionManager(session_factory) return _manager def get_feishu_long_connection_manager() -> Optional[FeishuLongConnectionManager]: return _manager