254 lines
8.1 KiB
Python
254 lines
8.1 KiB
Python
"""团队负载告警服务。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timedelta
|
||
from typing import Optional
|
||
|
||
from sqlalchemy import and_, func, select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from ...models import OperationLog, Task, User, UserProject, UserWorkload
|
||
from ..integrations.integration_core.lark_service import lark_service
|
||
|
||
from ...utils.timezone import now_cn_naive
|
||
DONE_STATUSES = {"done", "completed", "closed", "resolved", "archived"}
|
||
|
||
|
||
async def _list_alert_logs(
|
||
db: AsyncSession,
|
||
*,
|
||
tenant_id: int,
|
||
user_id: int,
|
||
) -> list[OperationLog]:
|
||
rows = await db.execute(
|
||
select(OperationLog).where(
|
||
and_(
|
||
OperationLog.tenant_id == tenant_id,
|
||
OperationLog.entity_type == "workload_alert",
|
||
OperationLog.entity_id == str(user_id),
|
||
)
|
||
).order_by(OperationLog.created_at.desc())
|
||
)
|
||
return list(rows.scalars().all())
|
||
|
||
|
||
def _latest_by_action(logs: list[OperationLog], action: str) -> Optional[OperationLog]:
|
||
for row in logs:
|
||
if row.action == action:
|
||
return row
|
||
return None
|
||
|
||
|
||
def _is_unresolved(logs: list[OperationLog]) -> bool:
|
||
latest_detected = _latest_by_action(logs, "overload_detected")
|
||
if not latest_detected:
|
||
return False
|
||
latest_resolved = _latest_by_action(logs, "overload_resolved")
|
||
if not latest_resolved:
|
||
return True
|
||
return bool(latest_detected.created_at and latest_resolved.created_at and latest_detected.created_at > latest_resolved.created_at)
|
||
|
||
|
||
async def on_user_workload_changed(
|
||
db: AsyncSession,
|
||
*,
|
||
tenant_id: int,
|
||
user_id: int,
|
||
workload_ratio: float,
|
||
) -> dict:
|
||
"""用户负载变化时触发告警/恢复。"""
|
||
logs = await _list_alert_logs(db, tenant_id=tenant_id, user_id=user_id)
|
||
unresolved = _is_unresolved(logs)
|
||
|
||
if workload_ratio > 100 and not unresolved:
|
||
text = (
|
||
f"成员 {user_id} 负载过载：{workload_ratio:.1f}% (>100%)，"
|
||
f"请尽快处理任务转派。"
|
||
)
|
||
notify_result = await lark_service.send_card_message(
|
||
chat_id=f"tenant-{tenant_id}-workload-alert",
|
||
content=text,
|
||
tasks=[],
|
||
)
|
||
db.add(
|
||
OperationLog(
|
||
tenant_id=tenant_id,
|
||
user_id=user_id,
|
||
project_id=None,
|
||
entity_type="workload_alert",
|
||
entity_id=str(user_id),
|
||
action="overload_detected",
|
||
before_data={"unresolved": unresolved},
|
||
after_data={
|
||
"workload_ratio": round(workload_ratio, 2),
|
||
"notify_channel": "feishu",
|
||
"notify_status": notify_result.get("status"),
|
||
},
|
||
remark=text,
|
||
)
|
||
)
|
||
return {"alerted": True, "resolved": False}
|
||
|
||
if workload_ratio <= 100 and unresolved:
|
||
db.add(
|
||
OperationLog(
|
||
tenant_id=tenant_id,
|
||
user_id=user_id,
|
||
project_id=None,
|
||
entity_type="workload_alert",
|
||
entity_id=str(user_id),
|
||
action="overload_resolved",
|
||
before_data={"unresolved": True},
|
||
after_data={"workload_ratio": round(workload_ratio, 2)},
|
||
remark="workload_back_to_normal",
|
||
)
|
||
)
|
||
return {"alerted": False, "resolved": True}
|
||
|
||
return {"alerted": False, "resolved": False}
|
||
|
||
|
||
async def _build_transfer_suggestions(
|
||
db: AsyncSession,
|
||
*,
|
||
tenant_id: int,
|
||
user_id: int,
|
||
max_items: int = 3,
|
||
) -> list[dict]:
|
||
tasks_result = await db.execute(
|
||
select(Task).where(
|
||
and_(
|
||
Task.tenant_id == tenant_id,
|
||
Task.assignee_id == user_id,
|
||
func.lower(Task.status).notin_(DONE_STATUSES),
|
||
)
|
||
).order_by(Task.updated_at.desc())
|
||
)
|
||
tasks = list(tasks_result.scalars().all())
|
||
if not tasks:
|
||
return []
|
||
|
||
suggestions: list[dict] = []
|
||
for task in tasks:
|
||
member_result = await db.execute(
|
||
select(UserProject.user_id).where(
|
||
and_(
|
||
UserProject.tenant_id == tenant_id,
|
||
UserProject.project_id == int(task.project_id),
|
||
)
|
||
)
|
||
)
|
||
member_ids = [int(v) for v in member_result.scalars().all() if int(v) != user_id]
|
||
if not member_ids:
|
||
continue
|
||
|
||
candidate_result = await db.execute(
|
||
select(UserWorkload.user_id, UserWorkload.workload_ratio)
|
||
.join(User, User.id == UserWorkload.user_id)
|
||
.where(
|
||
and_(
|
||
UserWorkload.tenant_id == tenant_id,
|
||
UserWorkload.user_id.in_(member_ids),
|
||
User.tenant_id == tenant_id,
|
||
User.status == "active",
|
||
User.is_active.is_(True),
|
||
)
|
||
)
|
||
.order_by(UserWorkload.workload_ratio.asc())
|
||
)
|
||
candidates = candidate_result.all()
|
||
if not candidates:
|
||
continue
|
||
|
||
target_user_id, target_ratio = candidates[0]
|
||
suggestions.append(
|
||
{
|
||
"task_id": int(task.id),
|
||
"project_id": int(task.project_id),
|
||
"old_assignee_id": user_id,
|
||
"new_assignee_id": int(target_user_id),
|
||
"new_assignee_ratio": round(float(target_ratio or 0.0), 2),
|
||
"reason": "auto_transfer_after_24h_overload",
|
||
}
|
||
)
|
||
if len(suggestions) >= max_items:
|
||
break
|
||
return suggestions
|
||
|
||
|
||
async def process_tenant_unresolved_overloads(
|
||
db: AsyncSession,
|
||
*,
|
||
tenant_id: int,
|
||
) -> dict:
|
||
"""扫描租户过载并为持续 24h 未恢复的成员生成转派建议。"""
|
||
rows = await db.execute(
|
||
select(UserWorkload.user_id, UserWorkload.workload_ratio).join(
|
||
User,
|
||
and_(
|
||
User.id == UserWorkload.user_id,
|
||
User.tenant_id == tenant_id,
|
||
),
|
||
).where(
|
||
and_(
|
||
UserWorkload.tenant_id == tenant_id,
|
||
User.status == "active",
|
||
User.is_active.is_(True),
|
||
)
|
||
)
|
||
)
|
||
user_rows = rows.all()
|
||
|
||
created = 0
|
||
now = now_cn_naive()
|
||
for user_id_raw, ratio_raw in user_rows:
|
||
user_id = int(user_id_raw)
|
||
ratio = float(ratio_raw or 0.0)
|
||
await on_user_workload_changed(
|
||
db,
|
||
tenant_id=tenant_id,
|
||
user_id=user_id,
|
||
workload_ratio=ratio,
|
||
)
|
||
if ratio <= 100:
|
||
continue
|
||
|
||
logs = await _list_alert_logs(db, tenant_id=tenant_id, user_id=user_id)
|
||
latest_detected = _latest_by_action(logs, "overload_detected")
|
||
latest_resolved = _latest_by_action(logs, "overload_resolved")
|
||
latest_suggestion = _latest_by_action(logs, "transfer_suggestion")
|
||
if not latest_detected or not latest_detected.created_at:
|
||
continue
|
||
if latest_resolved and latest_resolved.created_at and latest_resolved.created_at > latest_detected.created_at:
|
||
continue
|
||
if now - latest_detected.created_at < timedelta(hours=24):
|
||
continue
|
||
if latest_suggestion and latest_suggestion.created_at and latest_suggestion.created_at > latest_detected.created_at:
|
||
continue
|
||
|
||
suggestions = await _build_transfer_suggestions(
|
||
db,
|
||
tenant_id=tenant_id,
|
||
user_id=user_id,
|
||
)
|
||
if not suggestions:
|
||
continue
|
||
db.add(
|
||
OperationLog(
|
||
tenant_id=tenant_id,
|
||
user_id=user_id,
|
||
project_id=None,
|
||
entity_type="workload_alert",
|
||
entity_id=str(user_id),
|
||
action="transfer_suggestion",
|
||
before_data={"workload_ratio": ratio},
|
||
after_data={"suggestions": suggestions},
|
||
remark="overload_unresolved_24h",
|
||
)
|
||
)
|
||
created += 1
|
||
|
||
await db.commit()
|
||
return {"suggestion_count": created, "user_count": len(user_rows)}
|