124 lines
3.5 KiB
Bash
Executable File
124 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
|
cd "$ROOT_DIR"
|
|
|
|
CONTAINER_NAME="${CANARY_BACKEND_CONTAINER:-pmflow-backend}"
|
|
REPLAY_TENANT_ID="${CANARY_REPLAY_TENANT_ID:-9901}"
|
|
|
|
echo "[canary] step1/4 replay: normal + duplicate + schema_error + multi_write"
|
|
docker exec "$CONTAINER_NAME" sh -lc '
|
|
cd /app/backend &&
|
|
PYTHONPATH=/app/backend python evals/feishu_replay_eval.py \
|
|
--strict \
|
|
--tenant-id '"$REPLAY_TENANT_ID"' \
|
|
--case-ids FEI001,FEI048,FEI050,FEI051 \
|
|
--input evals/golden_feishu_replay_v1.jsonl \
|
|
--output-dir evals/reports
|
|
'
|
|
|
|
echo "[canary] step2/4 insert pending outbox row"
|
|
OUTBOX_ID="$(
|
|
CANARY_REPLAY_TENANT_ID="$REPLAY_TENANT_ID" docker exec "$CONTAINER_NAME" sh -lc "python - <<'PY'
|
|
import asyncio
|
|
import os
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import FeishuReplyOutbox
|
|
|
|
async def main():
|
|
tenant_id = int(os.getenv('CANARY_REPLAY_TENANT_ID', '9901'))
|
|
async with AsyncSessionLocal() as db:
|
|
row = FeishuReplyOutbox(
|
|
tenant_id=tenant_id,
|
|
trace_id='canary-outbox-restart',
|
|
chat_id='oc_canary_invalid_chat',
|
|
message_text='canary pending outbox',
|
|
status='pending',
|
|
attempts=0,
|
|
)
|
|
db.add(row)
|
|
await db.commit()
|
|
await db.refresh(row)
|
|
print(row.id)
|
|
|
|
asyncio.run(main())
|
|
PY"
|
|
)"
|
|
echo "[canary] pending outbox id=$OUTBOX_ID"
|
|
|
|
echo "[canary] step3/4 restart backend"
|
|
docker compose restart backend >/dev/null
|
|
|
|
echo "[canary] waiting backend healthy"
|
|
for i in $(seq 1 30); do
|
|
STATUS="$(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER_NAME" 2>/dev/null || true)"
|
|
if [[ "$STATUS" == "healthy" ]]; then
|
|
echo "[canary] backend healthy"
|
|
break
|
|
fi
|
|
sleep 1
|
|
if [[ "$i" -eq 30 ]]; then
|
|
echo "[canary][FAIL] backend not healthy"
|
|
exit 2
|
|
fi
|
|
done
|
|
|
|
echo "[canary] step4/4 verify outbox resumed after restart"
|
|
RESUMED="0"
|
|
for _ in $(seq 1 20); do
|
|
SNAPSHOT="$(
|
|
docker exec "$CONTAINER_NAME" sh -lc "python - <<'PY'
|
|
import asyncio
|
|
from sqlalchemy import select
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import FeishuReplyOutbox
|
|
|
|
async def main():
|
|
async with AsyncSessionLocal() as db:
|
|
row = (await db.execute(
|
|
select(FeishuReplyOutbox).where(FeishuReplyOutbox.trace_id=='canary-outbox-restart').order_by(FeishuReplyOutbox.id.desc())
|
|
)).scalar_one_or_none()
|
|
if row is None:
|
|
print('missing')
|
|
return
|
|
print(f'{row.status}|{int(row.attempts or 0)}')
|
|
|
|
asyncio.run(main())
|
|
PY"
|
|
)"
|
|
if [[ "$SNAPSHOT" == "missing" ]]; then
|
|
echo "[canary][FAIL] outbox row missing unexpectedly"
|
|
exit 2
|
|
fi
|
|
OUTBOX_STATUS="${SNAPSHOT%%|*}"
|
|
OUTBOX_ATTEMPTS="${SNAPSHOT##*|}"
|
|
if [[ "$OUTBOX_STATUS" != "pending" || "$OUTBOX_ATTEMPTS" -gt 0 ]]; then
|
|
RESUMED="1"
|
|
echo "[canary] outbox resumed: status=$OUTBOX_STATUS attempts=$OUTBOX_ATTEMPTS"
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if [[ "$RESUMED" != "1" ]]; then
|
|
echo "[canary][FAIL] outbox still pending after restart"
|
|
exit 2
|
|
fi
|
|
|
|
docker exec "$CONTAINER_NAME" sh -lc "python - <<'PY'
|
|
import asyncio
|
|
from sqlalchemy import delete
|
|
from app.database import AsyncSessionLocal
|
|
from app.models import FeishuReplyOutbox
|
|
|
|
async def main():
|
|
async with AsyncSessionLocal() as db:
|
|
await db.execute(delete(FeishuReplyOutbox).where(FeishuReplyOutbox.trace_id=='canary-outbox-restart'))
|
|
await db.commit()
|
|
|
|
asyncio.run(main())
|
|
PY" >/dev/null
|
|
|
|
echo "[canary][OK] feishu smoke passed"
|