#!/bin/bash
# Crash Loop Guard — auto-rollback on gateway crash loop
# Runs via SYSTEM cron (not Clawdbot — Clawdbot is dead during loops)
# Every 2 minutes: check restart frequency → restore backup → restart
# Created: 2026-02-06

set -euo pipefail

BACKUP_DIR="/root/.clawdbot/config-backups"
CONFIG_PATH="/root/.clawdbot/clawdbot.json"
AUTH_PATH="/root/.clawdbot/auth-profiles.json"
STATE_FILE="/tmp/crash-loop-guard.state"
LOG_FILE="/home/ccuser/rateright-growth/rivet/memory/watchdog-log.md"

# --- Thresholds ---
MAX_RESTARTS=5          # restarts in window = crash loop
WINDOW_SECONDS=300      # 5 minute window
COOLDOWN_SECONDS=600    # don't rollback again within 10 min

# --- Count recent restarts (systemd-level service restarts only) ---
RESTART_COUNT=$(journalctl -u clawdbot-gateway --since "${WINDOW_SECONDS} seconds ago" --no-pager 2>/dev/null \
    | grep -cE "^.*systemd.*: Started |^.*systemd.*: Stopped |Service hold-off time over, scheduling restart" || true)
RESTART_COUNT=${RESTART_COUNT:-0}
RESTART_COUNT=$(echo "$RESTART_COUNT" | tr -d '[:space:]')

# Also check process exit signals (systemd-level, not app log noise)
CRASH_COUNT=$(journalctl -u clawdbot-gateway --since "${WINDOW_SECONDS} seconds ago" --no-pager 2>/dev/null \
    | grep -cE "^.*systemd.*: clawdbot-gateway.*exited|^.*systemd.*: clawdbot-gateway.*failed|Main process exited|Failed with result" || true)
CRASH_COUNT=${CRASH_COUNT:-0}
CRASH_COUNT=$(echo "$CRASH_COUNT" | tr -d '[:space:]')

NOW=$(date +%s)
NOW_HUMAN=$(TZ=Australia/Sydney date '+%Y-%m-%d %H:%M AEST')

# --- Not in a loop? Exit silently ---
if [ "$RESTART_COUNT" -lt "$MAX_RESTARTS" ] && [ "$CRASH_COUNT" -lt "$MAX_RESTARTS" ]; then
    exit 0
fi

# --- Cooldown check: don't rollback repeatedly ---
if [ -f "$STATE_FILE" ]; then
    LAST_ROLLBACK=$(cat "$STATE_FILE" 2>/dev/null || echo 0)
    ELAPSED=$((NOW - LAST_ROLLBACK))
    if [ "$ELAPSED" -lt "$COOLDOWN_SECONDS" ]; then
        echo "[$NOW_HUMAN] Crash loop detected but in cooldown (${ELAPSED}s < ${COOLDOWN_SECONDS}s)" >> "$LOG_FILE"
        exit 0
    fi
fi

# --- CRASH LOOP DETECTED — but what kind? ---
echo "[$NOW_HUMAN] 🚨 CRASH LOOP DETECTED — ${RESTART_COUNT} restarts, ${CRASH_COUNT} crashes in ${WINDOW_SECONDS}s" >> "$LOG_FILE"

# --- Check if it's a port conflict (not a config problem) ---
PORT_CONFLICT=$(journalctl -u clawdbot-gateway --since "${WINDOW_SECONDS} seconds ago" --no-pager 2>/dev/null \
    | grep -c "Gateway already running locally\|Stop it first\|different port" || true)
PORT_CONFLICT=$(echo "${PORT_CONFLICT:-0}" | tr -d '[:space:]')

if [ "$PORT_CONFLICT" -gt 0 ]; then
    # Gateway is actually running fine, systemd is just confused
    if curl -sf -o /dev/null --max-time 5 http://127.0.0.1:18789/ 2>/dev/null; then
        echo "[$NOW_HUMAN] ℹ️ Port conflict — gateway alive but systemd fighting it. Stopping systemd service." >> "$LOG_FILE"
        systemctl stop clawdbot-gateway 2>/dev/null || true
        echo "$NOW" > "$STATE_FILE"
        exit 0
    fi
fi

# --- Find latest backup ---
LATEST_CONFIG=$(ls -1t "${BACKUP_DIR}"/config-2*.json 2>/dev/null | head -1)
LATEST_AUTH=$(ls -1t "${BACKUP_DIR}"/auth-profiles-2*.json 2>/dev/null | head -1)

if [ -z "$LATEST_CONFIG" ]; then
    echo "[$NOW_HUMAN] ❌ NO BACKUP FOUND — cannot auto-rollback" >> "$LOG_FILE"
    exit 1
fi

# --- Validate backup is valid JSON ---
if ! jq empty "$LATEST_CONFIG" 2>/dev/null; then
    echo "[$NOW_HUMAN] ❌ Latest backup is invalid JSON — cannot auto-rollback" >> "$LOG_FILE"
    exit 1
fi

# --- Check backup has critical keys ---
BACKUP_MODE=$(jq -r '.gateway.mode // ""' "$LATEST_CONFIG" 2>/dev/null)
if [ -z "$BACKUP_MODE" ]; then
    echo "[$NOW_HUMAN] ❌ Backup missing gateway.mode — unsafe to restore" >> "$LOG_FILE"
    exit 1
fi

# --- Stop the crash loop ---
echo "[$NOW_HUMAN] Stopping gateway to break crash loop..." >> "$LOG_FILE"
clawdbot gateway stop 2>/dev/null || true
sleep 3

# Kill any orphaned processes holding the port
pkill -f "clawdbot-gateway" 2>/dev/null || true
sleep 2

# --- Save the broken config for forensics ---
TIMESTAMP=$(date '+%Y%m%d-%H%M%S')
if [ -f "$CONFIG_PATH" ]; then
    cp "$CONFIG_PATH" "${BACKUP_DIR}/config-BROKEN-${TIMESTAMP}.json" 2>/dev/null || true
fi
if [ -f "$AUTH_PATH" ]; then
    cp "$AUTH_PATH" "${BACKUP_DIR}/auth-profiles-BROKEN-${TIMESTAMP}.json" 2>/dev/null || true
fi

# --- Restore from backup ---
cp "$LATEST_CONFIG" "$CONFIG_PATH"
echo "[$NOW_HUMAN] ✅ Restored config from: $(basename "$LATEST_CONFIG")" >> "$LOG_FILE"

if [ -n "$LATEST_AUTH" ] && [ -f "$LATEST_AUTH" ]; then
    cp "$LATEST_AUTH" "$AUTH_PATH"
    echo "[$NOW_HUMAN] ✅ Restored auth-profiles from: $(basename "$LATEST_AUTH")" >> "$LOG_FILE"
fi

# --- Restart gateway ---
sleep 2
clawdbot gateway start 2>/dev/null &
echo "[$NOW_HUMAN] ✅ Gateway restart initiated" >> "$LOG_FILE"

# --- Record rollback time for cooldown ---
echo "$NOW" > "$STATE_FILE"

# --- Wait and verify ---
sleep 15
if curl -sf -o /dev/null --max-time 5 http://127.0.0.1:18789/ 2>/dev/null; then
    echo "[$NOW_HUMAN] ✅ Gateway recovered — responding on port 18789" >> "$LOG_FILE"
    RECOVERY_MSG="🚨 SysOps Auto-Recovery\n\nCrash loop detected (${RESTART_COUNT} restarts in ${WINDOW_SECONDS}s).\nRolled back to: $(basename "$LATEST_CONFIG")\nGateway: ✅ recovered\n\nBroken config saved as: config-BROKEN-${TIMESTAMP}.json"
else
    echo "[$NOW_HUMAN] ❌ Gateway still not responding after rollback" >> "$LOG_FILE"
    RECOVERY_MSG="🚨 SysOps Auto-Recovery FAILED\n\nCrash loop detected (${RESTART_COUNT} restarts).\nRolled back config but gateway still not responding.\nManual intervention needed.\n\nBroken config: config-BROKEN-${TIMESTAMP}.json\nRestored from: $(basename "$LATEST_CONFIG")"
fi

echo "---"
echo -e "$RECOVERY_MSG"