#!/bin/bash
# Health Watchdog — SysOps automated monitoring
# Runs every 15 minutes via cron. Alerts only on failure.
# Created: 2026-02-06

set -o pipefail

ALERT_FILE="/tmp/health-watchdog-alerts.json"
LOCK_FILE="/tmp/health-watchdog.lock"
LOG_FILE="/home/ccuser/rateright-growth/rivet/memory/watchdog-log.md"
CONFIG_PATH="/root/.clawdbot/clawdbot.json"

# Prevent concurrent runs
if [ -f "$LOCK_FILE" ]; then
    pid=$(cat "$LOCK_FILE" 2>/dev/null)
    if kill -0 "$pid" 2>/dev/null; then
        exit 0
    fi
fi
echo $$ > "$LOCK_FILE"
trap "rm -f $LOCK_FILE" EXIT

FAILURES=()
WARNINGS=()

# --- Check 1: Gateway alive ---
if ! curl -sf -o /dev/null --max-time 5 http://127.0.0.1:18789/ 2>/dev/null; then
    # Try once more after 3s (might be mid-restart)
    sleep 3
    if ! curl -sf -o /dev/null --max-time 5 http://127.0.0.1:18789/ 2>/dev/null; then
        FAILURES+=("🔴 Gateway DOWN — not responding on port 18789")
    fi
fi

# --- Check 2: Gateway systemd service ---
GW_STATE=$(systemctl --user show openclaw-gateway --property=ActiveState --value 2>/dev/null || systemctl show openclaw-gateway --property=ActiveState --value 2>/dev/null || echo "unknown")
GW_SUB=$(systemctl --user show openclaw-gateway --property=SubState --value 2>/dev/null || systemctl show openclaw-gateway --property=SubState --value 2>/dev/null || echo "unknown")
if [ "$GW_STATE" = "active" ] && [ "$GW_SUB" = "running" ]; then
    : # all good
elif [ "$GW_STATE" = "activating" ] || [ "$GW_SUB" = "auto-restart" ]; then
    WARNINGS+=("🟡 Gateway service restarting (state: ${GW_STATE}/${GW_SUB})")
elif pgrep -f "openclaw-gateway" > /dev/null 2>&1; then
    : # process alive even if systemd confused
else
    FAILURES+=("🔴 openclaw-gateway not running (systemd: ${GW_STATE}/${GW_SUB})")
fi

# --- Check 3: Disk usage ---
DISK_PCT=$(df / --output=pcent | tail -1 | tr -d ' %')
if [ "$DISK_PCT" -ge 90 ]; then
    FAILURES+=("🔴 Disk usage CRITICAL: ${DISK_PCT}%")
elif [ "$DISK_PCT" -ge 80 ]; then
    WARNINGS+=("🟡 Disk usage high: ${DISK_PCT}%")
fi

# --- Check 4: Memory available ---
MEM_AVAIL=$(free -m | awk '/^Mem:/ {print $7}')
if [ "$MEM_AVAIL" -lt 300 ]; then
    FAILURES+=("🔴 Memory CRITICAL: only ${MEM_AVAIL}MB available")
elif [ "$MEM_AVAIL" -lt 500 ]; then
    WARNINGS+=("🟡 Memory low: ${MEM_AVAIL}MB available")
fi

# --- Check 5: Gateway error rate (last 15min) ---
JOURNAL_LOG=$(journalctl --user -u openclaw-gateway --since "15 minutes ago" --no-pager 2>/dev/null || journalctl -u openclaw-gateway --since "15 minutes ago" --no-pager 2>/dev/null | grep -v "^[+]" || true)
ERROR_COUNT=$(echo "$JOURNAL_LOG" | grep -ciE "crash|restart loop|FATAL|panic" || true)
if [ "$ERROR_COUNT" -gt 5 ]; then
    FAILURES+=("🔴 Gateway error storm: ${ERROR_COUNT} critical errors in last 15min")
elif [ "$ERROR_COUNT" -gt 0 ]; then
    WARNINGS+=("🟡 Gateway errors: ${ERROR_COUNT} critical-level entries in last 15min")
fi

# --- Check 6: Model rejection errors (last 15min) ---
MODEL_ERRORS=$(echo "$JOURNAL_LOG" | grep -c "model not allowed" || true)
if [ "$MODEL_ERRORS" -gt 0 ]; then
    # Get which models
    BAD_MODELS=$(echo "$JOURNAL_LOG" | grep "model not allowed" | sed -n 's/.*model not allowed: \([^ ]*\).*/\1/p' | sort -u | tr '\n' ', ' | sed 's/,$//' || echo "")
    WARNINGS+=("🟡 Model rejections (${MODEL_ERRORS}x): ${BAD_MODELS}")
fi

# --- Check 7: Brave Search API ---
CFG_BRAVE_KEY=$(jq -r '.tools.web.search.apiKey // ""' /root/.clawdbot/clawdbot.json 2>/dev/null || true)
BRAVE_KEY="${CFG_BRAVE_KEY:-${BRAVE_API_KEY:-}}"
if [ -n "$BRAVE_KEY" ]; then
    BRAVE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 \
        -H "X-Subscription-Token: $BRAVE_KEY" \
        "https://api.search.brave.com/res/v1/web/search?q=test&count=1" 2>/dev/null)
    BRAVE_STATUS="${BRAVE_STATUS:(-3)}"  # last 3 chars only (HTTP code)
    if [ "$BRAVE_STATUS" != "200" ]; then
        WARNINGS+=("🟡 Brave Search API failing (HTTP ${BRAVE_STATUS}) — web search degraded")
    fi
else
    WARNINGS+=("🟡 Brave Search API key not found in env or config")
fi

# --- Check 8: Notion API ---
NOTION_KEY=$(jq -r '.notion.apiKey // ""' /root/.clawdbot/secrets.json 2>/dev/null || true)
if [ -n "$NOTION_KEY" ]; then
    NOTION_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 \
        -H "Authorization: Bearer $NOTION_KEY" \
        -H "Notion-Version: 2022-06-28" \
        "https://api.notion.com/v1/users/me" 2>/dev/null)
    NOTION_STATUS="${NOTION_STATUS:(-3)}"
    if [ "$NOTION_STATUS" = "401" ]; then
        WARNINGS+=("🟡 Notion API key invalid (401) — needs rotation")
    elif [ "$NOTION_STATUS" != "200" ]; then
        WARNINGS+=("🟡 Notion API failing (HTTP ${NOTION_STATUS})")
    fi
else
    WARNINGS+=("🟡 Notion API key not found in secrets.json")
fi

# --- Check 9: Growth Engine health (Railway only) ---
# Growth Engine runs on Railway, not locally
RAILWAY_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 \
    "https://rateright-growth-production.up.railway.app/health" 2>/dev/null || echo "000")
if [ "$RAILWAY_STATUS" != "200" ]; then
    FAILURES+=("🔴 Growth Engine DOWN on Railway (HTTP ${RAILWAY_STATUS})")
fi

# --- Check 9: Swap usage (indicator of memory pressure) ---
SWAP_USED=$(free -m | awk '/^Swap:/ {print $3}')
if [ "$SWAP_USED" -gt 500 ]; then
    WARNINGS+=("🟡 High swap usage: ${SWAP_USED}MB — system under memory pressure")
fi

# --- Check 10: Load average ---
LOAD_1M=$(cat /proc/loadavg | cut -d' ' -f1)
CORES=$(nproc)
LOAD_RATIO=$(echo "$LOAD_1M $CORES" | awk '{printf "%.1f", $1/$2}')
HIGH_LOAD=$(echo "$LOAD_RATIO" | awk '{print ($1 > 2.0)}')
if [ "$HIGH_LOAD" = "1" ]; then
    WARNINGS+=("🟡 High load: ${LOAD_1M} (${CORES} cores, ratio ${LOAD_RATIO})")
fi

# --- Check 11: Heartbeat idle detection ---
TODO_FILE="/home/ccuser/rateright-growth/rivet/TODO.md"
OPEN_TASKS=$(grep -c '^\- \[ \]' "$TODO_FILE" 2>/dev/null | tr -d '\n' || echo 0)
if [ "$OPEN_TASKS" -gt 0 ]; then
    # Check if recent heartbeats are spawning or just returning HEARTBEAT_OK
    MAIN_TRANSCRIPT=$(ls -t /root/.clawdbot/agents/main/sessions/*.jsonl 2>/dev/null | grep -v topic | head -1)
    if [ -n "$MAIN_TRANSCRIPT" ]; then
        # Count last 3 heartbeat responses (use temp file to avoid SIGPIPE)
        TAIL_TMP=$(mktemp)
        tail -c 50000 "$MAIN_TRANSCRIPT" > "$TAIL_TMP" 2>/dev/null || true
        RECENT_OK=$(grep -c "HEARTBEAT_OK" "$TAIL_TMP" 2>/dev/null || echo 0)
        RECENT_SPAWN=$(grep -c "sessions_spawn\|auto-executor" "$TAIL_TMP" 2>/dev/null || echo 0)
        # Clean up variables to ensure they're integers
        RECENT_OK=${RECENT_OK//[^0-9]/}
        RECENT_SPAWN=${RECENT_SPAWN//[^0-9]/}
        # Set defaults if empty
        RECENT_OK=${RECENT_OK:-0}
        RECENT_SPAWN=${RECENT_SPAWN:-0}
        rm -f "$TAIL_TMP"
        if [ "$RECENT_OK" -gt 3 ] && [ "$RECENT_SPAWN" -eq 0 ]; then
            WARNINGS+=("🟡 Heartbeat idle: ${OPEN_TASKS} open tasks but recent heartbeats returned HEARTBEAT_OK without spawning")
        fi
    fi
fi

# --- Check 12: Config integrity (the Feb 6 killer) ---
if [ -f "$CONFIG_PATH" ]; then
    # Verify it's valid JSON
    if ! jq empty "$CONFIG_PATH" 2>/dev/null; then
        FAILURES+=("🔴 clawdbot.json is INVALID JSON — gateway will crash on restart")
    else
        # Check critical keys exist
        GW_MODE=$(jq -r '.gateway.mode // ""' "$CONFIG_PATH" 2>/dev/null)
        TG_ENABLED=$(jq -r '.channels.telegram.enabled // false' "$CONFIG_PATH" 2>/dev/null)
        if [ -z "$GW_MODE" ]; then
            FAILURES+=("🔴 gateway.mode MISSING from config — gateway won't start")
        fi
        if [ "$TG_ENABLED" != "true" ]; then
            WARNINGS+=("🟡 Telegram channel not enabled in config")
        fi
    fi
fi

# Check auth-profiles has at least one working provider
if [ -f "/root/.clawdbot/auth-profiles.json" ]; then
    PROFILE_COUNT=$(jq 'length' /root/.clawdbot/auth-profiles.json 2>/dev/null || echo 0)
    if [ "$PROFILE_COUNT" -lt 1 ]; then
        FAILURES+=("🔴 auth-profiles.json is empty — no model providers configured")
    fi
elif [ -f "$CONFIG_PATH" ]; then
    AUTH_COUNT=$(jq '.auth.profiles // {} | length' "$CONFIG_PATH" 2>/dev/null || echo 0)
    if [ "$AUTH_COUNT" -lt 1 ]; then
        WARNINGS+=("🟡 No auth profiles found in config")
    fi
fi

# --- Build alert message ---
NOW=$(TZ=Australia/Sydney date '+%Y-%m-%d %H:%M AEST')
TOTAL_ISSUES=$(( ${#FAILURES[@]} + ${#WARNINGS[@]} ))

if [ "$TOTAL_ISSUES" -eq 0 ]; then
    # All clear — log silently, don't alert
    echo "[$NOW] ✅ All checks passed" >> "$LOG_FILE"
    exit 0
fi

# --- Dedup: Don't alert if same issues already alerted in last hour ---
ALERT_HASH=$(echo "${FAILURES[*]:-} ${WARNINGS[*]:-}" | md5sum | cut -d' ' -f1)
if [ -f "$ALERT_FILE" ]; then
    LAST_HASH=$(jq -r '.hash // ""' "$ALERT_FILE" 2>/dev/null || true)
    LAST_TIME=$(jq -r '.time // 0' "$ALERT_FILE" 2>/dev/null || echo 0)
    NOW_EPOCH=$(date +%s)
    ELAPSED=$(( NOW_EPOCH - LAST_TIME ))
    
    # Same issues, alerted less than 1 hour ago — skip
    if [ "$ALERT_HASH" = "$LAST_HASH" ] && [ "$ELAPSED" -lt 3600 ]; then
        echo "[$NOW] ⏭️ Same issues, suppressed (last alert ${ELAPSED}s ago)" >> "$LOG_FILE"
        exit 0
    fi
fi

# --- Format alert ---
MSG="⚡ SysOps Watchdog — ${NOW}\n"

if [ ${#FAILURES[@]} -gt 0 ]; then
    MSG+="\n🚨 CRITICAL:\n"
    for f in "${FAILURES[@]}"; do
        MSG+="  ${f}\n"
    done
fi

if [ ${#WARNINGS[@]} -gt 0 ]; then
    MSG+="\n⚠️ WARNINGS:\n"
    for w in "${WARNINGS[@]}"; do
        MSG+="  ${w}\n"
    done
fi

# --- Log it ---
echo -e "[$NOW] ${TOTAL_ISSUES} issues found" >> "$LOG_FILE"
for f in "${FAILURES[@]:-}"; do [ -n "$f" ] && echo "  $f" >> "$LOG_FILE"; done
for w in "${WARNINGS[@]:-}"; do [ -n "$w" ] && echo "  $w" >> "$LOG_FILE"; done

# --- Save alert state for dedup ---
echo "{\"hash\":\"${ALERT_HASH}\",\"time\":$(date +%s),\"issues\":${TOTAL_ISSUES}}" > "$ALERT_FILE"

# --- Output for cron delivery ---
echo -e "$MSG"