#!/bin/bash
# Fleet Doctor — Diagnostic-first agent recovery
# Replaces the 4 disabled crons (stall-detector, crash-loop-guard, buddy-check, health-check)
# with ONE smart script that diagnoses before acting.
#
# Philosophy: Diagnose first, fix specifically, verify, escalate if failed.
# Lesson from Feb 18: Blind restarts corrupted Telegram offsets and CAUSED the fleet outage.
#
# Run via cron every 5 minutes:
#   */5 * * * * /home/ccuser/rateright-growth/rivet/scripts/fleet-doctor.sh >> /home/ccuser/rateright-growth/rivet/memory/fleet-doctor.log 2>&1
#
# Created: 2026-02-18

set -o pipefail

# --- Configuration ---
LOG_FILE="/home/ccuser/rateright-growth/rivet/memory/fleet-doctor.log"
STATE_DIR="/tmp/fleet-doctor"
COOLDOWN_SECONDS=600  # Don't fix same agent twice in 10 min
MAX_LOG_LINES=2000    # Rotate log when it gets big
NOW=$(date +%s)
NOW_HUMAN=$(TZ=Australia/Sydney date '+%Y-%m-%d %H:%M AEST')

mkdir -p "$STATE_DIR"

# --- Agent Registry ---
# Format: name:port:service:config_dir:workspace
AGENTS=(
  "rivet:18789:clawdbot-gateway:/root/.clawdbot:/home/ccuser/rateright-growth"
  "builder:18790:clawdbot-builder:/root/.clawdbot-builder:/home/ccuser/the-50-dollar-app"
  "susan:18792:clawdbot-susan:/root/.clawdbot-susan:/home/ccuser/susan"
  "harper:18796:clawdbot-harper:/root/.clawdbot-harper:/home/ccuser/harper"
  "herald:18808:clawdbot-herald:/root/.clawdbot-herald:/home/ccuser/herald"
)

# --- Counters ---
TOTAL_CHECKED=0
TOTAL_HEALTHY=0
TOTAL_FIXED=0
TOTAL_FAILED=0
FIXES_APPLIED=()
FAILURES=()

# --- Helpers ---
log() {
  echo "[$NOW_HUMAN] $1"
}

log_agent() {
  local agent="$1"
  local msg="$2"
  echo "[$NOW_HUMAN] [$agent] $msg"
}

is_on_cooldown() {
  local agent="$1"
  local cooldown_file="$STATE_DIR/${agent}.cooldown"
  if [ -f "$cooldown_file" ]; then
    local last_fix=$(cat "$cooldown_file" 2>/dev/null || echo 0)
    local elapsed=$((NOW - last_fix))
    if [ "$elapsed" -lt "$COOLDOWN_SECONDS" ]; then
      return 0  # ON cooldown
    fi
  fi
  return 1  # OFF cooldown
}

set_cooldown() {
  local agent="$1"
  echo "$NOW" > "$STATE_DIR/${agent}.cooldown"
}

# Track consecutive failures for escalation
get_failure_count() {
  local agent="$1"
  local fail_file="$STATE_DIR/${agent}.failures"
  cat "$fail_file" 2>/dev/null || echo 0
}

increment_failure() {
  local agent="$1"
  local fail_file="$STATE_DIR/${agent}.failures"
  local count=$(get_failure_count "$agent")
  echo $((count + 1)) > "$fail_file"
}

clear_failure() {
  local agent="$1"
  rm -f "$STATE_DIR/${agent}.failures"
}

# --- Diagnostic Functions ---

# Level 0: Is the service running?
check_service() {
  local service="$1"
  systemctl is-active --quiet "$service" 2>/dev/null
}

# Level 0: Is the port responding to HTTP?
check_port() {
  local port="$1"
  curl -sf -o /dev/null --max-time 5 "http://127.0.0.1:${port}/" 2>/dev/null
}

# Level 1: Is the config valid JSON?
check_config() {
  local config_dir="$1"
  local config_file="${config_dir}/clawdbot.json"
  if [ ! -f "$config_file" ]; then
    echo "MISSING"
    return 1
  fi
  if ! jq empty "$config_file" 2>/dev/null; then
    echo "INVALID_JSON"
    return 1
  fi
  echo "OK"
  return 0
}

# Level 1: Are model fallbacks Anthropic-only?
check_model_fallbacks() {
  local config_dir="$1"
  local config_file="${config_dir}/clawdbot.json"
  if [ ! -f "$config_file" ]; then
    return 1
  fi
  # Check if any non-anthropic models are in the config
  local bad_models=$(jq -r '.. | .model? // empty' "$config_file" 2>/dev/null | grep -ivE "anthropic|claude|sonnet|opus|haiku|openrouter|minimax|deepseek|moonshot|kimi|google|gemini|grok" | head -3)
  if [ -n "$bad_models" ]; then
    echo "$bad_models"
    return 1
  fi
  return 0
}

# Level 2: Is the Telegram offset potentially corrupted?
check_telegram_offset() {
  local config_dir="$1"
  local offset_file="${config_dir}/telegram/update-offset-default.json"
  if [ ! -f "$offset_file" ]; then
    echo "MISSING"
    return 1
  fi
  if ! jq empty "$offset_file" 2>/dev/null; then
    echo "INVALID_JSON"
    return 1
  fi
  echo "OK"
  return 0
}

# Level 2: Does sessions.json exist and is it valid?
check_sessions() {
  local config_dir="$1"
  local sessions_file="${config_dir}/agents/main/sessions/sessions.json"
  if [ ! -f "$sessions_file" ]; then
    echo "MISSING"
    return 1
  fi
  if ! jq empty "$sessions_file" 2>/dev/null; then
    echo "INVALID_JSON"
    return 1
  fi
  # Check for stale auth overrides (non-anthropic)
  local bad_auth=$(jq -r '.. | .authProfileOverride? // empty' "$sessions_file" 2>/dev/null | grep -ivE "anthropic|openrouter|minimax|deepseek|moonshot|kimi|google|gemini|grok" | head -1)
  if [ -n "$bad_auth" ]; then
    echo "STALE_AUTH:$bad_auth"
    return 1
  fi
  echo "OK"
  return 0
}

# Level 3: Check status.json freshness (agents write this every heartbeat)
check_status_freshness() {
  local workspace="$1"
  local status_file="${workspace}/status.json"
  if [ ! -f "$status_file" ]; then
    echo "NO_STATUS_FILE"
    return 1
  fi
  local last_update=$(jq -r '.last_heartbeat // .last_updated // .timestamp // ""' "$status_file" 2>/dev/null)
  if [ -z "$last_update" ]; then
    echo "NO_TIMESTAMP"
    return 1
  fi
  # Parse the timestamp and check if it's older than 30 min
  local update_epoch=$(date -d "$last_update" +%s 2>/dev/null || echo 0)
  if [ "$update_epoch" -eq 0 ]; then
    echo "UNPARSEABLE_TIMESTAMP"
    return 1
  fi
  local age=$((NOW - update_epoch))
  if [ "$age" -gt 1800 ]; then
    echo "STALE:${age}s"
    return 1
  fi
  echo "FRESH:${age}s"
  return 0
}

# --- Fix Functions ---
# Each fix is targeted, minimal, and logged.

fix_start_service() {
  local agent="$1"
  local service="$2"
  local port="$3"
  log_agent "$agent" "FIX: Starting stopped service $service"

  # Check for orphan process on the port first
  local orphan_pid=$(ss -tlnp "sport = :${port}" 2>/dev/null | grep -oP 'pid=\K[0-9]+' | head -1)
  if [ -n "$orphan_pid" ]; then
    log_agent "$agent" "FIX: Killing orphan process $orphan_pid on port $port"
    kill "$orphan_pid" 2>/dev/null || true
    sleep 2
  fi

  systemctl start "$service" 2>/dev/null
  sleep 10

  if check_port "$port"; then
    log_agent "$agent" "FIX: Service started, port $port responding"
    return 0
  else
    log_agent "$agent" "FIX: Service started but port $port not responding"
    return 1
  fi
}

fix_config_rollback() {
  local agent="$1"
  local service="$2"
  local config_dir="$3"
  local port="$4"
  local backup_dir="${config_dir}/config-backups"

  # Try standard backup location first, fall back to /root/.clawdbot/config-backups
  if [ ! -d "$backup_dir" ]; then
    backup_dir="/root/.clawdbot/config-backups"
  fi

  local latest_backup=$(ls -1t "${backup_dir}"/config-2*.json 2>/dev/null | head -1)
  if [ -z "$latest_backup" ]; then
    log_agent "$agent" "FIX: No config backup found, cannot rollback"
    return 1
  fi

  if ! jq empty "$latest_backup" 2>/dev/null; then
    log_agent "$agent" "FIX: Latest backup is invalid JSON"
    return 1
  fi

  log_agent "$agent" "FIX: Rolling back config from $(basename "$latest_backup")"
  local config_file="${config_dir}/clawdbot.json"

  # Save broken config
  cp "$config_file" "${config_dir}/clawdbot-BROKEN-$(date +%Y%m%d-%H%M%S).json" 2>/dev/null || true
  cp "$latest_backup" "$config_file"

  systemctl restart "$service" 2>/dev/null
  sleep 10

  if check_port "$port"; then
    log_agent "$agent" "FIX: Config rollback + restart successful"
    return 0
  fi
  return 1
}

fix_telegram_offset() {
  local agent="$1"
  local service="$2"
  local config_dir="$3"
  local port="$4"
  local offset_file="${config_dir}/telegram/update-offset-default.json"

  log_agent "$agent" "FIX: Resetting Telegram offset (most common silent failure fix)"

  systemctl stop "$service" 2>/dev/null || true
  sleep 3

  # Backup old offset
  if [ -f "$offset_file" ]; then
    cp "$offset_file" "${offset_file}.bak.$(date +%s)" 2>/dev/null || true
  fi

  # Reset to clean state
  mkdir -p "$(dirname "$offset_file")"
  echo '{"version":1}' > "$offset_file"

  systemctl start "$service" 2>/dev/null
  sleep 10

  if check_port "$port"; then
    log_agent "$agent" "FIX: Telegram offset reset + restart successful"
    return 0
  fi
  return 1
}

fix_sessions() {
  local agent="$1"
  local service="$2"
  local config_dir="$3"
  local port="$4"
  local sessions_file="${config_dir}/agents/main/sessions/sessions.json"

  log_agent "$agent" "FIX: Fixing sessions.json"

  local issue=$(check_sessions "$config_dir")

  if [[ "$issue" == "MISSING" ]]; then
    log_agent "$agent" "FIX: Creating missing sessions.json"
    mkdir -p "$(dirname "$sessions_file")"
    echo '{}' > "$sessions_file"
  elif [[ "$issue" == "INVALID_JSON" ]]; then
    log_agent "$agent" "FIX: sessions.json is invalid, archiving and recreating"
    mv "$sessions_file" "${sessions_file}.bak.$(date +%s)" 2>/dev/null || true
    echo '{}' > "$sessions_file"
  elif [[ "$issue" == STALE_AUTH:* ]]; then
    local bad_auth="${issue#STALE_AUTH:}"
    log_agent "$agent" "FIX: Clearing stale auth override ($bad_auth)"
    local tmp=$(mktemp)
    jq 'walk(if type == "object" then del(.authProfileOverride, .authProfileOverrideSource, .authProfileOverrideCompactionCount) else . end)' "$sessions_file" > "$tmp" 2>/dev/null && mv "$tmp" "$sessions_file"
  fi

  systemctl restart "$service" 2>/dev/null
  sleep 10

  if check_port "$port"; then
    log_agent "$agent" "FIX: Sessions fix + restart successful"
    return 0
  fi
  return 1
}

fix_nuclear() {
  local agent="$1"
  local service="$2"
  local config_dir="$3"
  local port="$4"

  log_agent "$agent" "FIX: Nuclear reset (offset + sessions + restart)"

  systemctl stop "$service" 2>/dev/null || true
  sleep 3

  # Kill any orphan processes
  pkill -f "clawdbot.*$(echo "$service" | sed 's/clawdbot-//')" 2>/dev/null || true
  sleep 2

  # Reset Telegram offset
  local offset_file="${config_dir}/telegram/update-offset-default.json"
  if [ -f "$offset_file" ]; then
    cp "$offset_file" "${offset_file}.bak.$(date +%s)" 2>/dev/null || true
  fi
  mkdir -p "$(dirname "$offset_file")"
  echo '{"version":1}' > "$offset_file"

  # Archive and recreate sessions
  local sessions_file="${config_dir}/agents/main/sessions/sessions.json"
  if [ -f "$sessions_file" ]; then
    mv "$sessions_file" "${sessions_file}.bak.$(date +%s)" 2>/dev/null || true
  fi
  mkdir -p "$(dirname "$sessions_file")"
  echo '{}' > "$sessions_file"

  systemctl start "$service" 2>/dev/null
  sleep 15

  if check_port "$port"; then
    log_agent "$agent" "FIX: Nuclear reset successful"
    return 0
  fi
  return 1
}

# --- Main Diagnostic Loop ---
log "Fleet Doctor starting — checking ${#AGENTS[@]} agents"

for entry in "${AGENTS[@]}"; do
  IFS=':' read -r name port service config_dir workspace <<< "$entry"
  TOTAL_CHECKED=$((TOTAL_CHECKED + 1))

  # --- Skip if on cooldown ---
  if is_on_cooldown "$name"; then
    log_agent "$name" "SKIP: On cooldown (fixed recently)"
    continue
  fi

  # --- Level 0: Is it alive? ---
  service_up=false
  port_up=false

  if check_service "$service"; then
    service_up=true
  fi

  if check_port "$port"; then
    port_up=true
  fi

  # Happy path: service running AND port responding
  if $service_up && $port_up; then
    clear_failure "$name"
    TOTAL_HEALTHY=$((TOTAL_HEALTHY + 1))
    continue
  fi

  # --- Something's wrong. Diagnose. ---
  log_agent "$name" "ISSUE: service=$service_up port=$port_up"

  # Track failures for escalation
  increment_failure "$name"
  local fail_count=$(get_failure_count "$name")

  # --- Level 1: Service not running at all ---
  if ! $service_up; then
    # Check config first
    config_status=$(check_config "$config_dir")
    if [ "$config_status" = "INVALID_JSON" ] || [ "$config_status" = "MISSING" ]; then
      log_agent "$name" "DIAG: Config is $config_status"
      if fix_config_rollback "$name" "$service" "$config_dir" "$port"; then
        FIXES_APPLIED+=("$name: config rollback")
        TOTAL_FIXED=$((TOTAL_FIXED + 1))
        set_cooldown "$name"
        clear_failure "$name"
        continue
      fi
    fi

    # Config is fine, just start the service
    if fix_start_service "$name" "$service" "$port"; then
      FIXES_APPLIED+=("$name: started service")
      TOTAL_FIXED=$((TOTAL_FIXED + 1))
      set_cooldown "$name"
      clear_failure "$name"
      continue
    fi
  fi

  # --- Level 2: Service up but port not responding ---
  if $service_up && ! $port_up; then
    log_agent "$name" "DIAG: Service running but port $port not responding (zombie)"

    # Check sessions.json first (less destructive)
    sessions_status=$(check_sessions "$config_dir")
    if [ "$sessions_status" != "OK" ]; then
      log_agent "$name" "DIAG: Sessions issue: $sessions_status"
      if fix_sessions "$name" "$service" "$config_dir" "$port"; then
        FIXES_APPLIED+=("$name: fixed sessions ($sessions_status)")
        TOTAL_FIXED=$((TOTAL_FIXED + 1))
        set_cooldown "$name"
        clear_failure "$name"
        continue
      fi
    fi

    # Try Telegram offset reset (most common fix per Feb 18 lessons)
    if fix_telegram_offset "$name" "$service" "$config_dir" "$port"; then
      FIXES_APPLIED+=("$name: reset Telegram offset")
      TOTAL_FIXED=$((TOTAL_FIXED + 1))
      set_cooldown "$name"
      clear_failure "$name"
      continue
    fi
  fi

  # --- Level 3: Nothing worked so far, try nuclear ---
  if [ "$fail_count" -ge 2 ]; then
    log_agent "$name" "DIAG: Multiple failures ($fail_count), trying nuclear reset"
    if fix_nuclear "$name" "$service" "$config_dir" "$port"; then
      FIXES_APPLIED+=("$name: nuclear reset")
      TOTAL_FIXED=$((TOTAL_FIXED + 1))
      set_cooldown "$name"
      clear_failure "$name"
      continue
    fi
  fi

  # --- Level 4: Everything failed. Escalate. ---
  log_agent "$name" "FAILED: All recovery attempts exhausted (failure #$fail_count)"
  FAILURES+=("$name (port $port, service $service) — $fail_count consecutive failures")
  TOTAL_FAILED=$((TOTAL_FAILED + 1))
  set_cooldown "$name"
done

# --- Also check RateRight App (non-agent, just auto-restart) ---
APP_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 "http://localhost:3000" 2>/dev/null || echo "000")
if [ "$APP_STATUS" != "200" ]; then
  log "APP: RateRight app down (HTTP $APP_STATUS), restarting..."
  systemctl restart rateright-app 2>/dev/null || true
  sleep 5
  APP_VERIFY=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 "http://localhost:3000" 2>/dev/null || echo "000")
  if [ "$APP_VERIFY" = "200" ]; then
    log "APP: Restarted successfully"
    FIXES_APPLIED+=("rateright-app: restarted")
    TOTAL_FIXED=$((TOTAL_FIXED + 1))
    chown -R ccuser:ccuser /home/ccuser/the-50-dollar-app/.next/ /home/ccuser/the-50-dollar-app/.git/ 2>/dev/null || true
  else
    log "APP: Still down after restart (HTTP $APP_VERIFY)"
    FAILURES+=("rateright-app — still down after restart")
    TOTAL_FAILED=$((TOTAL_FAILED + 1))
  fi
fi

# --- Also check Growth Engine on Railway ---
GE_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 "https://rateright-growth-production.up.railway.app/health" 2>/dev/null || echo "000")
if [ "$GE_STATUS" != "200" ]; then
  log "GROWTH ENGINE: Down on Railway (HTTP $GE_STATUS) — cannot auto-fix, alerting"
  FAILURES+=("Growth Engine on Railway — HTTP $GE_STATUS")
  TOTAL_FAILED=$((TOTAL_FAILED + 1))
fi

# --- Summary ---
log "Fleet Doctor complete: ${TOTAL_HEALTHY}/${TOTAL_CHECKED} healthy, ${TOTAL_FIXED} fixed, ${TOTAL_FAILED} failed"

# --- Escalate if there were failures ---
if [ ${#FAILURES[@]} -gt 0 ]; then
  ALERT_MSG="Fleet Doctor Alert — ${NOW_HUMAN}\n\n"

  if [ ${#FIXES_APPLIED[@]} -gt 0 ]; then
    ALERT_MSG+="Auto-fixed:\n"
    for fix in "${FIXES_APPLIED[@]}"; do
      ALERT_MSG+="  + $fix\n"
    done
    ALERT_MSG+="\n"
  fi

  ALERT_MSG+="NEEDS ATTENTION:\n"
  for fail in "${FAILURES[@]}"; do
    ALERT_MSG+="  ! $fail\n"
  done

  # Write to Herald's inbox for Telegram escalation
  INBOX_FILE="/home/ccuser/shared/inboxes/herald.jsonl"
  if [ -d "$(dirname "$INBOX_FILE")" ]; then
    echo "{\"from\":\"fleet-doctor\",\"subject\":\"Fleet Alert\",\"body\":\"$(echo -e "$ALERT_MSG" | tr '\n' ' ')\",\"priority\":\"high\",\"timestamp\":\"$NOW_HUMAN\"}" >> "$INBOX_FILE"
  fi

  # Also write to Rivet's inbox
  RIVET_INBOX="/home/ccuser/shared/inboxes/rivet.jsonl"
  if [ -d "$(dirname "$RIVET_INBOX")" ]; then
    echo "{\"from\":\"fleet-doctor\",\"subject\":\"Fleet Alert\",\"body\":\"$(echo -e "$ALERT_MSG" | tr '\n' ' ')\",\"priority\":\"high\",\"timestamp\":\"$NOW_HUMAN\"}" >> "$RIVET_INBOX"
  fi

  # Output for cron delivery
  echo -e "$ALERT_MSG"
fi

# --- Log rotation ---
if [ -f "$LOG_FILE" ]; then
  LINE_COUNT=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
  if [ "$LINE_COUNT" -gt "$MAX_LOG_LINES" ]; then
    tail -n 500 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
    log "Log rotated (was $LINE_COUNT lines, kept last 500)"
  fi
fi