#!/bin/bash
# launch-monitor.sh — Cog's launch day watchdog
# Cron: */5 * * * * bash /home/ccuser/cog/scripts/launch-monitor.sh
# Created: 2026-02-20 by Cog

set -o pipefail

LOGFILE="/home/ccuser/cog/memory/launch-monitor.log"
ALERT_FILE="/home/ccuser/cog/memory/launch-alerts.jsonl"
NOW=$(TZ=Australia/Sydney date '+%Y-%m-%d %H:%M:%S AEDT')
FAILURES=0
WARNINGS=0
ALERTS=""

log() { echo "[$NOW] $1" >> "$LOGFILE"; }
alert() {
  local severity="$1" target="$2" msg="$3"
  echo "{\"ts\":\"$NOW\",\"severity\":\"$severity\",\"target\":\"$target\",\"msg\":\"$msg\"}" >> "$ALERT_FILE"
  ALERTS="${ALERTS}${severity}: ${msg} -> ${target}\n"
  log "ALERT [$severity] $msg -> $target"
}

# --- Tier 1: CRITICAL ---

# App HTTP check (external)
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://rateright.com.au 2>/dev/null || echo "000")
RESPONSE_TIME=$(curl -s -o /dev/null -w "%{time_total}" --max-time 10 https://rateright.com.au 2>/dev/null || echo "99")

if [ "$HTTP_CODE" != "200" ]; then
  alert "CRITICAL" "sentinel" "rateright.com.au returned HTTP $HTTP_CODE"
  ((FAILURES++))
fi

RT_INT=$(echo "$RESPONSE_TIME" | cut -d. -f1)
if [ "${RT_INT:-0}" -ge 5 ]; then
  alert "WARNING" "sentinel" "Response time ${RESPONSE_TIME}s (>5s threshold)"
  ((WARNINGS++))
fi

# App via reverse proxy (internal)
HTTP_INTERNAL=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 https://rivet.rateright.com.au 2>/dev/null || echo "000")
if [ "$HTTP_INTERNAL" != "200" ]; then
  alert "WARNING" "sentinel" "rivet.rateright.com.au returned HTTP $HTTP_INTERNAL"
  ((WARNINGS++))
fi

# Process check
APP_STATUS=$(systemctl is-active rateright-app 2>/dev/null || echo "unknown")
if [ "$APP_STATUS" != "active" ]; then
  alert "CRITICAL" "sentinel" "rateright-app service: $APP_STATUS"
  ((FAILURES++))
fi

# Error log spike (last 5 min)
ERROR_COUNT=$(journalctl -u rateright-app --since "5 min ago" 2>/dev/null | grep -ci "error\|fatal\|unhandled" || echo "0")
if [ "$ERROR_COUNT" -gt 10 ]; then
  alert "HIGH" "builder" "Error spike: $ERROR_COUNT errors in 5 min"
  ((WARNINGS++))
fi

# Nginx
NGINX_STATUS=$(systemctl is-active nginx 2>/dev/null || echo "unknown")
if [ "$NGINX_STATUS" != "active" ]; then
  alert "CRITICAL" "sentinel" "nginx is $NGINX_STATUS"
  ((FAILURES++))
fi

# --- System Resources ---

MEM_PCT=$(free | awk '/Mem:/{printf "%.0f", $3/$2*100}' 2>/dev/null || echo "0")
if [ "${MEM_PCT:-0}" -gt 85 ]; then
  alert "HIGH" "sentinel" "RAM ${MEM_PCT}% (>85%)"
  ((WARNINGS++))
fi

DISK_PCT=$(df / | awk 'NR==2{print $5}' | tr -d '%' 2>/dev/null || echo "0")
if [ "${DISK_PCT:-0}" -gt 80 ]; then
  alert "HIGH" "sentinel" "Disk ${DISK_PCT}% (>80%)"
  ((WARNINGS++))
fi

LOAD=$(uptime | awk -F'load average: ' '{print $2}' | cut -d, -f1 | tr -d ' ')
LOAD_INT=$(echo "$LOAD" | cut -d. -f1)
if [ "${LOAD_INT:-0}" -gt 4 ]; then
  alert "WARNING" "sentinel" "Load $LOAD (>4.0 on 2-vCPU)"
  ((WARNINGS++))
fi

# --- Fleet Agents ---
AGENTS_DOWN=0
for port in 18789 18790 18792 18796 18800 18804 18808 18812; do
  if ! curl -s --max-time 3 http://localhost:$port/api/health >/dev/null 2>&1; then
    ((AGENTS_DOWN++))
  fi
done
if [ "$AGENTS_DOWN" -gt 2 ]; then
  alert "HIGH" "cog" "Fleet: $AGENTS_DOWN agents not responding on health endpoint"
  ((WARNINGS++))
fi

# --- PM2 conflict check ---
PM2_RUNNING=$(pm2 jlist 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(len([p for p in d if 'rateright' in p.get('name','')]))" 2>/dev/null || echo "0")
if [ "${PM2_RUNNING:-0}" -gt 0 ]; then
  alert "WARNING" "sentinel" "PM2 still managing rateright-app (port conflict risk)"
  ((WARNINGS++))
fi

# --- Summary line ---
log "CHECK: HTTP=$HTTP_CODE (${RESPONSE_TIME}s) | svc=$APP_STATUS | nginx=$NGINX_STATUS | errors=$ERROR_COUNT | RAM=${MEM_PCT}% | disk=${DISK_PCT}% | load=$LOAD | fleet_down=$AGENTS_DOWN | failures=$FAILURES warnings=$WARNINGS"

# Write to pending alerts if critical failures
if [ "$FAILURES" -gt 0 ]; then
  echo "{\"ts\":\"$NOW\",\"failures\":$FAILURES,\"warnings\":$WARNINGS}" >> /home/ccuser/cog/memory/launch-pending-alerts.jsonl
fi