#!/usr/bin/env bash # ============================================================================== # ClassNotify System Health Check & Resource Monitor # Executed every 5 minutes by systemd timer # ============================================================================== set -euo pipefail MONITOR_ENV="/etc/classnotify/monitor.env" if [ -f "${MONITOR_ENV}" ]; then source "${MONITOR_ENV}" fi ALERT_URL="${ALERT_WEBHOOK_URL:-}" STATUS_ALL="OK" ERROR_DETAILS=() echo "==========================================================" echo " ClassNotify Health Check: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" echo "==========================================================" # 1. Check Service Status: classnotify.service if systemctl is-active --quiet classnotify.service; then echo "[PASS] classnotify.service: active" else echo "[FAIL] classnotify.service: INACTIVE" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("classnotify.service is not active") fi # 2. Check Service Status: postgresql.service if systemctl is-active --quiet postgresql.service; then echo "[PASS] postgresql.service: active" else echo "[FAIL] postgresql.service: INACTIVE" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("postgresql.service is not active") fi # 3. Check Service Status: caddy.service if systemctl is-active --quiet caddy.service; then echo "[PASS] caddy.service: active" else echo "[FAIL] caddy.service: INACTIVE" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("caddy.service is not active") fi # 4. Check HTTP Health Endpoint (/api/health) HEALTH_JSON=$(curl -s -m 5 http://127.0.0.1:5000/api/health || true) SRV_STATUS=$(echo "${HEALTH_JSON}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${SRV_STATUS}" = "Healthy Connected" ]; then echo "[PASS] /api/health: Healthy, database: Connected" else echo "[FAIL] /api/health: status '${SRV_STATUS}'" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("/api/health returned '${SRV_STATUS}'") fi # 5. Check Disk Usage DISK_ROOT=$(df -P / | tail -n 1 | awk '{print $5}' | tr -d '%') DISK_BAK=$(df -P /var/backups | tail -n 1 | awk '{print $5}' | tr -d '%') MAX_DISK=$(( DISK_ROOT > DISK_BAK ? DISK_ROOT : DISK_BAK )) if [ "${MAX_DISK}" -lt 75 ]; then echo "[PASS] Disk Usage: ${MAX_DISK}% (threshold: <75% OK)" elif [ "${MAX_DISK}" -le 85 ]; then echo "[WARN] Disk Usage: ${MAX_DISK}% (threshold: 75-85% WARN)" if [ "${STATUS_ALL}" != "CRITICAL" ]; then STATUS_ALL="WARN"; fi else echo "[FAIL] Disk Usage: ${MAX_DISK}% (threshold: >85% CRITICAL)" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("Disk usage ${MAX_DISK}% exceeds critical threshold 85%") fi # 6. Check Database Backup Freshness & Integrity DAILY_DIR="/var/backups/classnotify/daily" LATEST_BACKUP=$(ls -t "${DAILY_DIR}"/classnotify_*.sql.gz 2>/dev/null | head -n 1 || true) if [ -n "${LATEST_BACKUP}" ] && [ -f "${LATEST_BACKUP}" ]; then NOW_TS=$(date +%s) FILE_TS=$(stat -c %Y "${LATEST_BACKUP}") AGE_SEC=$(( NOW_TS - FILE_TS )) AGE_HOURS=$(( AGE_SEC / 3600 )) FILE_SIZE=$(stat -c %s "${LATEST_BACKUP}") UNCOMP_SIZE=$(zcat "${LATEST_BACKUP}" 2>/dev/null | wc -c || echo 0) echo "Latest Backup: $(basename "${LATEST_BACKUP}")" echo " Backup Age: ${AGE_HOURS} hours (threshold: <26h OK)" echo " Backup Size: ${FILE_SIZE} bytes compressed (${UNCOMP_SIZE} bytes uncompressed)" if [ "${AGE_HOURS}" -ge 26 ]; then echo "[FAIL] Backup age ${AGE_HOURS}h exceeds 26-hour limit!" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("Latest backup is ${AGE_HOURS} hours old (> 26h)") else echo "[PASS] Backup age is within 26 hours" fi # Raw dump must be at least 10KB (10240 bytes) if [ "${UNCOMP_SIZE}" -lt 10240 ] && [ "${FILE_SIZE}" -lt 10240 ]; then echo "[FAIL] Backup size is abnormally small (< 10KB)!" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("Backup size abnormally small: ${FILE_SIZE} bytes") else echo "[PASS] Backup size is healthy (>= 10KB uncompressed data)" fi else echo "[FAIL] No daily backups found in ${DAILY_DIR}!" STATUS_ALL="CRITICAL" ERROR_DETAILS+=("No database backups found") fi echo "==========================================================" echo "Health Check Summary: ${STATUS_ALL}" echo "==========================================================" if [ "${STATUS_ALL}" = "CRITICAL" ]; then ERR_MSG=$(printf "; %s" "${ERROR_DETAILS[@]}") ERR_MSG="${ERR_MSG:2}" logger -p local0.crit -t classnotify-healthcheck "HEALTHCHECK CRITICAL: ${ERR_MSG}" if [ -n "${ALERT_URL}" ]; then echo "Sending webhook alert to ${ALERT_URL}..." curl -s -X POST "${ALERT_URL}" \ -H "Content-Type: application/json" \ -d "{\"service\":\"ClassNotify\",\"status\":\"CRITICAL\",\"errors\":\"${ERR_MSG}\"}" || true else echo "External alert destination not configured" fi exit 1 elif [ "${STATUS_ALL}" = "WARN" ]; then logger -p local0.warn -t classnotify-healthcheck "HEALTHCHECK WARNING" exit 0 else echo "All monitored subsystems healthy." exit 0 fi