#!/usr/bin/env bash # ============================================================================== # ClassNotify Systemd Unit Deployment & Auto-Rollback Guard # Usage: ./deploy_systemd_unit.sh # Exit codes: # 0 = Systemd Unit Deploy SUCCESS # 12 = Systemd Unit Deploy FAILED (Health Gate Failed, Rollback SUCCEEDED) # 22 = Systemd Unit Deploy FAILED and Rollback FAILED (Critical Incident) # 32 = Candidate Pre-Validation FAILED (Production Untouched) # ============================================================================== set -euo pipefail CANDIDATE_UNIT="${1:-/tmp/classnotify_candidate.service}" PROD_UNIT="/etc/systemd/system/classnotify.service" BACKUP_DIR="/etc/systemd/system/backups" echo "==========================================================" echo " ClassNotify Systemd Unit Deployment & Verification Guard" echo "==========================================================" if [ ! -f "${CANDIDATE_UNIT}" ]; then echo "ERROR: Candidate unit file ${CANDIDATE_UNIT} not found!" >&2 exit 1 fi # 1. Candidate Pre-Validation using systemd-analyze verify echo "[1/4] Executing 'systemd-analyze verify' on candidate unit..." VERIFY_OUTPUT="" set +e VERIFY_OUTPUT=$(systemd-analyze verify "${CANDIDATE_UNIT}" 2>&1) VERIFY_EXIT=$? set -e if [ "${VERIFY_EXIT}" -ne 0 ]; then echo "${VERIFY_OUTPUT}" echo "" echo "==========================================================" echo "ERROR: Candidate systemd unit validation FAILED (exit ${VERIFY_EXIT})!" echo "Production ${PROD_UNIT} is 100% UNTOUCHED." echo "SYSTEMD_PREVALIDATION=FAILED" echo "==========================================================" rm -f "${CANDIDATE_UNIT}" exit 32 fi echo " Candidate pre-validation PASSED! (Production untouched so far)" # 2. Validation passed: Create backup of current unit echo "[2/4] Creating unit backup and atomically activating candidate..." mkdir -p "${BACKUP_DIR}" TS=$(date +%Y%m%d_%H%M%S) BACKUP_FILE="${BACKUP_DIR}/classnotify.service.${TS}" if [ -f "${PROD_UNIT}" ]; then cp "${PROD_UNIT}" "${BACKUP_FILE}" echo " Backup saved: ${BACKUP_FILE}" fi # Atomically replace production unit cp "${CANDIDATE_UNIT}" "${PROD_UNIT}.new" chmod 644 "${PROD_UNIT}.new" mv -Tf "${PROD_UNIT}.new" "${PROD_UNIT}" # Reload systemd and restart service systemctl daemon-reload systemctl restart classnotify echo " Systemd reloaded and classnotify restarted with candidate unit." # 3. Post-Restart Health Gate (Local + Public) echo "[3/4] Executing Post-Activation Health Gate..." HEALTH_PASS=0 for i in $(seq 1 10); do sleep 2 echo " Health Gate check (attempt $i/10)..." # Layer 1: Local Kestrel Health Endpoint LOCAL_RESP=$(curl -s -m 5 http://127.0.0.1:5000/api/health || true) LOCAL_STATUS=$(echo "${LOCAL_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${LOCAL_STATUS}" != "Healthy Connected" ]; then echo " Local endpoint waiting: '${LOCAL_STATUS}'" continue fi # Layer 2: Public HTTPS Health Endpoint through Caddy PUBLIC_RESP=$(curl -s -m 5 https://classnotify.paradise.eu.org/api/health || true) PUBLIC_STATUS=$(echo "${PUBLIC_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${PUBLIC_STATUS}" = "Healthy Connected" ]; then echo " Health Gate PASSED: Both Local and Public endpoints reporting Healthy and Connected." HEALTH_PASS=1 break else echo " Public endpoint waiting: '${PUBLIC_STATUS}'" fi done # 4. Branch on health gate result if [ "${HEALTH_PASS}" -eq 1 ]; then rm -f "${CANDIDATE_UNIT}" echo "" echo "==========================================================" echo "SYSTEMD_DEPLOY=SUCCESS" echo "UNIT_SHA256=$(sha256sum "${PROD_UNIT}" | awk '{print toupper($1)}')" echo "==========================================================" exit 0 else echo "" echo "==========================================================" echo "=== [CRITICAL] Systemd Health Gate Failed! Initiating Rollback ===" echo "==========================================================" ROLLBACK_PASS=0 if [ -f "${BACKUP_FILE}" ]; then echo "Restoring ${PROD_UNIT} from backup ${BACKUP_FILE}..." cp "${BACKUP_FILE}" "${PROD_UNIT}.new" chmod 644 "${PROD_UNIT}.new" mv -Tf "${PROD_UNIT}.new" "${PROD_UNIT}" systemctl daemon-reload systemctl restart classnotify echo " Restored previous systemd unit and restarted service." # Verify rollback health echo "Executing Rollback Health Gate verification..." for r_i in $(seq 1 10); do sleep 2 echo " Rollback Health Gate check (attempt ${r_i}/10)..." R_LOCAL_RESP=$(curl -s -m 5 http://127.0.0.1:5000/api/health || true) R_LOCAL_STATUS=$(echo "${R_LOCAL_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${R_LOCAL_STATUS}" != "Healthy Connected" ]; then echo " Rollback local endpoint waiting: '${R_LOCAL_STATUS}'" continue fi R_PUB_RESP=$(curl -s -m 5 https://classnotify.paradise.eu.org/api/health || true) R_PUB_STATUS=$(echo "${R_PUB_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${R_PUB_STATUS}" = "Healthy Connected" ]; then echo " Rollback Health Gate PASSED: Unit restored, service Healthy and Connected." ROLLBACK_PASS=1 break else echo " Rollback public endpoint waiting: '${R_PUB_STATUS}'" fi done else echo "CRITICAL ERROR: No backup unit available to restore!" fi rm -f "${CANDIDATE_UNIT}" if [ "${ROLLBACK_PASS}" -eq 1 ]; then echo "" echo "==========================================================" echo "SYSTEMD_DEPLOY=FAILED" echo "SYSTEMD_ROLLBACK=SUCCESS" echo "FINAL_HEALTH=Healthy" echo "FINAL_UNIT_SHA256=$(sha256sum "${PROD_UNIT}" | awk '{print toupper($1)}')" echo "==========================================================" exit 12 else echo "" echo "==========================================================" echo "SYSTEMD_DEPLOY=FAILED" echo "SYSTEMD_ROLLBACK=FAILED" echo "FINAL_HEALTH=Unhealthy" echo "FINAL_UNIT_SHA256=$(sha256sum "${PROD_UNIT}" | awk '{print toupper($1)}')" echo "==========================================================" exit 22 fi fi