#!/usr/bin/env bash # ============================================================================== # ClassNotify Atomic Release Deployment & Automated Rollback Controller # Usage: ./deploy_server.sh # Exit codes: # 0 = Deployment SUCCESS # 10 = Deployment FAILED, Rollback SUCCEEDED and Health Verified # 20 = Deployment FAILED and Rollback FAILED (Critical Incident) # 30 = Archive SHA256 Mismatch (Untrusted / Corrupted Package) # ============================================================================== set -euo pipefail RELEASE_ID="$1" TARBALL="$2" EXPECTED_SHA256="${3:-}" BASE_DIR="/var/www/classnotify" RELEASES_DIR="${BASE_DIR}/releases" CURRENT_LINK="${BASE_DIR}/current" TARGET_DIR="${RELEASES_DIR}/${RELEASE_ID}" echo "==========================================================" echo " ClassNotify Atomic Server Deployment: ${RELEASE_ID}" echo "==========================================================" # 1. Verification of inputs if [ -z "${RELEASE_ID}" ] || [ -z "${TARBALL}" ] || [ -z "${EXPECTED_SHA256}" ]; then echo "ERROR: Missing required arguments. Usage: $0 " >&2 exit 1 fi if [ ! -f "${TARBALL}" ]; then echo "ERROR: Tarball archive ${TARBALL} not found!" >&2 exit 1 fi # 2. Server-side SHA256 Package Integrity Verification echo "[1/6] Verifying release package SHA256 integrity..." ACTUAL_SHA256=$(sha256sum "${TARBALL}" | awk '{print tolower($1)}') NORM_EXPECTED=$(echo "${EXPECTED_SHA256}" | tr '[:upper:]' '[:lower:]') if [ "${ACTUAL_SHA256}" != "${NORM_EXPECTED}" ]; then echo "ERROR: release archive SHA256 mismatch!" >&2 echo " Expected SHA256: ${NORM_EXPECTED}" >&2 echo " Actual SHA256: ${ACTUAL_SHA256}" >&2 rm -f "${TARBALL}" exit 30 fi echo " SHA256 Verified: ${ACTUAL_SHA256}" mkdir -p "${RELEASES_DIR}" if [ -d "${TARGET_DIR}" ]; then echo "ERROR: Target release directory ${TARGET_DIR} already exists!" >&2 exit 1 fi # 3. Extract into isolated target directory echo "[2/6] Extracting release package to ${TARGET_DIR}..." mkdir -p "${TARGET_DIR}" tar -xzf "${TARBALL}" -C "${TARGET_DIR}" rm -f "${TARBALL}" echo "${RELEASE_ID}" > "${TARGET_DIR}/release.txt" # Ensure www-data ownership chown -R www-data:www-data "${TARGET_DIR}" # Verify binary integrity if [ ! -f "${TARGET_DIR}/ClassNotify.Server.dll" ]; then echo "ERROR: Critical binary ClassNotify.Server.dll missing in release package!" >&2 rm -rf "${TARGET_DIR}" exit 1 fi # 4. Record existing active release for potential rollback echo "[3/6] Recording previous active release..." OLD_RELEASE="" if [ -L "${CURRENT_LINK}" ]; then OLD_RELEASE=$(readlink -f "${CURRENT_LINK}" 2>/dev/null || true) fi echo " Previous release: ${OLD_RELEASE:-}" # 5. Atomically switch 'current' symlink echo "[4/6] Atomically updating current symlink..." START_TIME=$(date +%s%3N) ln -sfn "${TARGET_DIR}" "${BASE_DIR}/current.new" mv -Tf "${BASE_DIR}/current.new" "${CURRENT_LINK}" # 6. Restart service and perform Health Gate validation echo "[5/6] Restarting classnotify.service and executing Health Gate..." systemctl restart classnotify HEALTH_PASS=0 for i in $(seq 1 10); do sleep 2 echo " Health Gate check (attempt $i/10)..." # Layer 1: Local Kestrel Health Endpoint LOCAL_RESP=$(curl -s -m 5 http://127.0.0.1:5000/api/health || true) LOCAL_STATUS=$(echo "${LOCAL_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${LOCAL_STATUS}" != "Healthy Connected" ]; then echo " Local endpoint waiting: '${LOCAL_STATUS}'" continue fi # Layer 2: Public HTTPS Health Endpoint through Caddy PUBLIC_RESP=$(curl -s -m 5 https://classnotify.paradise.eu.org/api/health || true) PUBLIC_STATUS=$(echo "${PUBLIC_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${PUBLIC_STATUS}" = "Healthy Connected" ]; then echo " Health Gate PASSED: Both Local and Public endpoints reporting Healthy and Connected." HEALTH_PASS=1 break else echo " Public endpoint waiting: '${PUBLIC_STATUS}'" fi done END_TIME=$(date +%s%3N) DOWNTIME_MS=$(( END_TIME - START_TIME )) # 7. Branch on health gate result if [ "${HEALTH_PASS}" -eq 1 ]; then echo "[6/6] Deployment Successful! Pruning old releases..." # Release Retention Policy: keep latest 5 releases, never delete current or previous cd "${RELEASES_DIR}" for dir in $(ls -dt */ 2>/dev/null | tail -n +6); do REL_NAME="${dir%/}" FULL_PATH="${RELEASES_DIR}/${REL_NAME}" if [ "${FULL_PATH}" != "${TARGET_DIR}" ] && [ "${FULL_PATH}" != "${OLD_RELEASE}" ]; then echo " Pruning old release: ${REL_NAME}" rm -rf "${FULL_PATH}" fi done echo "" echo "==========================================================" echo "DEPLOY SUCCESS" echo "RELEASE_ID=${RELEASE_ID}" echo "PREVIOUS_RELEASE_ID=$(basename "${OLD_RELEASE:-none}")" echo "CURRENT_SYMLINK=$(readlink -f "${CURRENT_LINK}")" echo "DEPLOYMENT_DOWNTIME_MS=${DOWNTIME_MS}" echo "==========================================================" exit 0 else echo "" echo "==========================================================" echo "=== [CRITICAL] Health Gate Failed! Initiating Rollback ===" echo "==========================================================" ROLLBACK_START=$(date +%s%3N) ROLLBACK_PASS=0 ROLLBACK_ATTEMPTS=0 if [ -n "${OLD_RELEASE}" ] && [ -d "${OLD_RELEASE}" ]; then echo "Reverting 'current' symlink to ${OLD_RELEASE}..." ln -sfn "${OLD_RELEASE}" "${BASE_DIR}/current.new" mv -Tf "${BASE_DIR}/current.new" "${CURRENT_LINK}" systemctl restart classnotify # Rigorous Rollback Health Gate Verification echo "Executing Rollback Health Gate validation..." for r_i in $(seq 1 10); do sleep 2 ROLLBACK_ATTEMPTS="${r_i}" echo " Rollback Health Gate check (attempt ${r_i}/10)..." R_LOCAL_RESP=$(curl -s -m 5 http://127.0.0.1:5000/api/health || true) R_LOCAL_STATUS=$(echo "${R_LOCAL_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${R_LOCAL_STATUS}" != "Healthy Connected" ]; then echo " Rollback local endpoint waiting: '${R_LOCAL_STATUS}'" continue fi R_PUB_RESP=$(curl -s -m 5 https://classnotify.paradise.eu.org/api/health || true) R_PUB_STATUS=$(echo "${R_PUB_RESP}" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status',''), data.get('database',''))" 2>/dev/null || true) if [ "${R_PUB_STATUS}" = "Healthy Connected" ]; then echo " Rollback Health Gate PASSED: Old release verified Healthy and Connected." ROLLBACK_PASS=1 break else echo " Rollback public endpoint waiting: '${R_PUB_STATUS}'" fi done else echo "CRITICAL ERROR: No previous release available to restore!" fi ROLLBACK_END=$(date +%s%3N) ROLLBACK_DURATION_MS=$(( ROLLBACK_END - ROLLBACK_START )) # Mark broken release directory with .failed suffix if [ -d "${TARGET_DIR}" ]; then mv "${TARGET_DIR}" "${TARGET_DIR}.failed" echo "Marked broken release as ${TARGET_DIR}.failed" fi echo "" echo "==========================================================" if [ "${ROLLBACK_PASS}" -eq 1 ]; then echo "DEPLOY FAILED" echo "ROLLBACK_STATUS=SUCCESS" echo "BROKEN_RELEASE_ID=${RELEASE_ID}" echo "ROLLBACK_RELEASE=$(basename "${OLD_RELEASE:-none}")" echo "FINAL_ACTIVE_RELEASE=$(basename "$(readlink -f "${CURRENT_LINK}")")" echo "FINAL_HEALTH=Healthy" echo "ROLLBACK_HEALTH_ATTEMPTS=${ROLLBACK_ATTEMPTS}" echo "ROLLBACK_DURATION_MS=${ROLLBACK_DURATION_MS}" echo "==========================================================" exit 10 else echo "DEPLOY FAILED" echo "ROLLBACK_STATUS=FAILED" echo "BROKEN_RELEASE_ID=${RELEASE_ID}" echo "FINAL_ACTIVE_RELEASE=$(basename "$(readlink -f "${CURRENT_LINK}")")" echo "FINAL_HEALTH=Unhealthy" echo "ROLLBACK_DURATION_MS=${ROLLBACK_DURATION_MS}" echo "==========================================================" logger -p local0.crit -t classnotify-deploy "DEPLOYMENT AND ROLLBACK FAILED for release ${RELEASE_ID}" exit 20 fi fi