From 95c83a57b6d6a6896cf5611ca3251c1fb0c10d3d Mon Sep 17 00:00:00 2001 From: olsch01 Date: Thu, 9 Apr 2026 09:05:45 -0400 Subject: [PATCH] feat: add production deploy script with auto-rollback and Gitea Actions workflow Add automated production deployment pipeline: - scripts/deploy-prod.sh: Full deployment script with pre/post DB backups, migration tracking via shared.schema_migrations table, health checks, and automatic rollback on failure (restores DB, reverts code, rebuilds) - .gitea/workflows/deploy.yml: Manual-trigger Gitea Actions workflow for intentional production deployments with optional --seed-existing flag - scripts/db-backup.sh: Add --yes/-y flag to skip interactive confirmation prompts, enabling automated restore during rollback Co-Authored-By: Claude Opus 4.6 --- .gitea/workflows/deploy.yml | 65 ++++++ scripts/db-backup.sh | 11 +- scripts/deploy-prod.sh | 410 ++++++++++++++++++++++++++++++++++++ 3 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 .gitea/workflows/deploy.yml create mode 100755 scripts/deploy-prod.sh diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..c30749b --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,65 @@ +# --------------------------------------------------------------------------- +# Production Deployment Workflow for HOA LedgerIQ +# +# Trigger: Manual only (workflow_dispatch) — production deploys are intentional. +# Runner: Self-hosted on the production server at /opt/hoa-ledgeriq. +# +# This workflow does NOT use actions/checkout. The runner operates directly +# on the production directory. The deploy script itself handles git pull. +# --------------------------------------------------------------------------- + +name: Deploy to Production + +on: + workflow_dispatch: + inputs: + seed_existing: + description: "Mark existing migrations as applied without running them (first deployment only)" + required: false + default: "false" + type: boolean + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + defaults: + run: + working-directory: /opt/hoa-ledgeriq + + steps: + - name: Pre-deploy info + run: | + echo "## Pre-Deploy Info" >> $GITHUB_STEP_SUMMARY + echo "- **Server:** $(hostname)" >> $GITHUB_STEP_SUMMARY + echo "- **Directory:** $(pwd)" >> $GITHUB_STEP_SUMMARY + echo "- **Current commit:** $(git rev-parse --short HEAD)" >> $GITHUB_STEP_SUMMARY + echo "- **Branch:** $(git branch --show-current || echo 'detached')" >> $GITHUB_STEP_SUMMARY + echo "- **Triggered by:** ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY + echo "- **Seed existing:** ${{ inputs.seed_existing }}" >> $GITHUB_STEP_SUMMARY + echo "- **Started at:** $(date -Iseconds)" >> $GITHUB_STEP_SUMMARY + + - name: Run deployment + run: | + DEPLOY_FLAGS="" + if [ "${{ inputs.seed_existing }}" = "true" ]; then + DEPLOY_FLAGS="--seed-existing" + fi + bash scripts/deploy-prod.sh $DEPLOY_FLAGS + env: + TERM: xterm + + - name: Deployment result + if: always() + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Deployment Result" >> $GITHUB_STEP_SUMMARY + if [ "${{ job.status }}" = "success" ]; then + echo "- **Status:** Successful" >> $GITHUB_STEP_SUMMARY + echo "- **Commit:** $(git rev-parse --short HEAD)" >> $GITHUB_STEP_SUMMARY + else + echo "- **Status:** FAILED (auto-rollback triggered)" >> $GITHUB_STEP_SUMMARY + echo "- **Commit (after rollback):** $(git rev-parse --short HEAD)" >> $GITHUB_STEP_SUMMARY + echo "- Check the deploy log on the server for details" >> $GITHUB_STEP_SUMMARY + fi + echo "- **Completed at:** $(date -Iseconds)" >> $GITHUB_STEP_SUMMARY diff --git a/scripts/db-backup.sh b/scripts/db-backup.sh index 84e097f..e4185ec 100755 --- a/scripts/db-backup.sh +++ b/scripts/db-backup.sh @@ -21,6 +21,7 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" BACKUP_DIR="$PROJECT_DIR/backups" KEEP_DAYS=0 # 0 = keep forever +FORCE_YES=false # skip interactive confirmations (for automation) DB_USER="${POSTGRES_USER:-hoafinance}" DB_NAME="${POSTGRES_DB:-hoafinance}" COMPOSE_CMD="docker compose" @@ -121,8 +122,12 @@ do_restore() { warn "This will DESTROY the current '${DB_NAME}' database and replace it" warn "with the contents of: $(basename "$file")" echo "" - read -rp "Type 'yes' to continue: " confirm - [ "$confirm" = "yes" ] || { info "Aborted."; exit 0; } + if [ "$FORCE_YES" = true ]; then + info "Skipping confirmation (--yes flag set)" + else + read -rp "Type 'yes' to continue: " confirm + [ "$confirm" = "yes" ] || { info "Aborted."; exit 0; } + fi echo "" info "Step 1/4 — Terminating active connections ..." @@ -229,6 +234,7 @@ Usage: Options: --dir DIR Backup directory (default: ./backups) --keep DAYS Auto-delete backups older than DAYS (default: keep all) + --yes, -y Skip interactive confirmation prompts (for automation) Supported restore formats: .dump.gz Custom-format pg_dump, gzipped (default backup format) @@ -255,6 +261,7 @@ while [ $# -gt 0 ]; do case "$1" in --dir) BACKUP_DIR="$2"; shift 2 ;; --keep) KEEP_DAYS="$2"; shift 2 ;; + --yes|-y) FORCE_YES=true; shift ;; --help) usage ;; *) if [ "$COMMAND" = "restore" ] && [ -z "$RESTORE_FILE" ]; then diff --git a/scripts/deploy-prod.sh b/scripts/deploy-prod.sh new file mode 100755 index 0000000..7e9619c --- /dev/null +++ b/scripts/deploy-prod.sh @@ -0,0 +1,410 @@ +#!/usr/bin/env bash +# --------------------------------------------------------------------------- +# deploy-prod.sh — Production deployment script for HOA LedgerIQ +# +# Usage: +# ./scripts/deploy-prod.sh [--seed-existing] +# +# This script performs a full production deployment: +# 1. Takes a pre-upgrade database backup +# 2. Pulls the latest code from the main branch +# 3. Rebuilds and restarts Docker containers +# 4. Runs any pending database migrations (tracked in shared.schema_migrations) +# 5. Verifies the application is healthy +# 6. Takes a post-upgrade database backup +# +# On failure (migration error or health check), the script automatically: +# - Restores the pre-upgrade database backup +# - Reverts the code to the previous commit +# - Rebuilds containers from the reverted code +# +# Flags: +# --seed-existing Mark all existing migration files as applied without +# executing them. Use this ONLY on the first deployment +# against an existing database where migrations were +# previously applied manually. +# +# Environment: +# PROJECT_DIR Override the project directory (default: /opt/hoa-ledgeriq) +# POSTGRES_USER Database user (default: hoafinance) +# POSTGRES_DB Database name (default: hoafinance) +# --------------------------------------------------------------------------- + +set -euo pipefail + +# ---- Defaults ---- +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="${PROJECT_DIR:-/opt/hoa-ledgeriq}" +COMPOSE_CMD="docker compose -f $PROJECT_DIR/docker-compose.yml -f $PROJECT_DIR/docker-compose.prod.yml" +DB_USER="${POSTGRES_USER:-hoafinance}" +DB_NAME="${POSTGRES_DB:-hoafinance}" +MIGRATION_DIR="$PROJECT_DIR/db/migrations" +HEALTH_URL="http://localhost:3000/api" +HEALTH_RETRIES=20 +HEALTH_INTERVAL=5 +HEALTH_START_WAIT=30 +LOG_DIR="$PROJECT_DIR/logs" +LOG_FILE="$LOG_DIR/deploy-$(date +%Y%m%d_%H%M%S).log" + +# State tracking +SEED_EXISTING=false +PREV_COMMIT="" +BACKUP_FILE="" +ROLLBACK_NEEDED=false +DEPLOY_SUCCESS=false +DEPLOY_START_TIME="" + +# ---- Colors ---- +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' + +# ---- Logging ---- +log() { echo -e "$(date -Iseconds) ${CYAN}[DEPLOY]${NC} $*"; } +ok() { echo -e "$(date -Iseconds) ${GREEN}[OK]${NC} $*"; } +warn() { echo -e "$(date -Iseconds) ${YELLOW}[WARN]${NC} $*"; } +err() { echo -e "$(date -Iseconds) ${RED}[ERROR]${NC} $*" >&2; } +die() { err "$@"; exit 1; } + +# ---- Parse flags ---- +while [ $# -gt 0 ]; do + case "$1" in + --seed-existing) SEED_EXISTING=true; shift ;; + --help|-h) + head -35 "$0" | tail -33 + exit 0 + ;; + *) die "Unknown argument: $1" ;; + esac +done + +# ---- Setup logging ---- +mkdir -p "$LOG_DIR" +exec > >(tee -a "$LOG_FILE") 2>&1 + +# ---- Cleanup / Rollback trap ---- +cleanup() { + if [ "$DEPLOY_SUCCESS" = true ]; then + return 0 + fi + + if [ "$ROLLBACK_NEEDED" = true ] && [ -n "$BACKUP_FILE" ]; then + echo "" + err "==========================================" + err " DEPLOYMENT FAILED — STARTING ROLLBACK" + err "==========================================" + echo "" + + # Step 1: Restore the pre-upgrade database backup + log "Restoring database from pre-upgrade backup: $(basename "$BACKUP_FILE")" + if "$SCRIPT_DIR/db-backup.sh" restore --yes "$BACKUP_FILE"; then + ok "Database restored successfully" + else + err "DATABASE RESTORE FAILED — manual intervention required!" + err "Backup file: $BACKUP_FILE" + exit 1 + fi + + # Step 2: Revert code to previous commit + if [ -n "$PREV_COMMIT" ]; then + log "Reverting code to previous commit: $PREV_COMMIT" + cd "$PROJECT_DIR" + git reset --hard "$PREV_COMMIT" + ok "Code reverted to $PREV_COMMIT" + fi + + # Step 3: Rebuild containers from old code + log "Rebuilding containers from reverted code ..." + cd "$PROJECT_DIR" + $COMPOSE_CMD up -d --build + ok "Containers rebuilt from previous version" + + echo "" + err "Rollback complete. The system is restored to the pre-deployment state." + err "Review the deploy log for details: $LOG_FILE" + exit 1 + elif [ "$ROLLBACK_NEEDED" = true ]; then + err "Rollback needed but no backup file available — manual intervention required!" + exit 1 + fi +} + +trap cleanup EXIT + +# ==================================================================== +# STEP 1: Pre-flight checks +# ==================================================================== +log "============================================" +log " HOA LedgerIQ — Production Deployment" +log "============================================" +log "Project directory: $PROJECT_DIR" +log "Timestamp: $(date -Iseconds)" +DEPLOY_START_TIME=$(date +%s) +echo "" + +cd "$PROJECT_DIR" + +# Verify prerequisites +command -v git >/dev/null 2>&1 || die "git is not installed" +command -v docker >/dev/null 2>&1 || die "docker is not installed" +docker compose version >/dev/null 2>&1 || die "docker compose is not available" + +# Verify we're in a git repo +[ -d ".git" ] || die "$PROJECT_DIR is not a git repository" + +# Verify postgres is running +if ! $COMPOSE_CMD ps postgres 2>/dev/null | grep -q "running\|Up"; then + die "PostgreSQL container is not running. Start it with: $COMPOSE_CMD up -d postgres" +fi + +# Store current commit for rollback +PREV_COMMIT=$(git rev-parse HEAD) +log "Current commit: $PREV_COMMIT" + +# ==================================================================== +# STEP 2: Pre-upgrade database backup +# ==================================================================== +echo "" +log "--- Step 1/6: Pre-upgrade database backup ---" + +BACKUP_OUTPUT=$("$SCRIPT_DIR/db-backup.sh" backup 2>&1) +echo "$BACKUP_OUTPUT" + +# Extract the backup file path from the output (strip ANSI color codes first) +BACKUP_FILE=$(echo "$BACKUP_OUTPUT" | sed 's/\x1b\[[0-9;]*m//g' | grep -oP 'Backup complete: \K\S+' || true) + +if [ -z "$BACKUP_FILE" ]; then + die "Failed to capture backup file path from db-backup.sh output" +fi + +if [ ! -f "$BACKUP_FILE" ]; then + die "Backup file does not exist: $BACKUP_FILE" +fi + +ok "Pre-upgrade backup saved: $(basename "$BACKUP_FILE")" + +# From this point forward, rollback is possible +ROLLBACK_NEEDED=true + +# ==================================================================== +# STEP 3: Pull latest code +# ==================================================================== +echo "" +log "--- Step 2/6: Pulling latest code from main ---" + +git fetch origin main +git reset --hard origin/main + +NEW_COMMIT=$(git rev-parse HEAD) +log "Updated to commit: $NEW_COMMIT" + +if [ "$PREV_COMMIT" = "$NEW_COMMIT" ]; then + warn "No new commits — continuing anyway (migrations or rebuilds may still be needed)" +fi + +# ==================================================================== +# STEP 4: Rebuild and restart containers +# ==================================================================== +echo "" +log "--- Step 3/6: Rebuilding and restarting containers ---" + +$COMPOSE_CMD up -d --build + +# Wait for postgres to be healthy before running migrations +log "Waiting for PostgreSQL to be healthy ..." +PG_RETRIES=30 +PG_COUNT=0 +while [ $PG_COUNT -lt $PG_RETRIES ]; do + if $COMPOSE_CMD exec -T postgres pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + ok "PostgreSQL is ready" + break + fi + ((PG_COUNT++)) + if [ $PG_COUNT -eq $PG_RETRIES ]; then + die "PostgreSQL did not become healthy after $((PG_RETRIES * 2))s" + fi + sleep 2 +done + +# ==================================================================== +# STEP 5: Run database migrations +# ==================================================================== +echo "" +log "--- Step 4/6: Running database migrations ---" + +# Helper: run SQL via psql in the postgres container +run_sql() { + $COMPOSE_CMD exec -T postgres psql -U "$DB_USER" -d "$DB_NAME" -v ON_ERROR_STOP=1 --quiet "$@" +} + +# Step 5a: Ensure the migration tracking table exists +log "Ensuring shared.schema_migrations table exists ..." +run_sql <<'SQL' +CREATE SCHEMA IF NOT EXISTS shared; +CREATE TABLE IF NOT EXISTS shared.schema_migrations ( + id SERIAL PRIMARY KEY, + filename TEXT NOT NULL UNIQUE, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + checksum TEXT +); +SQL +ok "Migration tracking table ready" + +# Step 5b: Get list of already-applied migrations +declare -A APPLIED_MIGRATIONS +while IFS= read -r fname; do + fname=$(echo "$fname" | xargs) # trim whitespace + [ -n "$fname" ] && APPLIED_MIGRATIONS["$fname"]=1 +done < <(run_sql -t -c "SELECT filename FROM shared.schema_migrations ORDER BY filename;") + +APPLIED_COUNT=${#APPLIED_MIGRATIONS[@]} +log "Previously applied migrations: $APPLIED_COUNT" + +# Step 5c: Scan migration directory for .sql files +MIGRATION_FILES=() +if [ -d "$MIGRATION_DIR" ]; then + while IFS= read -r f; do + MIGRATION_FILES+=("$(basename "$f")") + done < <(find "$MIGRATION_DIR" -name "*.sql" -type f | sort) +fi + +TOTAL_MIGRATIONS=${#MIGRATION_FILES[@]} +log "Total migration files found: $TOTAL_MIGRATIONS" + +# Step 5d: Handle --seed-existing (first deployment only) +if [ "$SEED_EXISTING" = true ]; then + if [ "$APPLIED_COUNT" -gt 0 ]; then + warn "--seed-existing flag set but $APPLIED_COUNT migrations are already tracked. Skipping seed." + else + log "Seeding migration tracking table with ${TOTAL_MIGRATIONS} existing migration files ..." + for filename in "${MIGRATION_FILES[@]}"; do + checksum=$(md5sum "$MIGRATION_DIR/$filename" | awk '{print $1}') + run_sql -c "INSERT INTO shared.schema_migrations (filename, checksum) VALUES ('$filename', '$checksum') ON CONFLICT (filename) DO NOTHING;" + log " Seeded: $filename" + done + ok "All existing migrations marked as applied (not executed)" + # Refresh the applied list + APPLIED_COUNT=$TOTAL_MIGRATIONS + for filename in "${MIGRATION_FILES[@]}"; do + APPLIED_MIGRATIONS["$filename"]=1 + done + fi +fi + +# Step 5e: Detect first-run without --seed-existing +if [ "$APPLIED_COUNT" -eq 0 ] && [ "$TOTAL_MIGRATIONS" -gt 0 ] && [ "$SEED_EXISTING" = false ]; then + warn "The migration tracking table is empty but $TOTAL_MIGRATIONS migration files exist." + warn "If these migrations were previously applied manually, re-run with --seed-existing" + warn "to register them without re-executing. Otherwise, all migrations will be applied." + warn "" + warn "Continuing in 10 seconds ... (Ctrl+C to abort)" + sleep 10 +fi + +# Step 5f: Apply pending migrations +PENDING_COUNT=0 +APPLIED_THIS_RUN=0 + +for filename in "${MIGRATION_FILES[@]}"; do + if [ -n "${APPLIED_MIGRATIONS[$filename]+x}" ]; then + continue + fi + ((PENDING_COUNT++)) +done + +if [ "$PENDING_COUNT" -eq 0 ]; then + ok "No pending migrations to apply" +else + log "$PENDING_COUNT pending migration(s) to apply" + echo "" + + for filename in "${MIGRATION_FILES[@]}"; do + if [ -n "${APPLIED_MIGRATIONS[$filename]+x}" ]; then + continue + fi + + checksum=$(md5sum "$MIGRATION_DIR/$filename" | awk '{print $1}') + log " Applying: $filename ..." + + # Run the migration in a single transaction with error stopping + if cat "$MIGRATION_DIR/$filename" | $COMPOSE_CMD exec -T postgres psql \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -v ON_ERROR_STOP=1 \ + --single-transaction \ + --quiet 2>&1; then + + # Record successful migration + run_sql -c "INSERT INTO shared.schema_migrations (filename, checksum) VALUES ('$filename', '$checksum');" + ok " Applied: $filename" + ((APPLIED_THIS_RUN++)) + else + err "Migration FAILED: $filename" + err "Triggering automatic rollback ..." + exit 1 # trap will handle rollback + fi + done + + echo "" + ok "Successfully applied $APPLIED_THIS_RUN migration(s)" +fi + +# ==================================================================== +# STEP 6: Health check +# ==================================================================== +echo "" +log "--- Step 5/6: Verifying application health ---" +log "Waiting ${HEALTH_START_WAIT}s for backend to initialize ..." +sleep "$HEALTH_START_WAIT" + +HEALTHY=false +for ((i=1; i<=HEALTH_RETRIES; i++)); do + if curl -sf "$HEALTH_URL" >/dev/null 2>&1; then + HEALTHY=true + break + fi + log " Health check attempt $i/$HEALTH_RETRIES failed, retrying in ${HEALTH_INTERVAL}s ..." + sleep "$HEALTH_INTERVAL" +done + +if [ "$HEALTHY" = true ]; then + ok "Backend is healthy and responding at $HEALTH_URL" +else + err "Backend failed to respond after $((HEALTH_START_WAIT + HEALTH_RETRIES * HEALTH_INTERVAL))s" + err "Triggering automatic rollback ..." + exit 1 # trap will handle rollback +fi + +# Also verify the container reports healthy via Docker +if $COMPOSE_CMD ps backend 2>/dev/null | grep -q "healthy"; then + ok "Backend container health check: healthy" +else + warn "Backend container health status is not 'healthy' yet (may still be within start_period)" +fi + +# ==================================================================== +# STEP 7: Post-upgrade database backup +# ==================================================================== +echo "" +log "--- Step 6/6: Post-upgrade database backup ---" + +"$SCRIPT_DIR/db-backup.sh" backup + +# ==================================================================== +# Deployment complete +# ==================================================================== +DEPLOY_SUCCESS=true +ROLLBACK_NEEDED=false + +DEPLOY_END_TIME=$(date +%s) +DEPLOY_DURATION=$((DEPLOY_END_TIME - DEPLOY_START_TIME)) + +echo "" +log "============================================" +ok " DEPLOYMENT COMPLETE" +log "============================================" +log " Previous commit : $PREV_COMMIT" +log " Current commit : $NEW_COMMIT" +log " Migrations run : $APPLIED_THIS_RUN" +log " Duration : ${DEPLOY_DURATION}s" +log " Log file : $LOG_FILE" +log "============================================" +echo "" -- 2.49.1