#!/bin/bash # Maintenance mode migration script # Enables maintenance mode, runs migrations, then restores service # # Usage: ./maintenance-migrate.sh [backup] # backup: If set, creates a database backup before migration # # This script is for BREAKING migrations that require downtime. # Non-breaking migrations run automatically on container start. # # Exit codes: # 0 - Migration successful # 1 - Migration failed (maintenance mode will remain active) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CREATE_BACKUP="${1:-}" COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yml" COMPOSE_BLUE_GREEN="$PROJECT_ROOT/docker-compose.blue-green.yml" STATE_FILE="$PROJECT_ROOT/config/deployment/state.json" NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh" TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml" BACKUP_DIR="$PROJECT_ROOT/data/backups" echo "========================================" echo "MAINTENANCE MODE MIGRATION" echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" echo "========================================" # Determine active stack if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE") else ACTIVE_STACK="blue" fi BACKEND_CONTAINER="mvp-backend-$ACTIVE_STACK" echo "" echo "Configuration:" echo " Active stack: $ACTIVE_STACK" echo " Backend container: $BACKEND_CONTAINER" echo " Create backup: ${CREATE_BACKUP:-no}" echo "" # Step 1: Send maintenance notification echo "Step 1/6: Sending maintenance notification..." echo "----------------------------------------" if [[ -x "$NOTIFY_SCRIPT" ]]; then "$NOTIFY_SCRIPT" "maintenance_start" "Starting maintenance window for database migration" || true fi echo " OK" # Step 2: Enable maintenance mode echo "" echo "Step 2/6: Enabling maintenance mode..." echo "----------------------------------------" # Update state file if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then jq '.maintenance_mode = true' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" fi # Stop traffic to both stacks (weight 0) echo " Stopping traffic to application stacks..." # Set both stacks to weight 0 - Traefik will return 503 sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" rm -f "${TRAEFIK_CONFIG}.bak" # Wait for Traefik to pick up changes sleep 3 echo " OK: Maintenance mode active" # Step 3: Create database backup (if requested) if [[ -n "$CREATE_BACKUP" ]]; then echo "" echo "Step 3/6: Creating database backup..." echo "----------------------------------------" mkdir -p "$BACKUP_DIR" BACKUP_FILE="$BACKUP_DIR/pre-migration-$(date +%Y%m%d-%H%M%S).sql" if docker exec mvp-postgres pg_dump -U postgres motovaultpro > "$BACKUP_FILE"; then echo " OK: Backup created: $BACKUP_FILE" ls -lh "$BACKUP_FILE" else echo " ERROR: Backup failed" echo "" echo "Aborting migration. Restoring traffic..." # Restore traffic if [[ "$ACTIVE_STACK" == "blue" ]]; then sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" else sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" fi if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then jq '.maintenance_mode = false' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" fi exit 1 fi else echo "" echo "Step 3/6: Skipping backup (not requested)..." echo "----------------------------------------" echo " OK" fi # Step 4: Run migrations echo "" echo "Step 4/6: Running database migrations..." echo "----------------------------------------" # Flush Redis cache before migration echo " Flushing Redis cache..." docker exec mvp-redis redis-cli FLUSHALL > /dev/null 2>&1 || true # Run migrations echo " Running migrations..." if docker exec "$BACKEND_CONTAINER" npm run migrate; then echo " OK: Migrations completed" else echo " ERROR: Migration failed" echo "" echo "Migration failed. Maintenance mode remains active." echo "To restore:" echo " 1. Fix migration issues" echo " 2. Re-run migrations: docker exec $BACKEND_CONTAINER npm run migrate" echo " 3. Run: $0 to retry, or manually restore traffic" if [[ -x "$NOTIFY_SCRIPT" ]]; then "$NOTIFY_SCRIPT" "failure" "Database migration failed during maintenance window. Manual intervention required." || true fi exit 1 fi # Step 5: Restart backend containers echo "" echo "Step 5/6: Restarting backend containers..." echo "----------------------------------------" # Restart both backends to pick up any schema changes docker restart mvp-backend-blue mvp-backend-green 2>/dev/null || true # Wait for backends to be healthy echo " Waiting for backends to be healthy..." sleep 10 for container in mvp-backend-blue mvp-backend-green; do for i in {1..12}; do health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown") if [[ "$health" == "healthy" ]]; then echo " OK: $container is healthy" break fi if [[ $i -eq 12 ]]; then echo " WARNING: $container health check timeout" fi sleep 5 done done # Step 6: Disable maintenance mode echo "" echo "Step 6/6: Restoring traffic..." echo "----------------------------------------" # Restore traffic to active stack if [[ "$ACTIVE_STACK" == "blue" ]]; then sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" else sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" fi # Update state file if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") jq --arg ts "$TIMESTAMP" '.maintenance_mode = false | .last_migration = $ts' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" fi # Wait for Traefik to pick up changes sleep 3 echo " OK: Traffic restored" # Send completion notification if [[ -x "$NOTIFY_SCRIPT" ]]; then "$NOTIFY_SCRIPT" "maintenance_end" "Maintenance window complete. Database migration successful." || true fi echo "" echo "========================================" echo "MAINTENANCE MIGRATION COMPLETE" echo "========================================" echo "" echo "Summary:" echo " - Migrations: Successful" echo " - Active stack: $ACTIVE_STACK" echo " - Maintenance mode: Disabled" if [[ -n "$CREATE_BACKUP" ]]; then echo " - Backup: $BACKUP_FILE" fi echo "" exit 0