221 lines
7.3 KiB
Bash
Executable File
221 lines
7.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Maintenance mode migration script
|
|
# Enables maintenance mode, runs migrations, then restores service
|
|
#
|
|
# Usage: ./maintenance-migrate.sh [backup]
|
|
# backup: If set, creates a database backup before migration
|
|
#
|
|
# This script is for BREAKING migrations that require downtime.
|
|
# Non-breaking migrations run automatically on container start.
|
|
#
|
|
# Exit codes:
|
|
# 0 - Migration successful
|
|
# 1 - Migration failed (maintenance mode will remain active)
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
CREATE_BACKUP="${1:-}"
|
|
|
|
COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yml"
|
|
COMPOSE_BLUE_GREEN="$PROJECT_ROOT/docker-compose.blue-green.yml"
|
|
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
|
NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh"
|
|
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
|
|
BACKUP_DIR="$PROJECT_ROOT/data/backups"
|
|
|
|
echo "========================================"
|
|
echo "MAINTENANCE MODE MIGRATION"
|
|
echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|
echo "========================================"
|
|
|
|
# Determine active stack
|
|
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
|
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
|
|
else
|
|
ACTIVE_STACK="blue"
|
|
fi
|
|
|
|
BACKEND_CONTAINER="mvp-backend-$ACTIVE_STACK"
|
|
|
|
echo ""
|
|
echo "Configuration:"
|
|
echo " Active stack: $ACTIVE_STACK"
|
|
echo " Backend container: $BACKEND_CONTAINER"
|
|
echo " Create backup: ${CREATE_BACKUP:-no}"
|
|
echo ""
|
|
|
|
# Step 1: Send maintenance notification
|
|
echo "Step 1/6: Sending maintenance notification..."
|
|
echo "----------------------------------------"
|
|
|
|
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
|
"$NOTIFY_SCRIPT" "maintenance_start" "Starting maintenance window for database migration" || true
|
|
fi
|
|
echo " OK"
|
|
|
|
# Step 2: Enable maintenance mode
|
|
echo ""
|
|
echo "Step 2/6: Enabling maintenance mode..."
|
|
echo "----------------------------------------"
|
|
|
|
# Update state file
|
|
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
|
jq '.maintenance_mode = true' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
|
fi
|
|
|
|
# Stop traffic to both stacks (weight 0)
|
|
echo " Stopping traffic to application stacks..."
|
|
|
|
# Set both stacks to weight 0 - Traefik will return 503
|
|
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
|
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
|
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
|
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
|
rm -f "${TRAEFIK_CONFIG}.bak"
|
|
|
|
# Wait for Traefik to pick up changes
|
|
sleep 3
|
|
echo " OK: Maintenance mode active"
|
|
|
|
# Step 3: Create database backup (if requested)
|
|
if [[ -n "$CREATE_BACKUP" ]]; then
|
|
echo ""
|
|
echo "Step 3/6: Creating database backup..."
|
|
echo "----------------------------------------"
|
|
|
|
mkdir -p "$BACKUP_DIR"
|
|
BACKUP_FILE="$BACKUP_DIR/pre-migration-$(date +%Y%m%d-%H%M%S).sql"
|
|
|
|
if docker exec mvp-postgres pg_dump -U postgres motovaultpro > "$BACKUP_FILE"; then
|
|
echo " OK: Backup created: $BACKUP_FILE"
|
|
ls -lh "$BACKUP_FILE"
|
|
else
|
|
echo " ERROR: Backup failed"
|
|
echo ""
|
|
echo "Aborting migration. Restoring traffic..."
|
|
|
|
# Restore traffic
|
|
if [[ "$ACTIVE_STACK" == "blue" ]]; then
|
|
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
else
|
|
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
fi
|
|
|
|
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
|
jq '.maintenance_mode = false' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
|
fi
|
|
|
|
exit 1
|
|
fi
|
|
else
|
|
echo ""
|
|
echo "Step 3/6: Skipping backup (not requested)..."
|
|
echo "----------------------------------------"
|
|
echo " OK"
|
|
fi
|
|
|
|
# Step 4: Run migrations
|
|
echo ""
|
|
echo "Step 4/6: Running database migrations..."
|
|
echo "----------------------------------------"
|
|
|
|
# Flush Redis cache before migration
|
|
echo " Flushing Redis cache..."
|
|
docker exec mvp-redis redis-cli FLUSHALL > /dev/null 2>&1 || true
|
|
|
|
# Run migrations
|
|
echo " Running migrations..."
|
|
if docker exec "$BACKEND_CONTAINER" npm run migrate; then
|
|
echo " OK: Migrations completed"
|
|
else
|
|
echo " ERROR: Migration failed"
|
|
echo ""
|
|
echo "Migration failed. Maintenance mode remains active."
|
|
echo "To restore:"
|
|
echo " 1. Fix migration issues"
|
|
echo " 2. Re-run migrations: docker exec $BACKEND_CONTAINER npm run migrate"
|
|
echo " 3. Run: $0 to retry, or manually restore traffic"
|
|
|
|
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
|
"$NOTIFY_SCRIPT" "failure" "Database migration failed during maintenance window. Manual intervention required." || true
|
|
fi
|
|
|
|
exit 1
|
|
fi
|
|
|
|
# Step 5: Restart backend containers
|
|
echo ""
|
|
echo "Step 5/6: Restarting backend containers..."
|
|
echo "----------------------------------------"
|
|
|
|
# Restart both backends to pick up any schema changes
|
|
docker restart mvp-backend-blue mvp-backend-green 2>/dev/null || true
|
|
|
|
# Wait for backends to be healthy
|
|
echo " Waiting for backends to be healthy..."
|
|
sleep 10
|
|
|
|
for container in mvp-backend-blue mvp-backend-green; do
|
|
for i in {1..12}; do
|
|
health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown")
|
|
if [[ "$health" == "healthy" ]]; then
|
|
echo " OK: $container is healthy"
|
|
break
|
|
fi
|
|
if [[ $i -eq 12 ]]; then
|
|
echo " WARNING: $container health check timeout"
|
|
fi
|
|
sleep 5
|
|
done
|
|
done
|
|
|
|
# Step 6: Disable maintenance mode
|
|
echo ""
|
|
echo "Step 6/6: Restoring traffic..."
|
|
echo "----------------------------------------"
|
|
|
|
# Restore traffic to active stack
|
|
if [[ "$ACTIVE_STACK" == "blue" ]]; then
|
|
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
else
|
|
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
|
fi
|
|
|
|
# Update state file
|
|
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
jq --arg ts "$TIMESTAMP" '.maintenance_mode = false | .last_migration = $ts' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
|
fi
|
|
|
|
# Wait for Traefik to pick up changes
|
|
sleep 3
|
|
echo " OK: Traffic restored"
|
|
|
|
# Send completion notification
|
|
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
|
"$NOTIFY_SCRIPT" "maintenance_end" "Maintenance window complete. Database migration successful." || true
|
|
fi
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "MAINTENANCE MIGRATION COMPLETE"
|
|
echo "========================================"
|
|
echo ""
|
|
echo "Summary:"
|
|
echo " - Migrations: Successful"
|
|
echo " - Active stack: $ACTIVE_STACK"
|
|
echo " - Maintenance mode: Disabled"
|
|
if [[ -n "$CREATE_BACKUP" ]]; then
|
|
echo " - Backup: $BACKUP_FILE"
|
|
fi
|
|
echo ""
|
|
|
|
exit 0
|