fix: I dunno, I'm making git server changes
This commit is contained in:
Eric Gullickson
220
scripts/ci/maintenance-migrate.sh
Executable file
220
scripts/ci/maintenance-migrate.sh
Executable file
@@ -0,0 +1,220 @@
|
||||
#!/bin/bash
|
||||
# Maintenance mode migration script
|
||||
# Enables maintenance mode, runs migrations, then restores service
|
||||
#
|
||||
# Usage: ./maintenance-migrate.sh [backup]
|
||||
# backup: If set, creates a database backup before migration
|
||||
#
|
||||
# This script is for BREAKING migrations that require downtime.
|
||||
# Non-breaking migrations run automatically on container start.
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Migration successful
|
||||
# 1 - Migration failed (maintenance mode will remain active)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
CREATE_BACKUP="${1:-}"
|
||||
|
||||
COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yml"
|
||||
COMPOSE_BLUE_GREEN="$PROJECT_ROOT/docker-compose.blue-green.yml"
|
||||
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
||||
NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh"
|
||||
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
|
||||
BACKUP_DIR="$PROJECT_ROOT/data/backups"
|
||||
|
||||
echo "========================================"
|
||||
echo "MAINTENANCE MODE MIGRATION"
|
||||
echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
echo "========================================"
|
||||
|
||||
# Determine active stack
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
|
||||
else
|
||||
ACTIVE_STACK="blue"
|
||||
fi
|
||||
|
||||
BACKEND_CONTAINER="mvp-backend-$ACTIVE_STACK"
|
||||
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Active stack: $ACTIVE_STACK"
|
||||
echo " Backend container: $BACKEND_CONTAINER"
|
||||
echo " Create backup: ${CREATE_BACKUP:-no}"
|
||||
echo ""
|
||||
|
||||
# Step 1: Send maintenance notification
|
||||
echo "Step 1/6: Sending maintenance notification..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "maintenance_start" "Starting maintenance window for database migration" || true
|
||||
fi
|
||||
echo " OK"
|
||||
|
||||
# Step 2: Enable maintenance mode
|
||||
echo ""
|
||||
echo "Step 2/6: Enabling maintenance mode..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Update state file
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
jq '.maintenance_mode = true' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
|
||||
# Stop traffic to both stacks (weight 0)
|
||||
echo " Stopping traffic to application stacks..."
|
||||
|
||||
# Set both stacks to weight 0 - Traefik will return 503
|
||||
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
rm -f "${TRAEFIK_CONFIG}.bak"
|
||||
|
||||
# Wait for Traefik to pick up changes
|
||||
sleep 3
|
||||
echo " OK: Maintenance mode active"
|
||||
|
||||
# Step 3: Create database backup (if requested)
|
||||
if [[ -n "$CREATE_BACKUP" ]]; then
|
||||
echo ""
|
||||
echo "Step 3/6: Creating database backup..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
BACKUP_FILE="$BACKUP_DIR/pre-migration-$(date +%Y%m%d-%H%M%S).sql"
|
||||
|
||||
if docker exec mvp-postgres pg_dump -U postgres motovaultpro > "$BACKUP_FILE"; then
|
||||
echo " OK: Backup created: $BACKUP_FILE"
|
||||
ls -lh "$BACKUP_FILE"
|
||||
else
|
||||
echo " ERROR: Backup failed"
|
||||
echo ""
|
||||
echo "Aborting migration. Restoring traffic..."
|
||||
|
||||
# Restore traffic
|
||||
if [[ "$ACTIVE_STACK" == "blue" ]]; then
|
||||
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
else
|
||||
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
fi
|
||||
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
jq '.maintenance_mode = false' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo "Step 3/6: Skipping backup (not requested)..."
|
||||
echo "----------------------------------------"
|
||||
echo " OK"
|
||||
fi
|
||||
|
||||
# Step 4: Run migrations
|
||||
echo ""
|
||||
echo "Step 4/6: Running database migrations..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Flush Redis cache before migration
|
||||
echo " Flushing Redis cache..."
|
||||
docker exec mvp-redis redis-cli FLUSHALL > /dev/null 2>&1 || true
|
||||
|
||||
# Run migrations
|
||||
echo " Running migrations..."
|
||||
if docker exec "$BACKEND_CONTAINER" npm run migrate; then
|
||||
echo " OK: Migrations completed"
|
||||
else
|
||||
echo " ERROR: Migration failed"
|
||||
echo ""
|
||||
echo "Migration failed. Maintenance mode remains active."
|
||||
echo "To restore:"
|
||||
echo " 1. Fix migration issues"
|
||||
echo " 2. Re-run migrations: docker exec $BACKEND_CONTAINER npm run migrate"
|
||||
echo " 3. Run: $0 to retry, or manually restore traffic"
|
||||
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "failure" "Database migration failed during maintenance window. Manual intervention required." || true
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 5: Restart backend containers
|
||||
echo ""
|
||||
echo "Step 5/6: Restarting backend containers..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Restart both backends to pick up any schema changes
|
||||
docker restart mvp-backend-blue mvp-backend-green 2>/dev/null || true
|
||||
|
||||
# Wait for backends to be healthy
|
||||
echo " Waiting for backends to be healthy..."
|
||||
sleep 10
|
||||
|
||||
for container in mvp-backend-blue mvp-backend-green; do
|
||||
for i in {1..12}; do
|
||||
health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown")
|
||||
if [[ "$health" == "healthy" ]]; then
|
||||
echo " OK: $container is healthy"
|
||||
break
|
||||
fi
|
||||
if [[ $i -eq 12 ]]; then
|
||||
echo " WARNING: $container health check timeout"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
done
|
||||
|
||||
# Step 6: Disable maintenance mode
|
||||
echo ""
|
||||
echo "Step 6/6: Restoring traffic..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Restore traffic to active stack
|
||||
if [[ "$ACTIVE_STACK" == "blue" ]]; then
|
||||
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
else
|
||||
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
fi
|
||||
|
||||
# Update state file
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
jq --arg ts "$TIMESTAMP" '.maintenance_mode = false | .last_migration = $ts' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
|
||||
# Wait for Traefik to pick up changes
|
||||
sleep 3
|
||||
echo " OK: Traffic restored"
|
||||
|
||||
# Send completion notification
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "maintenance_end" "Maintenance window complete. Database migration successful." || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "MAINTENANCE MIGRATION COMPLETE"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " - Migrations: Successful"
|
||||
echo " - Active stack: $ACTIVE_STACK"
|
||||
echo " - Maintenance mode: Disabled"
|
||||
if [[ -n "$CREATE_BACKUP" ]]; then
|
||||
echo " - Backup: $BACKUP_FILE"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user