fix: I dunno, I'm making git server changes
This commit is contained in:
Eric Gullickson
152
scripts/ci/auto-rollback.sh
Executable file
152
scripts/ci/auto-rollback.sh
Executable file
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
# Auto-rollback script for blue-green deployment
|
||||
# Reverts traffic to the previous healthy stack
|
||||
#
|
||||
# Usage: ./auto-rollback.sh [reason]
|
||||
# reason: Optional description of why rollback is happening
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Rollback successful
|
||||
# 1 - Rollback failed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
REASON="${1:-Automatic rollback triggered}"
|
||||
|
||||
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
||||
SWITCH_SCRIPT="$SCRIPT_DIR/switch-traffic.sh"
|
||||
HEALTH_SCRIPT="$SCRIPT_DIR/health-check.sh"
|
||||
NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh"
|
||||
|
||||
echo "========================================"
|
||||
echo "AUTO-ROLLBACK INITIATED"
|
||||
echo "Reason: $REASON"
|
||||
echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
echo "========================================"
|
||||
|
||||
# Determine current and rollback stacks
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
CURRENT_STACK=$(jq -r '.active_stack // "unknown"' "$STATE_FILE")
|
||||
ROLLBACK_STACK=$(jq -r '.inactive_stack // "unknown"' "$STATE_FILE")
|
||||
else
|
||||
echo "ERROR: Cannot determine current stack state"
|
||||
echo "State file: $STATE_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$CURRENT_STACK" == "unknown" ]] || [[ "$ROLLBACK_STACK" == "unknown" ]]; then
|
||||
echo "ERROR: Invalid stack state"
|
||||
echo " Current: $CURRENT_STACK"
|
||||
echo " Rollback target: $ROLLBACK_STACK"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Stack Status:"
|
||||
echo " Currently active: $CURRENT_STACK"
|
||||
echo " Rollback target: $ROLLBACK_STACK"
|
||||
echo ""
|
||||
|
||||
# Verify rollback stack is healthy before switching
|
||||
echo "Step 1/3: Verifying rollback stack health..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ -x "$HEALTH_SCRIPT" ]]; then
|
||||
if ! "$HEALTH_SCRIPT" "$ROLLBACK_STACK" 30; then
|
||||
echo ""
|
||||
echo "CRITICAL: Rollback stack ($ROLLBACK_STACK) is NOT healthy!"
|
||||
echo "Manual intervention required."
|
||||
echo ""
|
||||
echo "Troubleshooting steps:"
|
||||
echo " 1. Check container logs: docker logs mvp-backend-$ROLLBACK_STACK"
|
||||
echo " 2. Check container status: docker ps -a"
|
||||
echo " 3. Consider restarting rollback stack"
|
||||
echo ""
|
||||
|
||||
# Send critical notification
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "rollback_failed" \
|
||||
"Rollback to $ROLLBACK_STACK failed - stack unhealthy. Manual intervention required. Reason: $REASON" \
|
||||
|| true
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
echo " OK: Rollback stack is healthy"
|
||||
else
|
||||
echo " WARNING: Health check script not found, proceeding anyway"
|
||||
fi
|
||||
|
||||
# Switch traffic to rollback stack
|
||||
echo ""
|
||||
echo "Step 2/3: Switching traffic to $ROLLBACK_STACK..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ -x "$SWITCH_SCRIPT" ]]; then
|
||||
if ! "$SWITCH_SCRIPT" "$ROLLBACK_STACK" instant; then
|
||||
echo "ERROR: Traffic switch failed"
|
||||
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "rollback_failed" \
|
||||
"Rollback traffic switch failed. Manual intervention required. Reason: $REASON" \
|
||||
|| true
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "ERROR: Traffic switch script not found: $SWITCH_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Update state file with rollback info
|
||||
echo ""
|
||||
echo "Step 3/3: Updating deployment state..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
jq --arg stack "$ROLLBACK_STACK" \
|
||||
--arg reason "$REASON" \
|
||||
--arg ts "$TIMESTAMP" \
|
||||
--arg failed "$CURRENT_STACK" \
|
||||
'.active_stack = $stack |
|
||||
.inactive_stack = $failed |
|
||||
.last_rollback = $ts |
|
||||
.last_rollback_reason = $reason |
|
||||
.rollback_available = false' \
|
||||
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
|
||||
echo " State updated"
|
||||
fi
|
||||
|
||||
# Send notification
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
echo ""
|
||||
echo "Sending rollback notification..."
|
||||
"$NOTIFY_SCRIPT" "rollback" \
|
||||
"Rollback executed. Traffic switched from $CURRENT_STACK to $ROLLBACK_STACK. Reason: $REASON" \
|
||||
|| echo " WARNING: Notification failed"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "ROLLBACK COMPLETE"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " Previous stack: $CURRENT_STACK (now inactive)"
|
||||
echo " Current stack: $ROLLBACK_STACK (now active)"
|
||||
echo " Reason: $REASON"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Investigate why $CURRENT_STACK failed"
|
||||
echo " 2. Check logs: docker logs mvp-backend-$CURRENT_STACK"
|
||||
echo " 3. Fix issues before next deployment"
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
185
scripts/ci/health-check.sh
Executable file
185
scripts/ci/health-check.sh
Executable file
@@ -0,0 +1,185 @@
|
||||
#!/bin/bash
|
||||
# Health check script for blue-green deployment
|
||||
# Verifies container health and HTTP endpoints
|
||||
#
|
||||
# Usage: ./health-check.sh <stack> [timeout_seconds]
|
||||
# stack: blue or green
|
||||
# timeout_seconds: max wait time (default: 60)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - All health checks passed
|
||||
# 1 - Health check failed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
STACK="${1:-}"
|
||||
TIMEOUT="${2:-60}"
|
||||
|
||||
if [[ -z "$STACK" ]] || [[ ! "$STACK" =~ ^(blue|green)$ ]]; then
|
||||
echo "Usage: $0 <blue|green> [timeout_seconds]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FRONTEND_CONTAINER="mvp-frontend-$STACK"
|
||||
BACKEND_CONTAINER="mvp-backend-$STACK"
|
||||
|
||||
echo "========================================"
|
||||
echo "Health Check - $STACK Stack"
|
||||
echo "Timeout: ${TIMEOUT}s"
|
||||
echo "========================================"
|
||||
|
||||
# Function to check Docker container health
|
||||
check_container_health() {
|
||||
local container="$1"
|
||||
local status
|
||||
|
||||
status=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "not found")
|
||||
|
||||
case "$status" in
|
||||
"healthy")
|
||||
return 0
|
||||
;;
|
||||
"starting")
|
||||
return 2 # Still starting
|
||||
;;
|
||||
"unhealthy"|"not found"|"")
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Function to check HTTP endpoint
|
||||
check_http_endpoint() {
|
||||
local container="$1"
|
||||
local port="$2"
|
||||
local path="$3"
|
||||
|
||||
if docker exec "$container" curl -sf "http://localhost:${port}${path}" > /dev/null 2>&1; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to check database connectivity via backend
|
||||
check_database_connectivity() {
|
||||
local container="$1"
|
||||
|
||||
# The /health endpoint should verify database connectivity
|
||||
if docker exec "$container" curl -sf "http://localhost:3001/health" 2>/dev/null | grep -q '"database"'; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Wait for containers to be healthy
|
||||
wait_for_health() {
|
||||
local container="$1"
|
||||
local elapsed=0
|
||||
|
||||
while [[ $elapsed -lt $TIMEOUT ]]; do
|
||||
check_container_health "$container"
|
||||
local status=$?
|
||||
|
||||
if [[ $status -eq 0 ]]; then
|
||||
return 0
|
||||
elif [[ $status -eq 1 ]]; then
|
||||
echo " ERROR: Container $container is unhealthy"
|
||||
docker logs "$container" --tail 20 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Still starting, wait
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
echo " Waiting for $container... (${elapsed}s/${TIMEOUT}s)"
|
||||
done
|
||||
|
||||
echo " ERROR: Timeout waiting for $container"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Main health check sequence
|
||||
echo ""
|
||||
echo "Step 1/4: Checking container status..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
for container in "$FRONTEND_CONTAINER" "$BACKEND_CONTAINER"; do
|
||||
running=$(docker inspect --format='{{.State.Running}}' "$container" 2>/dev/null || echo "false")
|
||||
if [[ "$running" != "true" ]]; then
|
||||
echo " ERROR: Container $container is not running"
|
||||
docker ps -a --filter "name=$container" --format "table {{.Names}}\t{{.Status}}"
|
||||
exit 1
|
||||
fi
|
||||
echo " OK: $container is running"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Step 2/4: Waiting for Docker health checks..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
for container in "$FRONTEND_CONTAINER" "$BACKEND_CONTAINER"; do
|
||||
echo " Checking $container..."
|
||||
if ! wait_for_health "$container"; then
|
||||
echo " FAILED: $container health check"
|
||||
exit 1
|
||||
fi
|
||||
echo " OK: $container is healthy"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Step 3/4: Verifying HTTP endpoints..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Check frontend
|
||||
echo " Checking frontend HTTP..."
|
||||
if ! check_http_endpoint "$FRONTEND_CONTAINER" 3000 "/"; then
|
||||
echo " FAILED: Frontend HTTP check"
|
||||
exit 1
|
||||
fi
|
||||
echo " OK: Frontend responds on port 3000"
|
||||
|
||||
# Check backend health endpoint
|
||||
echo " Checking backend HTTP..."
|
||||
if ! check_http_endpoint "$BACKEND_CONTAINER" 3001 "/health"; then
|
||||
echo " FAILED: Backend HTTP check"
|
||||
exit 1
|
||||
fi
|
||||
echo " OK: Backend responds on port 3001"
|
||||
|
||||
echo ""
|
||||
echo "Step 4/4: Verifying database connectivity..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if ! check_database_connectivity "$BACKEND_CONTAINER"; then
|
||||
echo " WARNING: Could not verify database connectivity"
|
||||
echo " (Backend may not expose database status in /health)"
|
||||
else
|
||||
echo " OK: Database connectivity verified"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Health Check PASSED - $STACK Stack"
|
||||
echo "========================================"
|
||||
|
||||
# Update state file
|
||||
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
||||
if [[ -f "$STATE_FILE" ]]; then
|
||||
# Update stack health status using jq if available
|
||||
if command -v jq &> /dev/null; then
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
jq --arg stack "$STACK" --arg ts "$TIMESTAMP" \
|
||||
'.[$stack].healthy = true | .[$stack].last_health_check = $ts' \
|
||||
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
220
scripts/ci/maintenance-migrate.sh
Executable file
220
scripts/ci/maintenance-migrate.sh
Executable file
@@ -0,0 +1,220 @@
|
||||
#!/bin/bash
|
||||
# Maintenance mode migration script
|
||||
# Enables maintenance mode, runs migrations, then restores service
|
||||
#
|
||||
# Usage: ./maintenance-migrate.sh [backup]
|
||||
# backup: If set, creates a database backup before migration
|
||||
#
|
||||
# This script is for BREAKING migrations that require downtime.
|
||||
# Non-breaking migrations run automatically on container start.
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Migration successful
|
||||
# 1 - Migration failed (maintenance mode will remain active)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
CREATE_BACKUP="${1:-}"
|
||||
|
||||
COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yml"
|
||||
COMPOSE_BLUE_GREEN="$PROJECT_ROOT/docker-compose.blue-green.yml"
|
||||
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
||||
NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh"
|
||||
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
|
||||
BACKUP_DIR="$PROJECT_ROOT/data/backups"
|
||||
|
||||
echo "========================================"
|
||||
echo "MAINTENANCE MODE MIGRATION"
|
||||
echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
echo "========================================"
|
||||
|
||||
# Determine active stack
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
|
||||
else
|
||||
ACTIVE_STACK="blue"
|
||||
fi
|
||||
|
||||
BACKEND_CONTAINER="mvp-backend-$ACTIVE_STACK"
|
||||
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Active stack: $ACTIVE_STACK"
|
||||
echo " Backend container: $BACKEND_CONTAINER"
|
||||
echo " Create backup: ${CREATE_BACKUP:-no}"
|
||||
echo ""
|
||||
|
||||
# Step 1: Send maintenance notification
|
||||
echo "Step 1/6: Sending maintenance notification..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "maintenance_start" "Starting maintenance window for database migration" || true
|
||||
fi
|
||||
echo " OK"
|
||||
|
||||
# Step 2: Enable maintenance mode
|
||||
echo ""
|
||||
echo "Step 2/6: Enabling maintenance mode..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Update state file
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
jq '.maintenance_mode = true' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
|
||||
# Stop traffic to both stacks (weight 0)
|
||||
echo " Stopping traffic to application stacks..."
|
||||
|
||||
# Set both stacks to weight 0 - Traefik will return 503
|
||||
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
|
||||
rm -f "${TRAEFIK_CONFIG}.bak"
|
||||
|
||||
# Wait for Traefik to pick up changes
|
||||
sleep 3
|
||||
echo " OK: Maintenance mode active"
|
||||
|
||||
# Step 3: Create database backup (if requested)
|
||||
if [[ -n "$CREATE_BACKUP" ]]; then
|
||||
echo ""
|
||||
echo "Step 3/6: Creating database backup..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
BACKUP_FILE="$BACKUP_DIR/pre-migration-$(date +%Y%m%d-%H%M%S).sql"
|
||||
|
||||
if docker exec mvp-postgres pg_dump -U postgres motovaultpro > "$BACKUP_FILE"; then
|
||||
echo " OK: Backup created: $BACKUP_FILE"
|
||||
ls -lh "$BACKUP_FILE"
|
||||
else
|
||||
echo " ERROR: Backup failed"
|
||||
echo ""
|
||||
echo "Aborting migration. Restoring traffic..."
|
||||
|
||||
# Restore traffic
|
||||
if [[ "$ACTIVE_STACK" == "blue" ]]; then
|
||||
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
else
|
||||
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
fi
|
||||
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
jq '.maintenance_mode = false' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo "Step 3/6: Skipping backup (not requested)..."
|
||||
echo "----------------------------------------"
|
||||
echo " OK"
|
||||
fi
|
||||
|
||||
# Step 4: Run migrations
|
||||
echo ""
|
||||
echo "Step 4/6: Running database migrations..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Flush Redis cache before migration
|
||||
echo " Flushing Redis cache..."
|
||||
docker exec mvp-redis redis-cli FLUSHALL > /dev/null 2>&1 || true
|
||||
|
||||
# Run migrations
|
||||
echo " Running migrations..."
|
||||
if docker exec "$BACKEND_CONTAINER" npm run migrate; then
|
||||
echo " OK: Migrations completed"
|
||||
else
|
||||
echo " ERROR: Migration failed"
|
||||
echo ""
|
||||
echo "Migration failed. Maintenance mode remains active."
|
||||
echo "To restore:"
|
||||
echo " 1. Fix migration issues"
|
||||
echo " 2. Re-run migrations: docker exec $BACKEND_CONTAINER npm run migrate"
|
||||
echo " 3. Run: $0 to retry, or manually restore traffic"
|
||||
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "failure" "Database migration failed during maintenance window. Manual intervention required." || true
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 5: Restart backend containers
|
||||
echo ""
|
||||
echo "Step 5/6: Restarting backend containers..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Restart both backends to pick up any schema changes
|
||||
docker restart mvp-backend-blue mvp-backend-green 2>/dev/null || true
|
||||
|
||||
# Wait for backends to be healthy
|
||||
echo " Waiting for backends to be healthy..."
|
||||
sleep 10
|
||||
|
||||
for container in mvp-backend-blue mvp-backend-green; do
|
||||
for i in {1..12}; do
|
||||
health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown")
|
||||
if [[ "$health" == "healthy" ]]; then
|
||||
echo " OK: $container is healthy"
|
||||
break
|
||||
fi
|
||||
if [[ $i -eq 12 ]]; then
|
||||
echo " WARNING: $container health check timeout"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
done
|
||||
|
||||
# Step 6: Disable maintenance mode
|
||||
echo ""
|
||||
echo "Step 6/6: Restoring traffic..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Restore traffic to active stack
|
||||
if [[ "$ACTIVE_STACK" == "blue" ]]; then
|
||||
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
else
|
||||
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
|
||||
fi
|
||||
|
||||
# Update state file
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
jq --arg ts "$TIMESTAMP" '.maintenance_mode = false | .last_migration = $ts' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
fi
|
||||
|
||||
# Wait for Traefik to pick up changes
|
||||
sleep 3
|
||||
echo " OK: Traffic restored"
|
||||
|
||||
# Send completion notification
|
||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||
"$NOTIFY_SCRIPT" "maintenance_end" "Maintenance window complete. Database migration successful." || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "MAINTENANCE MIGRATION COMPLETE"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " - Migrations: Successful"
|
||||
echo " - Active stack: $ACTIVE_STACK"
|
||||
echo " - Maintenance mode: Disabled"
|
||||
if [[ -n "$CREATE_BACKUP" ]]; then
|
||||
echo " - Backup: $BACKUP_FILE"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
exit 0
|
||||
84
scripts/ci/mirror-base-images.sh
Executable file
84
scripts/ci/mirror-base-images.sh
Executable file
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
# Mirror upstream Docker images to GitLab Container Registry
|
||||
# Run manually or via scheduled GitLab pipeline
|
||||
# This avoids Docker Hub rate limits and ensures build reliability
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REGISTRY="${REGISTRY:-registry.motovaultpro.com/mirrors}"
|
||||
|
||||
# Base images required by MotoVaultPro
|
||||
IMAGES=(
|
||||
"node:20-alpine"
|
||||
"nginx:alpine"
|
||||
"postgres:18-alpine"
|
||||
"redis:8.4-alpine"
|
||||
"traefik:v3.6"
|
||||
"docker:24.0"
|
||||
"docker:24.0-dind"
|
||||
)
|
||||
|
||||
echo "========================================"
|
||||
echo "Base Image Mirroring Script"
|
||||
echo "Registry: $REGISTRY"
|
||||
echo "========================================"
|
||||
|
||||
# Check if logged into registry
|
||||
if ! docker info 2>/dev/null | grep -q "Username"; then
|
||||
echo "WARNING: Not logged into Docker registry"
|
||||
echo "Run: docker login registry.motovaultpro.com"
|
||||
fi
|
||||
|
||||
FAILED=()
|
||||
SUCCESS=()
|
||||
|
||||
for img in "${IMAGES[@]}"; do
|
||||
echo ""
|
||||
echo "Processing: $img"
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Pull from upstream
|
||||
echo " Pulling from upstream..."
|
||||
if ! docker pull "$img"; then
|
||||
echo " ERROR: Failed to pull $img"
|
||||
FAILED+=("$img")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Tag for local registry
|
||||
local_tag="$REGISTRY/$img"
|
||||
echo " Tagging as: $local_tag"
|
||||
docker tag "$img" "$local_tag"
|
||||
|
||||
# Push to local registry
|
||||
echo " Pushing to registry..."
|
||||
if ! docker push "$local_tag"; then
|
||||
echo " ERROR: Failed to push $local_tag"
|
||||
FAILED+=("$img")
|
||||
continue
|
||||
fi
|
||||
|
||||
SUCCESS+=("$img")
|
||||
echo " OK: $img mirrored successfully"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Summary"
|
||||
echo "========================================"
|
||||
echo "Successful: ${#SUCCESS[@]}"
|
||||
for img in "${SUCCESS[@]}"; do
|
||||
echo " - $img"
|
||||
done
|
||||
|
||||
if [ ${#FAILED[@]} -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Failed: ${#FAILED[@]}"
|
||||
for img in "${FAILED[@]}"; do
|
||||
echo " - $img"
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "All images mirrored successfully"
|
||||
195
scripts/ci/notify.sh
Executable file
195
scripts/ci/notify.sh
Executable file
@@ -0,0 +1,195 @@
|
||||
#!/bin/bash
|
||||
# Deployment notification script using Resend API
|
||||
# Sends email notifications for deployment events
|
||||
#
|
||||
# Usage: ./notify.sh <event_type> [message] [commit_sha]
|
||||
# event_type: success, failure, rollback, rollback_failed, maintenance_start, maintenance_end
|
||||
# message: Optional custom message
|
||||
# commit_sha: Optional commit SHA for context
|
||||
#
|
||||
# Required environment variables:
|
||||
# DEPLOY_NOTIFY_EMAIL - Recipient email address
|
||||
#
|
||||
# Reads Resend API key from:
|
||||
# /run/secrets/resend-api-key (container)
|
||||
# ./secrets/app/resend-api-key.txt (local)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Notification sent (or skipped if not configured)
|
||||
# 1 - Notification failed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
EVENT_TYPE="${1:-}"
|
||||
MESSAGE="${2:-}"
|
||||
COMMIT_SHA="${3:-${CI_COMMIT_SHORT_SHA:-unknown}}"
|
||||
|
||||
if [[ -z "$EVENT_TYPE" ]]; then
|
||||
echo "Usage: $0 <success|failure|rollback|rollback_failed|maintenance_start|maintenance_end> [message] [commit_sha]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get recipient email
|
||||
NOTIFY_EMAIL="${DEPLOY_NOTIFY_EMAIL:-}"
|
||||
if [[ -z "$NOTIFY_EMAIL" ]]; then
|
||||
echo "DEPLOY_NOTIFY_EMAIL not set, skipping notification"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Get Resend API key
|
||||
RESEND_API_KEY=""
|
||||
if [[ -f "/run/secrets/resend-api-key" ]]; then
|
||||
RESEND_API_KEY=$(cat /run/secrets/resend-api-key)
|
||||
elif [[ -f "$PROJECT_ROOT/secrets/app/resend-api-key.txt" ]]; then
|
||||
RESEND_API_KEY=$(cat "$PROJECT_ROOT/secrets/app/resend-api-key.txt")
|
||||
fi
|
||||
|
||||
if [[ -z "$RESEND_API_KEY" ]]; then
|
||||
echo "WARNING: Resend API key not found, skipping notification"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine subject and styling based on event type
|
||||
case "$EVENT_TYPE" in
|
||||
"success")
|
||||
SUBJECT="Deployment Successful - MotoVaultPro"
|
||||
STATUS_COLOR="#22c55e"
|
||||
STATUS_EMOJI="[OK]"
|
||||
STATUS_TEXT="Deployment Successful"
|
||||
DEFAULT_MESSAGE="Version $COMMIT_SHA is now live."
|
||||
;;
|
||||
"failure")
|
||||
SUBJECT="Deployment Failed - MotoVaultPro"
|
||||
STATUS_COLOR="#ef4444"
|
||||
STATUS_EMOJI="[FAILED]"
|
||||
STATUS_TEXT="Deployment Failed"
|
||||
DEFAULT_MESSAGE="Deployment of $COMMIT_SHA failed. Check pipeline logs."
|
||||
;;
|
||||
"rollback")
|
||||
SUBJECT="Auto-Rollback Executed - MotoVaultPro"
|
||||
STATUS_COLOR="#f59e0b"
|
||||
STATUS_EMOJI="[ROLLBACK]"
|
||||
STATUS_TEXT="Rollback Executed"
|
||||
DEFAULT_MESSAGE="Automatic rollback was triggered. Previous version restored."
|
||||
;;
|
||||
"rollback_failed")
|
||||
SUBJECT="CRITICAL: Rollback Failed - MotoVaultPro"
|
||||
STATUS_COLOR="#dc2626"
|
||||
STATUS_EMOJI="[CRITICAL]"
|
||||
STATUS_TEXT="Rollback Failed"
|
||||
DEFAULT_MESSAGE="Rollback attempt failed. Manual intervention required immediately."
|
||||
;;
|
||||
"maintenance_start")
|
||||
SUBJECT="Maintenance Mode Started - MotoVaultPro"
|
||||
STATUS_COLOR="#6366f1"
|
||||
STATUS_EMOJI="[MAINTENANCE]"
|
||||
STATUS_TEXT="Maintenance Mode Active"
|
||||
DEFAULT_MESSAGE="Application is in maintenance mode for database migration."
|
||||
;;
|
||||
"maintenance_end")
|
||||
SUBJECT="Maintenance Mode Ended - MotoVaultPro"
|
||||
STATUS_COLOR="#22c55e"
|
||||
STATUS_EMOJI="[ONLINE]"
|
||||
STATUS_TEXT="Maintenance Complete"
|
||||
DEFAULT_MESSAGE="Maintenance window complete. Application is online."
|
||||
;;
|
||||
*)
|
||||
echo "Unknown event type: $EVENT_TYPE"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Use custom message or default
|
||||
FINAL_MESSAGE="${MESSAGE:-$DEFAULT_MESSAGE}"
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
# Build HTML email
|
||||
HTML_BODY=$(cat <<EOF
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #f3f4f6; padding: 20px; }
|
||||
.container { max-width: 600px; margin: 0 auto; background: white; border-radius: 8px; overflow: hidden; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
|
||||
.header { background: ${STATUS_COLOR}; color: white; padding: 20px; text-align: center; }
|
||||
.header h1 { margin: 0; font-size: 24px; }
|
||||
.content { padding: 30px; }
|
||||
.status { font-size: 18px; font-weight: 600; margin-bottom: 15px; }
|
||||
.message { color: #374151; line-height: 1.6; margin-bottom: 20px; }
|
||||
.details { background: #f9fafb; border-radius: 6px; padding: 15px; font-size: 14px; }
|
||||
.details-row { display: flex; justify-content: space-between; margin-bottom: 8px; }
|
||||
.details-label { color: #6b7280; }
|
||||
.details-value { color: #111827; font-weight: 500; }
|
||||
.footer { padding: 20px; text-align: center; color: #6b7280; font-size: 12px; border-top: 1px solid #e5e7eb; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>MotoVaultPro</h1>
|
||||
</div>
|
||||
<div class="content">
|
||||
<div class="status">${STATUS_EMOJI} ${STATUS_TEXT}</div>
|
||||
<div class="message">${FINAL_MESSAGE}</div>
|
||||
<div class="details">
|
||||
<div class="details-row">
|
||||
<span class="details-label">Environment:</span>
|
||||
<span class="details-value">Production</span>
|
||||
</div>
|
||||
<div class="details-row">
|
||||
<span class="details-label">Commit:</span>
|
||||
<span class="details-value">${COMMIT_SHA}</span>
|
||||
</div>
|
||||
<div class="details-row">
|
||||
<span class="details-label">Time:</span>
|
||||
<span class="details-value">${TIMESTAMP}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer">
|
||||
MotoVaultPro CI/CD Notification System
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
)
|
||||
|
||||
# Send email via Resend API
|
||||
echo "Sending notification: $EVENT_TYPE"
|
||||
echo " To: $NOTIFY_EMAIL"
|
||||
echo " Subject: $SUBJECT"
|
||||
|
||||
# Build JSON payload
|
||||
JSON_PAYLOAD=$(cat <<EOF
|
||||
{
|
||||
"from": "MotoVaultPro <deploy@motovaultpro.com>",
|
||||
"to": ["$NOTIFY_EMAIL"],
|
||||
"subject": "$SUBJECT",
|
||||
"html": $(echo "$HTML_BODY" | jq -Rs .)
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
# Send via Resend API
|
||||
RESPONSE=$(curl -s -w "\n%{http_code}" \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer $RESEND_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$JSON_PAYLOAD" \
|
||||
"https://api.resend.com/emails")
|
||||
|
||||
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
|
||||
BODY=$(echo "$RESPONSE" | sed '$d')
|
||||
|
||||
if [[ "$HTTP_CODE" == "200" ]] || [[ "$HTTP_CODE" == "201" ]]; then
|
||||
echo " OK: Notification sent successfully"
|
||||
exit 0
|
||||
else
|
||||
echo " ERROR: Failed to send notification (HTTP $HTTP_CODE)"
|
||||
echo " Response: $BODY"
|
||||
exit 1
|
||||
fi
|
||||
157
scripts/ci/switch-traffic.sh
Executable file
157
scripts/ci/switch-traffic.sh
Executable file
@@ -0,0 +1,157 @@
|
||||
#!/bin/bash
|
||||
# Traffic switching script for blue-green deployment
|
||||
# Updates Traefik weighted routing configuration
|
||||
#
|
||||
# Usage: ./switch-traffic.sh <target_stack> [mode]
|
||||
# target_stack: blue or green (the stack to switch TO)
|
||||
# mode: instant (default) or gradual
|
||||
#
|
||||
# Gradual mode: 25% -> 50% -> 75% -> 100% with 3s intervals
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - Traffic switch successful
|
||||
# 1 - Switch failed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
TARGET_STACK="${1:-}"
|
||||
MODE="${2:-instant}"
|
||||
|
||||
if [[ -z "$TARGET_STACK" ]] || [[ ! "$TARGET_STACK" =~ ^(blue|green)$ ]]; then
|
||||
echo "Usage: $0 <blue|green> [instant|gradual]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
|
||||
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
||||
|
||||
if [[ ! -f "$TRAEFIK_CONFIG" ]]; then
|
||||
echo "ERROR: Traefik config not found: $TRAEFIK_CONFIG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "========================================"
|
||||
echo "Traffic Switch"
|
||||
echo "Target Stack: $TARGET_STACK"
|
||||
echo "Mode: $MODE"
|
||||
echo "========================================"
|
||||
|
||||
# Determine source and target weights
|
||||
if [[ "$TARGET_STACK" == "blue" ]]; then
|
||||
SOURCE_STACK="green"
|
||||
else
|
||||
SOURCE_STACK="blue"
|
||||
fi
|
||||
|
||||
# Function to update weights in Traefik config
|
||||
update_weights() {
|
||||
local blue_weight="$1"
|
||||
local green_weight="$2"
|
||||
|
||||
echo " Setting weights: blue=$blue_weight, green=$green_weight"
|
||||
|
||||
# Use sed to update weights in the YAML file
|
||||
# Frontend blue weight
|
||||
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG"
|
||||
# Frontend green weight
|
||||
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG"
|
||||
# Backend blue weight
|
||||
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG"
|
||||
# Backend green weight
|
||||
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG"
|
||||
|
||||
# Clean up backup files
|
||||
rm -f "${TRAEFIK_CONFIG}.bak"
|
||||
|
||||
# Traefik watches the file and reloads automatically
|
||||
# Give it a moment to pick up changes
|
||||
sleep 1
|
||||
}
|
||||
|
||||
# Verify Traefik has picked up the changes
|
||||
verify_traefik_reload() {
|
||||
# Give Traefik time to reload config
|
||||
sleep 2
|
||||
|
||||
# Check if Traefik is still healthy
|
||||
if docker exec mvp-traefik traefik healthcheck > /dev/null 2>&1; then
|
||||
echo " OK: Traefik config reloaded"
|
||||
return 0
|
||||
else
|
||||
echo " WARNING: Could not verify Traefik health"
|
||||
return 0 # Don't fail on this, file watcher is reliable
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "$MODE" == "gradual" ]]; then
|
||||
echo ""
|
||||
echo "Gradual traffic switch starting..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ "$TARGET_STACK" == "blue" ]]; then
|
||||
STEPS=("25 75" "50 50" "75 25" "100 0")
|
||||
else
|
||||
STEPS=("75 25" "50 50" "25 75" "0 100")
|
||||
fi
|
||||
|
||||
step=1
|
||||
for weights in "${STEPS[@]}"; do
|
||||
blue_w=$(echo "$weights" | cut -d' ' -f1)
|
||||
green_w=$(echo "$weights" | cut -d' ' -f2)
|
||||
|
||||
echo ""
|
||||
echo "Step $step/4: blue=$blue_w%, green=$green_w%"
|
||||
update_weights "$blue_w" "$green_w"
|
||||
verify_traefik_reload
|
||||
|
||||
if [[ $step -lt 4 ]]; then
|
||||
echo " Waiting 3 seconds before next step..."
|
||||
sleep 3
|
||||
fi
|
||||
|
||||
step=$((step + 1))
|
||||
done
|
||||
|
||||
else
|
||||
# Instant switch
|
||||
echo ""
|
||||
echo "Instant traffic switch..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ "$TARGET_STACK" == "blue" ]]; then
|
||||
update_weights 100 0
|
||||
else
|
||||
update_weights 0 100
|
||||
fi
|
||||
|
||||
verify_traefik_reload
|
||||
fi
|
||||
|
||||
# Update state file
|
||||
echo ""
|
||||
echo "Updating deployment state..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
jq --arg active "$TARGET_STACK" \
|
||||
--arg inactive "$SOURCE_STACK" \
|
||||
--arg ts "$TIMESTAMP" \
|
||||
'.active_stack = $active | .inactive_stack = $inactive | .last_switch = $ts' \
|
||||
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
|
||||
echo " Active stack: $TARGET_STACK"
|
||||
echo " Inactive stack: $SOURCE_STACK"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Traffic Switch COMPLETE"
|
||||
echo "All traffic now routed to: $TARGET_STACK"
|
||||
echo "========================================"
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user