Files
motovaultpro/scripts/ci/switch-traffic.sh
Eric Gullickson 13abbc16d7 fix: CI/CD blue-green deployment path bug causing stale production content
Root cause: switch-traffic.sh was modifying Traefik config in the CI checkout
directory ($GITHUB_WORKSPACE) instead of the deployment directory ($DEPLOY_PATH).
Traefik never saw the weight changes, so traffic stayed on old containers.

Changes:
- Add DEPLOY_PATH environment variable support to all CI scripts
- Add --force-recreate flag to ensure containers are recreated with new images
- Add image verification step to confirm containers use expected images
- Add weight verification to confirm Traefik routing was updated
- Add routing validation step to verify traffic switch succeeded

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-31 10:37:18 -06:00

185 lines
5.4 KiB
Bash
Executable File

#!/bin/bash
# Traffic switching script for blue-green deployment
# Updates Traefik weighted routing configuration
#
# Usage: ./switch-traffic.sh <target_stack> [mode]
# target_stack: blue or green (the stack to switch TO)
# mode: instant (default) or gradual
#
# Gradual mode: 25% -> 50% -> 75% -> 100% with 3s intervals
#
# Exit codes:
# 0 - Traffic switch successful
# 1 - Switch failed
set -euo pipefail
# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
# This is critical: CI workflows must pass DEPLOY_PATH to ensure we modify
# the actual deployment config, not the checkout directory
PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
echo "Using PROJECT_ROOT: $PROJECT_ROOT"
TARGET_STACK="${1:-}"
MODE="${2:-instant}"
if [[ -z "$TARGET_STACK" ]] || [[ ! "$TARGET_STACK" =~ ^(blue|green)$ ]]; then
echo "Usage: $0 <blue|green> [instant|gradual]"
exit 1
fi
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
if [[ ! -f "$TRAEFIK_CONFIG" ]]; then
echo "ERROR: Traefik config not found: $TRAEFIK_CONFIG"
exit 1
fi
echo "========================================"
echo "Traffic Switch"
echo "Target Stack: $TARGET_STACK"
echo "Mode: $MODE"
echo "========================================"
# Determine source and target weights
if [[ "$TARGET_STACK" == "blue" ]]; then
SOURCE_STACK="green"
else
SOURCE_STACK="blue"
fi
# Function to update weights in Traefik config
update_weights() {
local blue_weight="$1"
local green_weight="$2"
echo " Setting weights: blue=$blue_weight, green=$green_weight"
# Use sed to update weights in the YAML file
# Frontend blue weight
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG"
# Frontend green weight
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG"
# Backend blue weight
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG"
# Backend green weight
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG"
# Clean up backup files
rm -f "${TRAEFIK_CONFIG}.bak"
# Traefik watches the file and reloads automatically
# Give it a moment to pick up changes
sleep 1
}
# Verify weights were actually written to config file
verify_weights_applied() {
local expected_blue="$1"
local expected_green="$2"
# Extract actual weights from the config file
local actual_blue=$(grep -A1 "mvp-frontend-blue-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
local actual_green=$(grep -A1 "mvp-frontend-green-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
if [[ "$actual_blue" != "$expected_blue" ]] || [[ "$actual_green" != "$expected_green" ]]; then
echo " ERROR: Weight verification failed!"
echo " Expected: blue=$expected_blue, green=$expected_green"
echo " Actual: blue=$actual_blue, green=$actual_green"
echo " Config file: $TRAEFIK_CONFIG"
return 1
fi
echo " OK: Weights verified (blue=$actual_blue, green=$actual_green)"
return 0
}
# Verify Traefik has picked up the changes
verify_traefik_reload() {
# Give Traefik time to reload config
sleep 2
# Check if Traefik is still healthy
if docker exec mvp-traefik traefik healthcheck > /dev/null 2>&1; then
echo " OK: Traefik config reloaded"
return 0
else
echo " WARNING: Could not verify Traefik health"
return 0 # Don't fail on this, file watcher is reliable
fi
}
if [[ "$MODE" == "gradual" ]]; then
echo ""
echo "Gradual traffic switch starting..."
echo "----------------------------------------"
if [[ "$TARGET_STACK" == "blue" ]]; then
STEPS=("25 75" "50 50" "75 25" "100 0")
else
STEPS=("75 25" "50 50" "25 75" "0 100")
fi
step=1
for weights in "${STEPS[@]}"; do
blue_w=$(echo "$weights" | cut -d' ' -f1)
green_w=$(echo "$weights" | cut -d' ' -f2)
echo ""
echo "Step $step/4: blue=$blue_w%, green=$green_w%"
update_weights "$blue_w" "$green_w"
verify_traefik_reload
if [[ $step -lt 4 ]]; then
echo " Waiting 3 seconds before next step..."
sleep 3
fi
step=$((step + 1))
done
else
# Instant switch
echo ""
echo "Instant traffic switch..."
echo "----------------------------------------"
if [[ "$TARGET_STACK" == "blue" ]]; then
update_weights 100 0
verify_weights_applied 100 0 || exit 1
else
update_weights 0 100
verify_weights_applied 0 100 || exit 1
fi
verify_traefik_reload
fi
# Update state file
echo ""
echo "Updating deployment state..."
echo "----------------------------------------"
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg active "$TARGET_STACK" \
--arg inactive "$SOURCE_STACK" \
--arg ts "$TIMESTAMP" \
'.active_stack = $active | .inactive_stack = $inactive | .last_switch = $ts' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
echo " Active stack: $TARGET_STACK"
echo " Inactive stack: $SOURCE_STACK"
fi
echo ""
echo "========================================"
echo "Traffic Switch COMPLETE"
echo "All traffic now routed to: $TARGET_STACK"
echo "========================================"
exit 0