All checks were successful
Deploy to Staging / Build Images (push) Successful in 23s
Deploy to Staging / Deploy to Staging (push) Successful in 26s
Deploy to Staging / Verify Staging (push) Successful in 5s
Deploy to Staging / Notify Staging Ready (push) Successful in 5s
Deploy to Staging / Notify Staging Failure (push) Has been skipped
Root cause: switch-traffic.sh was modifying Traefik config in the CI checkout directory ($GITHUB_WORKSPACE) instead of the deployment directory ($DEPLOY_PATH). Additionally, the sed patterns didn't work with multi-line YAML structure. Changes: - Add DEPLOY_PATH environment variable support to all CI scripts - Add --force-recreate flag to ensure containers are recreated with new images - Fix weight update to use awk for reliable multi-line YAML editing - Add scripts/ directory to rsync so SREs can run scripts from /opt/motovaultpro - Add image verification step to confirm containers use expected images - Add weight verification to confirm Traefik routing was updated - Add routing validation step to verify traffic switch succeeded 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
186 lines
5.6 KiB
Bash
Executable File
186 lines
5.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# Traffic switching script for blue-green deployment
|
|
# Updates Traefik weighted routing configuration
|
|
#
|
|
# Usage: ./switch-traffic.sh <target_stack> [mode]
|
|
# target_stack: blue or green (the stack to switch TO)
|
|
# mode: instant (default) or gradual
|
|
#
|
|
# Gradual mode: 25% -> 50% -> 75% -> 100% with 3s intervals
|
|
#
|
|
# Exit codes:
|
|
# 0 - Traffic switch successful
|
|
# 1 - Switch failed
|
|
|
|
set -euo pipefail
|
|
|
|
# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
|
|
# This is critical: CI workflows must pass DEPLOY_PATH to ensure we modify
|
|
# the actual deployment config, not the checkout directory
|
|
PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
|
|
|
|
echo "Using PROJECT_ROOT: $PROJECT_ROOT"
|
|
|
|
TARGET_STACK="${1:-}"
|
|
MODE="${2:-instant}"
|
|
|
|
if [[ -z "$TARGET_STACK" ]] || [[ ! "$TARGET_STACK" =~ ^(blue|green)$ ]]; then
|
|
echo "Usage: $0 <blue|green> [instant|gradual]"
|
|
exit 1
|
|
fi
|
|
|
|
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
|
|
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
|
|
|
|
if [[ ! -f "$TRAEFIK_CONFIG" ]]; then
|
|
echo "ERROR: Traefik config not found: $TRAEFIK_CONFIG"
|
|
exit 1
|
|
fi
|
|
|
|
echo "========================================"
|
|
echo "Traffic Switch"
|
|
echo "Target Stack: $TARGET_STACK"
|
|
echo "Mode: $MODE"
|
|
echo "========================================"
|
|
|
|
# Determine source and target weights
|
|
if [[ "$TARGET_STACK" == "blue" ]]; then
|
|
SOURCE_STACK="green"
|
|
else
|
|
SOURCE_STACK="blue"
|
|
fi
|
|
|
|
# Function to update weights in Traefik config
|
|
update_weights() {
|
|
local blue_weight="$1"
|
|
local green_weight="$2"
|
|
|
|
echo " Setting weights: blue=$blue_weight, green=$green_weight"
|
|
|
|
# YAML structure has name and weight on separate lines:
|
|
# - name: mvp-frontend-blue-svc
|
|
# weight: 100
|
|
# Use awk for reliable multi-line pattern matching (more portable than sed)
|
|
# Match "- name: service-name" to avoid matching the loadBalancer section
|
|
|
|
awk -v blue="$blue_weight" -v green="$green_weight" '
|
|
/- name: mvp-frontend-blue-svc$/ {print; getline; sub(/weight: [0-9]+/, "weight: " blue); print; next}
|
|
/- name: mvp-frontend-green-svc$/ {print; getline; sub(/weight: [0-9]+/, "weight: " green); print; next}
|
|
/- name: mvp-backend-blue-svc$/ {print; getline; sub(/weight: [0-9]+/, "weight: " blue); print; next}
|
|
/- name: mvp-backend-green-svc$/ {print; getline; sub(/weight: [0-9]+/, "weight: " green); print; next}
|
|
{print}
|
|
' "$TRAEFIK_CONFIG" > "${TRAEFIK_CONFIG}.tmp" && mv "${TRAEFIK_CONFIG}.tmp" "$TRAEFIK_CONFIG"
|
|
|
|
# Traefik watches the file and reloads automatically
|
|
# Give it a moment to pick up changes
|
|
sleep 1
|
|
}
|
|
|
|
# Verify weights were actually written to config file
|
|
verify_weights_applied() {
|
|
local expected_blue="$1"
|
|
local expected_green="$2"
|
|
|
|
# Extract actual weights from the config file
|
|
local actual_blue=$(grep -A1 "mvp-frontend-blue-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
|
|
local actual_green=$(grep -A1 "mvp-frontend-green-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
|
|
|
|
if [[ "$actual_blue" != "$expected_blue" ]] || [[ "$actual_green" != "$expected_green" ]]; then
|
|
echo " ERROR: Weight verification failed!"
|
|
echo " Expected: blue=$expected_blue, green=$expected_green"
|
|
echo " Actual: blue=$actual_blue, green=$actual_green"
|
|
echo " Config file: $TRAEFIK_CONFIG"
|
|
return 1
|
|
fi
|
|
|
|
echo " OK: Weights verified (blue=$actual_blue, green=$actual_green)"
|
|
return 0
|
|
}
|
|
|
|
# Verify Traefik has picked up the changes
|
|
verify_traefik_reload() {
|
|
# Give Traefik time to reload config
|
|
sleep 2
|
|
|
|
# Check if Traefik is still healthy
|
|
if docker exec mvp-traefik traefik healthcheck > /dev/null 2>&1; then
|
|
echo " OK: Traefik config reloaded"
|
|
return 0
|
|
else
|
|
echo " WARNING: Could not verify Traefik health"
|
|
return 0 # Don't fail on this, file watcher is reliable
|
|
fi
|
|
}
|
|
|
|
if [[ "$MODE" == "gradual" ]]; then
|
|
echo ""
|
|
echo "Gradual traffic switch starting..."
|
|
echo "----------------------------------------"
|
|
|
|
if [[ "$TARGET_STACK" == "blue" ]]; then
|
|
STEPS=("25 75" "50 50" "75 25" "100 0")
|
|
else
|
|
STEPS=("75 25" "50 50" "25 75" "0 100")
|
|
fi
|
|
|
|
step=1
|
|
for weights in "${STEPS[@]}"; do
|
|
blue_w=$(echo "$weights" | cut -d' ' -f1)
|
|
green_w=$(echo "$weights" | cut -d' ' -f2)
|
|
|
|
echo ""
|
|
echo "Step $step/4: blue=$blue_w%, green=$green_w%"
|
|
update_weights "$blue_w" "$green_w"
|
|
verify_traefik_reload
|
|
|
|
if [[ $step -lt 4 ]]; then
|
|
echo " Waiting 3 seconds before next step..."
|
|
sleep 3
|
|
fi
|
|
|
|
step=$((step + 1))
|
|
done
|
|
|
|
else
|
|
# Instant switch
|
|
echo ""
|
|
echo "Instant traffic switch..."
|
|
echo "----------------------------------------"
|
|
|
|
if [[ "$TARGET_STACK" == "blue" ]]; then
|
|
update_weights 100 0
|
|
verify_weights_applied 100 0 || exit 1
|
|
else
|
|
update_weights 0 100
|
|
verify_weights_applied 0 100 || exit 1
|
|
fi
|
|
|
|
verify_traefik_reload
|
|
fi
|
|
|
|
# Update state file
|
|
echo ""
|
|
echo "Updating deployment state..."
|
|
echo "----------------------------------------"
|
|
|
|
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
|
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
jq --arg active "$TARGET_STACK" \
|
|
--arg inactive "$SOURCE_STACK" \
|
|
--arg ts "$TIMESTAMP" \
|
|
'.active_stack = $active | .inactive_stack = $inactive | .last_switch = $ts' \
|
|
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
|
|
|
echo " Active stack: $TARGET_STACK"
|
|
echo " Inactive stack: $SOURCE_STACK"
|
|
fi
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "Traffic Switch COMPLETE"
|
|
echo "All traffic now routed to: $TARGET_STACK"
|
|
echo "========================================"
|
|
|
|
exit 0
|