egullickson/motovaultpro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: CI/CD blue-green deployment path bug causing stale production content

Browse Source 
Root cause: switch-traffic.sh was modifying Traefik config in the CI checkout
directory ($GITHUB_WORKSPACE) instead of the deployment directory ($DEPLOY_PATH).
Traefik never saw the weight changes, so traffic stayed on old containers.

Changes:
- Add DEPLOY_PATH environment variable support to all CI scripts
- Add --force-recreate flag to ensure containers are recreated with new images
- Add image verification step to confirm containers use expected images
- Add weight verification to confirm Traefik routing was updated
- Add routing validation step to verify traffic switch succeeded

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Eric Gullickson
2025-12-31 10:37:18 -06:00
parent 3321d826a2
commit 13abbc16d7
4 changed files with 106 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/ci/auto-rollback.sh
View File

@@ -11,8 +11,11 @@
set -euo pipefail
# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
echo "Using PROJECT_ROOT: $PROJECT_ROOT"
REASON="${1:-Automatic rollback triggered}"

6
scripts/ci/health-check.sh
View File

@@ -12,8 +12,10 @@
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
echo "Using PROJECT_ROOT: $PROJECT_ROOT"
STACK="${1:-}"
TIMEOUT="${2:-60}"

31
scripts/ci/switch-traffic.sh
View File

@@ -14,8 +14,12 @@
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
# This is critical: CI workflows must pass DEPLOY_PATH to ensure we modify
# the actual deployment config, not the checkout directory
PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
echo "Using PROJECT_ROOT: $PROJECT_ROOT"
TARGET_STACK="${1:-}"
MODE="${2:-instant}"
@@ -71,6 +75,27 @@ update_weights() {
sleep 1
}
# Verify weights were actually written to config file
verify_weights_applied() {
local expected_blue="$1"
local expected_green="$2"
# Extract actual weights from the config file
local actual_blue=$(grep -A1 "mvp-frontend-blue-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
local actual_green=$(grep -A1 "mvp-frontend-green-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
if [[ "$actual_blue" != "$expected_blue" ]] || [[ "$actual_green" != "$expected_green" ]]; then
echo " ERROR: Weight verification failed!"
echo " Expected: blue=$expected_blue, green=$expected_green"
echo " Actual: blue=$actual_blue, green=$actual_green"
echo " Config file: $TRAEFIK_CONFIG"
return 1
fi
echo " OK: Weights verified (blue=$actual_blue, green=$actual_green)"
return 0
}
# Verify Traefik has picked up the changes
verify_traefik_reload() {
# Give Traefik time to reload config
@@ -123,8 +148,10 @@ else
if [[ "$TARGET_STACK" == "blue" ]]; then
update_weights 100 0
verify_weights_applied 100 0 || exit 1
else
update_weights 0 100
verify_weights_applied 0 100 || exit 1
fi
verify_traefik_reload