fix: CI/CD blue-green deployment path bug causing stale production content

Root cause: switch-traffic.sh was modifying Traefik config in the CI checkout
directory ($GITHUB_WORKSPACE) instead of the deployment directory ($DEPLOY_PATH).
Traefik never saw the weight changes, so traffic stayed on old containers.

Changes:
- Add DEPLOY_PATH environment variable support to all CI scripts
- Add --force-recreate flag to ensure containers are recreated with new images
- Add image verification step to confirm containers use expected images
- Add weight verification to confirm Traefik routing was updated
- Add routing validation step to verify traffic switch succeeded

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Eric Gullickson
2025-12-31 10:37:18 -06:00
parent 3321d826a2
commit 13abbc16d7
4 changed files with 106 additions and 9 deletions

View File

@@ -136,21 +136,59 @@ jobs:
docker pull $BACKEND_IMAGE
docker pull $FRONTEND_IMAGE
- name: Record expected image IDs
id: expected-images
run: |
# Get the image IDs we just pulled - these are what containers should use
FRONTEND_ID=$(docker images --format '{{.ID}}' $FRONTEND_IMAGE | head -1)
BACKEND_ID=$(docker images --format '{{.ID}}' $BACKEND_IMAGE | head -1)
echo "Expected frontend image ID: $FRONTEND_ID"
echo "Expected backend image ID: $BACKEND_ID"
echo "frontend_id=$FRONTEND_ID" >> $GITHUB_OUTPUT
echo "backend_id=$BACKEND_ID" >> $GITHUB_OUTPUT
- name: Start target stack
run: |
cd "$DEPLOY_PATH"
export BACKEND_IMAGE=$BACKEND_IMAGE
export FRONTEND_IMAGE=$FRONTEND_IMAGE
docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \
# --force-recreate ensures containers are recreated even if image tag is same
# This prevents stale container content when image digest changes
docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d --force-recreate \
mvp-frontend-$TARGET_STACK mvp-backend-$TARGET_STACK
- name: Wait for stack initialization
run: sleep 10
- name: Verify container images
run: |
# Verify containers are running the expected images
EXPECTED_FRONTEND="${{ steps.expected-images.outputs.frontend_id }}"
EXPECTED_BACKEND="${{ steps.expected-images.outputs.backend_id }}"
RUNNING_FRONTEND=$(docker inspect --format='{{.Image}}' mvp-frontend-$TARGET_STACK | sed 's/sha256://' | cut -c1-12)
RUNNING_BACKEND=$(docker inspect --format='{{.Image}}' mvp-backend-$TARGET_STACK | sed 's/sha256://' | cut -c1-12)
echo "Frontend - Expected: $EXPECTED_FRONTEND, Running: $RUNNING_FRONTEND"
echo "Backend - Expected: $EXPECTED_BACKEND, Running: $RUNNING_BACKEND"
if [[ "$RUNNING_FRONTEND" != "$EXPECTED_FRONTEND" ]]; then
echo "ERROR: Frontend container not using expected image!"
echo "Container may be stale. Force recreate should have prevented this."
exit 1
fi
if [[ "$RUNNING_BACKEND" != "$EXPECTED_BACKEND" ]]; then
echo "ERROR: Backend container not using expected image!"
exit 1
fi
echo "OK: All containers using correct images"
- name: Run health check
run: |
chmod +x "$GITHUB_WORKSPACE/scripts/ci/health-check.sh"
"$GITHUB_WORKSPACE/scripts/ci/health-check.sh" $TARGET_STACK $HEALTH_CHECK_TIMEOUT
DEPLOY_PATH="$DEPLOY_PATH" "$GITHUB_WORKSPACE/scripts/ci/health-check.sh" $TARGET_STACK $HEALTH_CHECK_TIMEOUT
- name: Start Traefik
run: |
@@ -171,7 +209,8 @@ jobs:
- name: Switch traffic
run: |
chmod +x "$GITHUB_WORKSPACE/scripts/ci/switch-traffic.sh"
"$GITHUB_WORKSPACE/scripts/ci/switch-traffic.sh" $TARGET_STACK instant
# DEPLOY_PATH ensures script modifies config at /opt/motovaultpro, not checkout dir
DEPLOY_PATH="$DEPLOY_PATH" "$GITHUB_WORKSPACE/scripts/ci/switch-traffic.sh" $TARGET_STACK instant
- name: Update deployment state
run: |
@@ -250,6 +289,32 @@ jobs:
echo "OK: $service is running and healthy"
done
- name: Validate Traefik routing weights
run: |
# Verify traffic has actually switched to the new stack
BLUE_GREEN_CONFIG="$DEPLOY_PATH/config/traefik/dynamic/blue-green.yml"
if [[ "$TARGET_STACK" == "green" ]]; then
EXPECTED_TARGET_WEIGHT=100
EXPECTED_OTHER_WEIGHT=0
TARGET_SVC="mvp-frontend-green-svc"
else
EXPECTED_TARGET_WEIGHT=100
EXPECTED_OTHER_WEIGHT=0
TARGET_SVC="mvp-frontend-blue-svc"
fi
ACTUAL_WEIGHT=$(grep -A1 "$TARGET_SVC" "$BLUE_GREEN_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
if [[ "$ACTUAL_WEIGHT" != "$EXPECTED_TARGET_WEIGHT" ]]; then
echo "ERROR: Traffic not routed to $TARGET_STACK stack!"
echo "Expected weight for $TARGET_SVC: $EXPECTED_TARGET_WEIGHT, Actual: $ACTUAL_WEIGHT"
cat "$BLUE_GREEN_CONFIG" | grep -A2 weight
exit 1
fi
echo "OK: Traffic correctly routed to $TARGET_STACK (weight: $ACTUAL_WEIGHT)"
# ============================================
# ROLLBACK - Auto-rollback on failure
# ============================================
@@ -269,7 +334,7 @@ jobs:
- name: Execute rollback
run: |
chmod +x "$GITHUB_WORKSPACE/scripts/ci/auto-rollback.sh"
"$GITHUB_WORKSPACE/scripts/ci/auto-rollback.sh" "Production verification failed - automatic rollback"
DEPLOY_PATH="$DEPLOY_PATH" "$GITHUB_WORKSPACE/scripts/ci/auto-rollback.sh" "Production verification failed - automatic rollback"
- name: Update state
run: |