fix: CI/CD blue-green deployment path bug causing stale production content

Root cause: switch-traffic.sh was modifying Traefik config in the CI checkout directory ($GITHUB_WORKSPACE) instead of the deployment directory ($DEPLOY_PATH). Traefik never saw the weight changes, so traffic stayed on old containers. Changes: - Add DEPLOY_PATH environment variable support to all CI scripts - Add --force-recreate flag to ensure containers are recreated with new images - Add image verification step to confirm containers use expected images - Add weight verification to confirm Traefik routing was updated - Add routing validation step to verify traffic switch succeeded 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-31 10:37:18 -06:00
parent 3321d826a2
commit 13abbc16d7
4 changed files with 106 additions and 9 deletions
--- a/.gitea/workflows/production.yaml
+++ b/.gitea/workflows/production.yaml
@@ -136,21 +136,59 @@ jobs:
          docker pull $BACKEND_IMAGE
          docker pull $FRONTEND_IMAGE

+      - name: Record expected image IDs
+        id: expected-images
+        run: |
+          # Get the image IDs we just pulled - these are what containers should use
+          FRONTEND_ID=$(docker images --format '{{.ID}}' $FRONTEND_IMAGE | head -1)
+          BACKEND_ID=$(docker images --format '{{.ID}}' $BACKEND_IMAGE | head -1)
+          echo "Expected frontend image ID: $FRONTEND_ID"
+          echo "Expected backend image ID: $BACKEND_ID"
+          echo "frontend_id=$FRONTEND_ID" >> $GITHUB_OUTPUT
+          echo "backend_id=$BACKEND_ID" >> $GITHUB_OUTPUT
+
      - name: Start target stack
        run: |
          cd "$DEPLOY_PATH"
          export BACKEND_IMAGE=$BACKEND_IMAGE
          export FRONTEND_IMAGE=$FRONTEND_IMAGE
-          docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \
+          # --force-recreate ensures containers are recreated even if image tag is same
+          # This prevents stale container content when image digest changes
+          docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d --force-recreate \
            mvp-frontend-$TARGET_STACK mvp-backend-$TARGET_STACK

      - name: Wait for stack initialization
        run: sleep 10

+      - name: Verify container images
+        run: |
+          # Verify containers are running the expected images
+          EXPECTED_FRONTEND="${{ steps.expected-images.outputs.frontend_id }}"
+          EXPECTED_BACKEND="${{ steps.expected-images.outputs.backend_id }}"
+
+          RUNNING_FRONTEND=$(docker inspect --format='{{.Image}}' mvp-frontend-$TARGET_STACK | sed 's/sha256://' | cut -c1-12)
+          RUNNING_BACKEND=$(docker inspect --format='{{.Image}}' mvp-backend-$TARGET_STACK | sed 's/sha256://' | cut -c1-12)
+
+          echo "Frontend - Expected: $EXPECTED_FRONTEND, Running: $RUNNING_FRONTEND"
+          echo "Backend - Expected: $EXPECTED_BACKEND, Running: $RUNNING_BACKEND"
+
+          if [[ "$RUNNING_FRONTEND" != "$EXPECTED_FRONTEND" ]]; then
+            echo "ERROR: Frontend container not using expected image!"
+            echo "Container may be stale. Force recreate should have prevented this."
+            exit 1
+          fi
+
+          if [[ "$RUNNING_BACKEND" != "$EXPECTED_BACKEND" ]]; then
+            echo "ERROR: Backend container not using expected image!"
+            exit 1
+          fi
+
+          echo "OK: All containers using correct images"
+
      - name: Run health check
        run: |
          chmod +x "$GITHUB_WORKSPACE/scripts/ci/health-check.sh"
-          "$GITHUB_WORKSPACE/scripts/ci/health-check.sh" $TARGET_STACK $HEALTH_CHECK_TIMEOUT
+          DEPLOY_PATH="$DEPLOY_PATH" "$GITHUB_WORKSPACE/scripts/ci/health-check.sh" $TARGET_STACK $HEALTH_CHECK_TIMEOUT

      - name: Start Traefik
        run: |
@@ -171,7 +209,8 @@ jobs:
      - name: Switch traffic
        run: |
          chmod +x "$GITHUB_WORKSPACE/scripts/ci/switch-traffic.sh"
-          "$GITHUB_WORKSPACE/scripts/ci/switch-traffic.sh" $TARGET_STACK instant
+          # DEPLOY_PATH ensures script modifies config at /opt/motovaultpro, not checkout dir
+          DEPLOY_PATH="$DEPLOY_PATH" "$GITHUB_WORKSPACE/scripts/ci/switch-traffic.sh" $TARGET_STACK instant

      - name: Update deployment state
        run: |
@@ -250,6 +289,32 @@ jobs:
            echo "OK: $service is running and healthy"
          done

+      - name: Validate Traefik routing weights
+        run: |
+          # Verify traffic has actually switched to the new stack
+          BLUE_GREEN_CONFIG="$DEPLOY_PATH/config/traefik/dynamic/blue-green.yml"
+
+          if [[ "$TARGET_STACK" == "green" ]]; then
+            EXPECTED_TARGET_WEIGHT=100
+            EXPECTED_OTHER_WEIGHT=0
+            TARGET_SVC="mvp-frontend-green-svc"
+          else
+            EXPECTED_TARGET_WEIGHT=100
+            EXPECTED_OTHER_WEIGHT=0
+            TARGET_SVC="mvp-frontend-blue-svc"
+          fi
+
+          ACTUAL_WEIGHT=$(grep -A1 "$TARGET_SVC" "$BLUE_GREEN_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
+
+          if [[ "$ACTUAL_WEIGHT" != "$EXPECTED_TARGET_WEIGHT" ]]; then
+            echo "ERROR: Traffic not routed to $TARGET_STACK stack!"
+            echo "Expected weight for $TARGET_SVC: $EXPECTED_TARGET_WEIGHT, Actual: $ACTUAL_WEIGHT"
+            cat "$BLUE_GREEN_CONFIG" | grep -A2 weight
+            exit 1
+          fi
+
+          echo "OK: Traffic correctly routed to $TARGET_STACK (weight: $ACTUAL_WEIGHT)"
+
  # ============================================
  # ROLLBACK - Auto-rollback on failure
  # ============================================
@@ -269,7 +334,7 @@ jobs:
      - name: Execute rollback
        run: |
          chmod +x "$GITHUB_WORKSPACE/scripts/ci/auto-rollback.sh"
-          "$GITHUB_WORKSPACE/scripts/ci/auto-rollback.sh" "Production verification failed - automatic rollback"
+          DEPLOY_PATH="$DEPLOY_PATH" "$GITHUB_WORKSPACE/scripts/ci/auto-rollback.sh" "Production verification failed - automatic rollback"

      - name: Update state
        run: |
--- a/scripts/ci/auto-rollback.sh
+++ b/scripts/ci/auto-rollback.sh
@@ -11,8 +11,11 @@

 set -euo pipefail

+# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
+
+echo "Using PROJECT_ROOT: $PROJECT_ROOT"

 REASON="${1:-Automatic rollback triggered}"

--- a/scripts/ci/health-check.sh
+++ b/scripts/ci/health-check.sh
@@ -12,8 +12,10 @@

 set -euo pipefail

-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
+PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
+
+echo "Using PROJECT_ROOT: $PROJECT_ROOT"

 STACK="${1:-}"
 TIMEOUT="${2:-60}"
--- a/scripts/ci/switch-traffic.sh
+++ b/scripts/ci/switch-traffic.sh
@@ -14,8 +14,12 @@

 set -euo pipefail

-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# Use DEPLOY_PATH if set (CI environment), otherwise calculate from script location
+# This is critical: CI workflows must pass DEPLOY_PATH to ensure we modify
+# the actual deployment config, not the checkout directory
+PROJECT_ROOT="${DEPLOY_PATH:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
+
+echo "Using PROJECT_ROOT: $PROJECT_ROOT"

 TARGET_STACK="${1:-}"
 MODE="${2:-instant}"
@@ -71,6 +75,27 @@ update_weights() {
  sleep 1
 }

+# Verify weights were actually written to config file
+verify_weights_applied() {
+  local expected_blue="$1"
+  local expected_green="$2"
+
+  # Extract actual weights from the config file
+  local actual_blue=$(grep -A1 "mvp-frontend-blue-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
+  local actual_green=$(grep -A1 "mvp-frontend-green-svc" "$TRAEFIK_CONFIG" | grep weight | grep -oE '[0-9]+' | head -1)
+
+  if [[ "$actual_blue" != "$expected_blue" ]] || [[ "$actual_green" != "$expected_green" ]]; then
+    echo "  ERROR: Weight verification failed!"
+    echo "  Expected: blue=$expected_blue, green=$expected_green"
+    echo "  Actual: blue=$actual_blue, green=$actual_green"
+    echo "  Config file: $TRAEFIK_CONFIG"
+    return 1
+  fi
+
+  echo "  OK: Weights verified (blue=$actual_blue, green=$actual_green)"
+  return 0
+}
+
 # Verify Traefik has picked up the changes
 verify_traefik_reload() {
  # Give Traefik time to reload config
@@ -123,8 +148,10 @@ else

  if [[ "$TARGET_STACK" == "blue" ]]; then
    update_weights 100 0
+    verify_weights_applied 100 0 || exit 1
  else
    update_weights 0 100
+    verify_weights_applied 0 100 || exit 1
  fi

  verify_traefik_reload