diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml deleted file mode 100644 index b4b5fe8..0000000 --- a/.github/workflows/build-and-push-image.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build and Push Images to Dockerhub and GHCR - -on: - push: - branches: ["main"] - release: - types: ["published"] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - service: [backend, frontend] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: "${{ secrets.DH_USER }}" - password: "${{ secrets.DH_PASS }}" - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: "${{ secrets.GHCR_USER }}" - password: "${{ secrets.GHCR_PAT }}" - - - name: Docker Metadata - id: meta - uses: docker/metadata-action@v5 - with: - context: workflow - images: | - ericgullickson/motovaultpro-${{ matrix.service }} - ghcr.io/ericgullickson/motovaultpro-${{ matrix.service }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=tag - labels: | - org.opencontainers.image.title=MotoVaultPro ${{ matrix.service }} - org.opencontainers.image.description=MotoVaultPro ${{ matrix.service }} service - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: ./${{ matrix.service }} - platforms: linux/amd64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40748c4..202b688 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,50 @@ -# MotoVaultPro GitLab CI/CD Pipeline -# GitLab 18.6+ with shell executor +# MotoVaultPro GitLab CI/CD Pipeline - Blue-Green Deployment +# GitLab 18.6+ with separate build and production runners # See docs/CICD-DEPLOY.md for complete documentation -# v1.6 +# v2.0 - Blue-Green with Auto-Rollback stages: - validate - build - - deploy + - deploy-prepare + - deploy-switch - verify + - rollback + - notify variables: - # Use stable clone path instead of runner-specific path - GIT_CLONE_PATH: $CI_BUILDS_DIR/motovaultpro - DEPLOY_PATH: $CI_BUILDS_DIR/motovaultpro - DOCKER_COMPOSE_FILE: docker-compose.yml - DOCKER_COMPOSE_PROD_FILE: docker-compose.prod.yml + # Registry configuration + REGISTRY: registry.motovaultpro.com + REGISTRY_MIRRORS: ${REGISTRY}/mirrors + IMAGE_TAG: ${CI_COMMIT_SHORT_SHA} + BACKEND_IMAGE: ${REGISTRY}/motovaultpro/backend:${IMAGE_TAG} + FRONTEND_IMAGE: ${REGISTRY}/motovaultpro/frontend:${IMAGE_TAG} -# Fix permissions after every job - docker creates files as root + # Deployment configuration + GIT_CLONE_PATH: ${CI_BUILDS_DIR}/motovaultpro + DEPLOY_PATH: ${CI_BUILDS_DIR}/motovaultpro + COMPOSE_FILE: docker-compose.yml + COMPOSE_BLUE_GREEN: docker-compose.blue-green.yml + + # Health check configuration + HEALTH_CHECK_TIMEOUT: "60" + +# Default after_script to fix permissions default: after_script: - echo "Fixing file permissions..." - sudo chown -R gitlab-runner:gitlab-runner "$DEPLOY_PATH" 2>/dev/null || true - # Keep data directories owned by container user - sudo chown -R 1001:1001 "$DEPLOY_PATH/data/backups" "$DEPLOY_PATH/data/documents" 2>/dev/null || true -# Validate Stage - Check prerequisites +# ============================================ +# Stage 1: VALIDATE +# Check prerequisites before starting pipeline +# ============================================ validate: stage: validate + tags: + - production + - shell only: - main script: @@ -34,129 +52,385 @@ validate: - echo "Validating deployment prerequisites..." - echo "==========================================" - echo "Checking Docker..." - - 'docker info > /dev/null 2>&1 || (echo "ERROR: Docker not accessible" && exit 1)' - - echo "OK Docker is accessible" + - docker info > /dev/null 2>&1 || (echo "ERROR - Docker not accessible" && exit 1) + - echo "OK - Docker is accessible" - echo "Checking Docker Compose..." - - 'docker compose version > /dev/null 2>&1 || (echo "ERROR: Docker Compose not available" && exit 1)' - - echo "OK Docker Compose is available" + - docker compose version > /dev/null 2>&1 || (echo "ERROR - Docker Compose not available" && exit 1) + - echo "OK - Docker Compose is available" - echo "Checking deployment path..." - - 'test -d "$DEPLOY_PATH" || (echo "ERROR: DEPLOY_PATH not found" && exit 1)' - - echo "OK Deployment path exists" + - test -d "$DEPLOY_PATH" || (echo "ERROR - DEPLOY_PATH not found" && exit 1) + - echo "OK - Deployment path exists" + - echo "Checking registry access..." + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" || true + - echo "OK - Registry authentication configured" + - echo "Determining target stack..." + - | + STATE_FILE="$DEPLOY_PATH/config/deployment/state.json" + if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then + ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE") + if [ "$ACTIVE_STACK" = "blue" ]; then + echo "TARGET_STACK=green" >> deploy.env + else + echo "TARGET_STACK=blue" >> deploy.env + fi + else + echo "TARGET_STACK=green" >> deploy.env + fi + cat deploy.env - echo "==========================================" - echo "Validation complete" - echo "==========================================" + artifacts: + reports: + dotenv: deploy.env -# Build Stage - Build Docker images +# ============================================ +# Stage 2: BUILD +# Build and push images to GitLab Container Registry +# Runs on dedicated build server (shell executor) +# ============================================ build: stage: build + tags: + - build only: - main script: + - echo "Authenticating with GitLab Container Registry..." + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" - echo "==========================================" - echo "Building Docker images..." + - echo "Commit - ${CI_COMMIT_SHORT_SHA}" + - echo "Backend - ${BACKEND_IMAGE}" + - echo "Frontend - ${FRONTEND_IMAGE}" - echo "==========================================" - - cd "$DEPLOY_PATH" - - echo "Building images..." - - docker compose -f $DOCKER_COMPOSE_FILE build --no-cache + + # Build backend + - echo "Building backend..." + - | + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from ${REGISTRY}/motovaultpro/backend:latest \ + -t ${BACKEND_IMAGE} \ + -t ${REGISTRY}/motovaultpro/backend:latest \ + -f backend/Dockerfile \ + . + + # Build frontend + - echo "Building frontend..." + - | + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg VITE_AUTH0_DOMAIN=${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com} \ + --build-arg VITE_AUTH0_CLIENT_ID=${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3} \ + --build-arg VITE_AUTH0_AUDIENCE=${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com} \ + --build-arg VITE_API_BASE_URL=/api \ + --cache-from ${REGISTRY}/motovaultpro/frontend:latest \ + -t ${FRONTEND_IMAGE} \ + -t ${REGISTRY}/motovaultpro/frontend:latest \ + -f frontend/Dockerfile \ + frontend + + # Push images + - echo "Pushing images to registry..." + - docker push ${BACKEND_IMAGE} + - docker push ${FRONTEND_IMAGE} + - docker push ${REGISTRY}/motovaultpro/backend:latest + - docker push ${REGISTRY}/motovaultpro/frontend:latest + - echo "==========================================" - echo "Build complete" - echo "==========================================" -# Deploy Stage - Inject secrets and deploy services -deploy: - stage: deploy +# ============================================ +# Stage 3: DEPLOY-PREPARE +# Pull images, start inactive stack, run health checks +# ============================================ +deploy-prepare: + stage: deploy-prepare + tags: + - production + - shell only: - main + needs: + - job: validate + artifacts: true + - job: build environment: name: production url: https://motovaultpro.com script: - echo "==========================================" - - echo "Deploying MotoVaultPro..." + - echo "Preparing deployment to ${TARGET_STACK} stack..." - echo "==========================================" - cd "$DEPLOY_PATH" - - echo "Step 1/8 Initializing data directories..." + + # Authenticate with registry + - echo "Authenticating with registry..." + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" + + # Inject secrets + - echo "Step 1/5 - Injecting secrets..." + - chmod +x scripts/inject-secrets.sh + - ./scripts/inject-secrets.sh + + # Initialize data directories + - echo "Step 2/5 - Initializing data directories..." - sudo mkdir -p data/backups data/documents - sudo chown -R 1001:1001 data/backups data/documents - sudo chmod 755 data/backups data/documents - - echo "Step 2/8 Injecting secrets..." - - chmod +x scripts/inject-secrets.sh - - ./scripts/inject-secrets.sh - - echo "Step 3/8 Stopping existing services..." - - docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE down --timeout 30 || true - - echo "Step 4/8 Pulling base images..." - - docker compose -f $DOCKER_COMPOSE_FILE pull - - echo "Step 5/8 Starting database services..." - - docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d mvp-postgres mvp-redis - - echo "Waiting for database to be ready..." - - sleep 15 - - echo "Step 6/8 Running database migrations..." - - docker compose -f $DOCKER_COMPOSE_FILE run --rm mvp-backend npm run migrate || echo "Migration skipped" - - echo "Step 7/8 Vehicle catalog data..." - # Schema and data now loaded via standard migration system - # Migration runner handles table creation and data loading automatically - - echo "Vehicle catalog loaded via platform feature migration" - - echo "Flushing Redis cache..." - - docker exec mvp-redis redis-cli FLUSHALL - - echo "Step 8/8 Starting all services..." - - docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d - - echo "Waiting for services to initialize..." - - sleep 30 + + # Pull new images + - echo "Step 3/5 - Pulling images..." + - docker pull ${BACKEND_IMAGE} + - docker pull ${FRONTEND_IMAGE} + + # Start inactive stack + - echo "Step 4/5 - Starting ${TARGET_STACK} stack..." + - | + export BACKEND_IMAGE=${BACKEND_IMAGE} + export FRONTEND_IMAGE=${FRONTEND_IMAGE} + docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \ + mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK} + + # Wait for stack to be ready + - echo "Step 5/5 - Waiting for stack health..." + - sleep 10 + + # Run health check + - echo "Running health check on ${TARGET_STACK} stack..." + - chmod +x scripts/ci/health-check.sh + - ./scripts/ci/health-check.sh ${TARGET_STACK} ${HEALTH_CHECK_TIMEOUT} + + # Update state with deployment info + - | + STATE_FILE="config/deployment/state.json" + if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + jq --arg stack "$TARGET_STACK" \ + --arg commit "$CI_COMMIT_SHORT_SHA" \ + --arg ts "$TIMESTAMP" \ + '.[$stack].version = $commit | .[$stack].commit = $commit | .[$stack].deployed_at = $ts | .[$stack].healthy = true' \ + "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" + fi + - echo "==========================================" - - echo "Deployment complete" + - echo "Deploy preparation complete" - echo "==========================================" -# Verify Stage - Health checks -verify: - stage: verify +# ============================================ +# Stage 4: DEPLOY-SWITCH +# Switch traffic to new stack +# ============================================ +deploy-switch: + stage: deploy-switch + tags: + - production + - shell only: - main + needs: + - job: validate + artifacts: true + - job: deploy-prepare script: - echo "==========================================" - - echo "Verifying deployment..." + - echo "Switching traffic to ${TARGET_STACK} stack..." - echo "==========================================" - cd "$DEPLOY_PATH" - - echo "Checking container status..." + + # Switch traffic + - chmod +x scripts/ci/switch-traffic.sh + - ./scripts/ci/switch-traffic.sh ${TARGET_STACK} instant + + # Update state - | - FAILED=0 - for service in mvp-traefik mvp-frontend mvp-backend mvp-postgres mvp-redis; do - status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found") - if [ "$status" != "running" ]; then - echo "ERROR: $service is not running (status: $status)" - docker logs $service --tail 50 2>/dev/null || true - FAILED=1 - else - echo "OK: $service is running" - fi - done - if [ $FAILED -eq 1 ]; then - echo "One or more services failed to start" - exit 1 + STATE_FILE="config/deployment/state.json" + if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + jq --arg commit "$CI_COMMIT_SHORT_SHA" \ + --arg ts "$TIMESTAMP" \ + '.last_deployment = $ts | .last_deployment_commit = $commit | .last_deployment_status = "success" | .rollback_available = true' \ + "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" fi - - echo "Checking backend health..." + + - echo "==========================================" + - echo "Traffic switch complete" + - echo "==========================================" + +# ============================================ +# Stage 5: VERIFY +# Production health verification after switch +# ============================================ +verify: + stage: verify + tags: + - production + - shell + only: + - main + needs: + - job: validate + artifacts: true + - job: deploy-switch + script: + - echo "==========================================" + - echo "Verifying production deployment..." + - echo "==========================================" + - cd "$DEPLOY_PATH" + + # Wait for Traefik to propagate routing + - echo "Waiting for traffic routing to stabilize..." + - sleep 5 + + # Verify via external endpoint + - echo "Checking external endpoint..." - | - HEALTH_OK=0 for i in 1 2 3 4 5 6; do - if docker exec mvp-backend curl -sf http://localhost:3001/health > /dev/null 2>&1; then - echo "OK: Backend health check passed" - HEALTH_OK=1 + if curl -sf https://motovaultpro.com/api/health > /dev/null 2>&1; then + echo "OK - External health check passed" break fi - echo "Attempt $i/6: Backend not ready, waiting 10s..." + if [ $i -eq 6 ]; then + echo "ERROR - External health check failed after 6 attempts" + exit 1 + fi + echo "Attempt $i/6 - Waiting 10s..." sleep 10 done - if [ $HEALTH_OK -eq 0 ]; then - echo "ERROR: Backend health check failed after 6 attempts" - docker logs mvp-backend --tail 100 - exit 1 - fi - - echo "Checking frontend..." + + # Verify container status + - echo "Checking container status..." - | - if docker compose -f $DOCKER_COMPOSE_FILE exec -T mvp-frontend curl -sf http://localhost:3000 > /dev/null 2>&1; then - echo "OK: Frontend is accessible" - else - echo "WARNING: Frontend check failed (might need Traefik routing)" - fi + for service in mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}; do + status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found") + health=$(docker inspect --format='{{.State.Health.Status}}' $service 2>/dev/null || echo "unknown") + if [ "$status" != "running" ] || [ "$health" != "healthy" ]; then + echo "ERROR - $service is not healthy (status: $status, health: $health)" + docker logs $service --tail 50 2>/dev/null || true + exit 1 + fi + echo "OK - $service is running and healthy" + done + - echo "==========================================" - echo "Deployment verified successfully!" + - echo "Version ${CI_COMMIT_SHORT_SHA} is now live" - echo "==========================================" + +# ============================================ +# Stage 6: ROLLBACK (on failure) +# Automatic rollback if verify stage fails +# ============================================ +rollback: + stage: rollback + tags: + - production + - shell + only: + - main + when: on_failure + needs: + - job: validate + artifacts: true + - job: deploy-switch + - job: verify + script: + - echo "==========================================" + - echo "INITIATING AUTO-ROLLBACK" + - echo "==========================================" + - cd "$DEPLOY_PATH" + + # Run rollback script + - chmod +x scripts/ci/auto-rollback.sh + - ./scripts/ci/auto-rollback.sh "Verify stage failed - automatic rollback" + + # Update state + - | + STATE_FILE="config/deployment/state.json" + if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then + jq '.last_deployment_status = "rolled_back"' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" + fi + + - echo "==========================================" + - echo "Rollback complete" + - echo "==========================================" + +# ============================================ +# Stage 7: NOTIFY +# Send deployment notifications +# ============================================ +notify-success: + stage: notify + tags: + - production + - shell + only: + - main + needs: + - job: verify + script: + - echo "Sending success notification..." + - cd "$DEPLOY_PATH" + - chmod +x scripts/ci/notify.sh + - ./scripts/ci/notify.sh success "Version ${CI_COMMIT_SHORT_SHA} deployed successfully" ${CI_COMMIT_SHORT_SHA} + +notify-failure: + stage: notify + tags: + - production + - shell + only: + - main + when: on_failure + needs: + - job: build + optional: true + - job: deploy-prepare + optional: true + - job: deploy-switch + optional: true + - job: verify + optional: true + script: + - echo "Sending failure notification..." + - cd "$DEPLOY_PATH" + - chmod +x scripts/ci/notify.sh + - ./scripts/ci/notify.sh failure "Deployment of ${CI_COMMIT_SHORT_SHA} failed" ${CI_COMMIT_SHORT_SHA} + +# ============================================ +# Manual Jobs +# ============================================ + +# Manual maintenance migration job +maintenance-migration: + stage: deploy-prepare + tags: + - production + - shell + only: + - main + when: manual + script: + - echo "==========================================" + - echo "MAINTENANCE MODE MIGRATION" + - echo "==========================================" + - cd "$DEPLOY_PATH" + - chmod +x scripts/ci/maintenance-migrate.sh + - ./scripts/ci/maintenance-migrate.sh backup + +# Mirror base images (scheduled or manual) +mirror-images: + stage: build + tags: + - build + only: + - schedules + - web + when: manual + script: + - echo "Mirroring base images to GitLab Container Registry..." + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" + - chmod +x scripts/ci/mirror-base-images.sh + - REGISTRY=${REGISTRY}/mirrors ./scripts/ci/mirror-base-images.sh diff --git a/backend/Dockerfile b/backend/Dockerfile index 5362640..7de3b33 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,7 +1,11 @@ # Production Dockerfile for MotoVaultPro Backend +# Uses mirrored base images from GitLab Container Registry + +# Build argument for registry (defaults to GitLab mirrors, falls back to Docker Hub) +ARG REGISTRY_MIRRORS=registry.motovaultpro.com/mirrors # Stage 1: Build stage -FROM node:lts-alpine AS builder +FROM ${REGISTRY_MIRRORS}/node:20-alpine AS builder # Install build dependencies RUN apk add --no-cache dumb-init git curl @@ -27,7 +31,7 @@ ENV DOCKER_BUILD=true RUN npm run build # Stage 2: Production runtime -FROM node:lts-alpine AS production +FROM ${REGISTRY_MIRRORS}/node:20-alpine AS production # Install runtime dependencies only (postgresql-client for backup/restore) RUN apk add --no-cache dumb-init curl postgresql-client diff --git a/config/deployment/state.json b/config/deployment/state.json new file mode 100644 index 0000000..e3dd8cd --- /dev/null +++ b/config/deployment/state.json @@ -0,0 +1,21 @@ +{ + "active_stack": "blue", + "inactive_stack": "green", + "last_deployment": null, + "last_deployment_commit": null, + "last_deployment_status": null, + "blue": { + "version": null, + "commit": null, + "deployed_at": null, + "healthy": false + }, + "green": { + "version": null, + "commit": null, + "deployed_at": null, + "healthy": false + }, + "rollback_available": false, + "maintenance_mode": false +} diff --git a/config/traefik/dynamic/blue-green.yml b/config/traefik/dynamic/blue-green.yml new file mode 100644 index 0000000..2bad4dd --- /dev/null +++ b/config/traefik/dynamic/blue-green.yml @@ -0,0 +1,116 @@ +# Traefik Dynamic Configuration for Blue-Green Deployment +# This file is watched by Traefik and reloaded on changes +# Traffic weights are updated by scripts/ci/switch-traffic.sh +# +# Current active stack is determined by weights: +# - blue=100, green=0 -> Blue is active +# - blue=0, green=100 -> Green is active +# - Gradual: 75/25, 50/50, 25/75 for canary deployments + +http: + # ======================================== + # Routers - Route traffic to weighted services + # ======================================== + routers: + # Frontend router with weighted service + mvp-frontend-bluegreen: + rule: "(Host(`motovaultpro.com`) || Host(`www.motovaultpro.com`)) && !PathPrefix(`/api`)" + entryPoints: + - websecure + tls: + certResolver: letsencrypt + service: mvp-frontend-weighted + priority: 10 + + # Backend API router with weighted service + mvp-backend-bluegreen: + rule: "(Host(`motovaultpro.com`) || Host(`www.motovaultpro.com`)) && PathPrefix(`/api`)" + entryPoints: + - websecure + tls: + certResolver: letsencrypt + service: mvp-backend-weighted + priority: 20 + + # Health check router (always routes to active stack) + mvp-backend-health: + rule: "(Host(`motovaultpro.com`) || Host(`www.motovaultpro.com`)) && Path(`/api/health`)" + entryPoints: + - websecure + tls: + certResolver: letsencrypt + service: mvp-backend-weighted + priority: 30 + + # ======================================== + # Services - Weighted load balancers + # ======================================== + services: + # Frontend weighted service + # Weights are updated by switch-traffic.sh + mvp-frontend-weighted: + weighted: + services: + - name: mvp-frontend-blue-svc + weight: 100 + - name: mvp-frontend-green-svc + weight: 0 + healthCheck: {} + + # Backend weighted service + # Weights are updated by switch-traffic.sh + mvp-backend-weighted: + weighted: + services: + - name: mvp-backend-blue-svc + weight: 100 + - name: mvp-backend-green-svc + weight: 0 + healthCheck: {} + + # Individual stack services + mvp-frontend-blue-svc: + loadBalancer: + servers: + - url: "http://mvp-frontend-blue:3000" + healthCheck: + path: / + interval: 10s + timeout: 3s + passHostHeader: true + + mvp-frontend-green-svc: + loadBalancer: + servers: + - url: "http://mvp-frontend-green:3000" + healthCheck: + path: / + interval: 10s + timeout: 3s + passHostHeader: true + + mvp-backend-blue-svc: + loadBalancer: + servers: + - url: "http://mvp-backend-blue:3001" + healthCheck: + path: /health + interval: 10s + timeout: 3s + passHostHeader: true + + mvp-backend-green-svc: + loadBalancer: + servers: + - url: "http://mvp-backend-green:3001" + healthCheck: + path: /health + interval: 10s + timeout: 3s + passHostHeader: true + + # Maintenance mode service (optional) + mvp-maintenance: + loadBalancer: + servers: + - url: "http://mvp-maintenance:80" diff --git a/config/traefik/dynamic/middleware.yml b/config/traefik/dynamic/middleware.yml new file mode 100755 index 0000000..a827196 --- /dev/null +++ b/config/traefik/dynamic/middleware.yml @@ -0,0 +1,180 @@ +http: + middlewares: + # Security headers middleware + secure-headers: + headers: + accessControlAllowMethods: + - GET + - OPTIONS + - PUT + - POST + - DELETE + accessControlAllowOriginList: + - "https://admin.motovaultpro.com" + - "https://motovaultpro.com" + accessControlMaxAge: 100 + addVaryHeader: true + browserXssFilter: true + contentTypeNosniff: true + forceSTSHeader: true + frameDeny: true + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + customRequestHeaders: + X-Forwarded-Proto: https + + # CORS middleware for API endpoints + cors: + headers: + accessControlAllowCredentials: true + accessControlAllowHeaders: + - "Authorization" + - "Content-Type" + - "X-Requested-With" + - "X-Tenant-ID" + accessControlAllowMethods: + - "GET" + - "POST" + - "PUT" + - "DELETE" + - "OPTIONS" + accessControlAllowOriginList: + - "https://admin.motovaultpro.com" + - "https://motovaultpro.com" + accessControlMaxAge: 100 + + # API authentication middleware + api-auth: + forwardAuth: + address: "http://admin-backend:3001/auth/verify" + authResponseHeaders: + - "X-Auth-User" + - "X-Auth-Roles" + - "X-Tenant-ID" + authRequestHeaders: + - "Authorization" + - "X-Tenant-ID" + trustForwardHeader: true + + # Platform API authentication middleware + platform-auth: + forwardAuth: + address: "http://admin-backend:3001/auth/verify-platform" + authResponseHeaders: + - "X-Service-Name" + - "X-Auth-Scope" + authRequestHeaders: + - "X-API-Key" + - "Authorization" + trustForwardHeader: true + + # Rate limiting middleware + rate-limit: + rateLimit: + burst: 100 + average: 50 + period: 1m + + # Request/response size limits + size-limit: + buffering: + maxRequestBodyBytes: 26214400 # 25MB + maxResponseBodyBytes: 26214400 # 25MB + + # IP whitelist for development (optional) + local-ips: + ipWhiteList: + sourceRange: + - "127.0.0.1/32" + - "10.0.0.0/8" + - "172.16.0.0/12" + - "192.168.0.0/16" + + # Advanced security headers for production + security-headers-strict: + headers: + accessControlAllowCredentials: false + accessControlAllowMethods: + - GET + - POST + - OPTIONS + accessControlAllowOriginList: + - "https://admin.motovaultpro.com" + - "https://motovaultpro.com" + browserXssFilter: true + contentTypeNosniff: true + customRequestHeaders: + X-Forwarded-Proto: https + customResponseHeaders: + X-Frame-Options: DENY + X-Content-Type-Options: nosniff + Referrer-Policy: strict-origin-when-cross-origin + Permissions-Policy: "geolocation=(), microphone=(), camera=()" + forceSTSHeader: true + frameDeny: true + stsIncludeSubdomains: true + stsPreload: true + stsSeconds: 31536000 + + # Circuit breaker for reliability + circuit-breaker: + circuitBreaker: + expression: "NetworkErrorRatio() > 0.3 || ResponseCodeRatio(500, 600, 0, 600) > 0.3" + checkPeriod: 30s + fallbackDuration: 10s + recoveryDuration: 30s + + # Request retry for resilience + retry-policy: + retry: + attempts: 3 + initialInterval: 100ms + + # Timeout middleware + timeout: + timeout: 30s + + # Compress responses for performance + compression: + compress: {} + + # Health check middleware chain + health-check-chain: + chain: + middlewares: + - compression + - secure-headers + - timeout + + # API middleware chain + api-chain: + chain: + middlewares: + - compression + - security-headers-strict + - cors + - rate-limit + - api-auth + - retry-policy + - timeout + + # Platform API middleware chain + platform-chain: + chain: + middlewares: + - compression + - security-headers-strict + - rate-limit + - platform-auth + - circuit-breaker + - retry-policy + - timeout + + # Public frontend middleware chain + frontend-chain: + chain: + middlewares: + - compression + - secure-headers + - timeout \ No newline at end of file diff --git a/config/traefik/traefik.yml b/config/traefik/traefik.yml index 62167fc..bfceb1f 100755 --- a/config/traefik/traefik.yml +++ b/config/traefik/traefik.yml @@ -21,7 +21,8 @@ providers: exposedByDefault: false # Network auto-discovery - Traefik will use the networks it's connected to file: - filename: /etc/traefik/middleware.yml + # Watch directory for dynamic configuration (blue-green routing, middleware) + directory: /etc/traefik/dynamic watch: true certificatesResolvers: diff --git a/docker-compose.blue-green.yml b/docker-compose.blue-green.yml new file mode 100644 index 0000000..977aee3 --- /dev/null +++ b/docker-compose.blue-green.yml @@ -0,0 +1,196 @@ +# Blue-Green Deployment Overlay for MotoVaultPro +# Usage: docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d +# +# This overlay defines blue and green stacks that share the same database layer. +# Traffic routing is handled by Traefik's weighted load balancer. +# +# Stack naming: +# BLUE: mvp-frontend-blue, mvp-backend-blue +# GREEN: mvp-frontend-green, mvp-backend-green +# +# Shared services (from base compose): +# mvp-traefik, mvp-postgres, mvp-redis + +services: + # ======================================== + # BLUE Stack - Frontend + # ======================================== + mvp-frontend-blue: + image: ${FRONTEND_IMAGE:-registry.motovaultpro.com/motovaultpro/frontend:latest} + container_name: mvp-frontend-blue + restart: unless-stopped + environment: + VITE_API_BASE_URL: /api + VITE_AUTH0_DOMAIN: ${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com} + VITE_AUTH0_CLIENT_ID: ${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3} + VITE_AUTH0_AUDIENCE: ${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com} + SECRETS_DIR: /run/secrets + volumes: + - ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro + - ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro + networks: + - frontend + depends_on: + - mvp-backend-blue + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:3000 || exit 1"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 10s + deploy: + resources: + limits: + memory: 512M + labels: + - "traefik.enable=true" + - "traefik.docker.network=motovaultpro_frontend" + - "com.motovaultpro.stack=blue" + - "com.motovaultpro.service=frontend" + + # ======================================== + # BLUE Stack - Backend + # ======================================== + mvp-backend-blue: + image: ${BACKEND_IMAGE:-registry.motovaultpro.com/motovaultpro/backend:latest} + container_name: mvp-backend-blue + restart: unless-stopped + environment: + NODE_ENV: production + CONFIG_PATH: /app/config/production.yml + SECRETS_DIR: /run/secrets + DATABASE_HOST: mvp-postgres + REDIS_HOST: mvp-redis + volumes: + - ./config/app/production.yml:/app/config/production.yml:ro + - ./config/shared/production.yml:/app/config/shared.yml:ro + - ./secrets/app/postgres-password.txt:/run/secrets/postgres-password:ro + - ./secrets/app/auth0-client-secret.txt:/run/secrets/auth0-client-secret:ro + - ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro + - ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro + - ./secrets/app/resend-api-key.txt:/run/secrets/resend-api-key:ro + - ./secrets/app/auth0-management-client-id.txt:/run/secrets/auth0-management-client-id:ro + - ./secrets/app/auth0-management-client-secret.txt:/run/secrets/auth0-management-client-secret:ro + - ./data/documents:/app/data/documents + - ./data/backups:/app/data/backups + networks: + - backend + - database + depends_on: + - mvp-postgres + - mvp-redis + healthcheck: + test: + - CMD-SHELL + - node -e "require('http').get('http://localhost:3001/health', r => process.exit(r.statusCode===200?0:1)).on('error', () => process.exit(1))" + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + memory: 1G + labels: + - "traefik.enable=true" + - "traefik.docker.network=motovaultpro_backend" + - "com.motovaultpro.stack=blue" + - "com.motovaultpro.service=backend" + + # ======================================== + # GREEN Stack - Frontend + # ======================================== + mvp-frontend-green: + image: ${FRONTEND_IMAGE:-registry.motovaultpro.com/motovaultpro/frontend:latest} + container_name: mvp-frontend-green + restart: unless-stopped + environment: + VITE_API_BASE_URL: /api + VITE_AUTH0_DOMAIN: ${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com} + VITE_AUTH0_CLIENT_ID: ${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3} + VITE_AUTH0_AUDIENCE: ${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com} + SECRETS_DIR: /run/secrets + volumes: + - ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro + - ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro + networks: + - frontend + depends_on: + - mvp-backend-green + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:3000 || exit 1"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 10s + deploy: + resources: + limits: + memory: 512M + labels: + - "traefik.enable=true" + - "traefik.docker.network=motovaultpro_frontend" + - "com.motovaultpro.stack=green" + - "com.motovaultpro.service=frontend" + + # ======================================== + # GREEN Stack - Backend + # ======================================== + mvp-backend-green: + image: ${BACKEND_IMAGE:-registry.motovaultpro.com/motovaultpro/backend:latest} + container_name: mvp-backend-green + restart: unless-stopped + environment: + NODE_ENV: production + CONFIG_PATH: /app/config/production.yml + SECRETS_DIR: /run/secrets + DATABASE_HOST: mvp-postgres + REDIS_HOST: mvp-redis + volumes: + - ./config/app/production.yml:/app/config/production.yml:ro + - ./config/shared/production.yml:/app/config/shared.yml:ro + - ./secrets/app/postgres-password.txt:/run/secrets/postgres-password:ro + - ./secrets/app/auth0-client-secret.txt:/run/secrets/auth0-client-secret:ro + - ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro + - ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro + - ./secrets/app/resend-api-key.txt:/run/secrets/resend-api-key:ro + - ./secrets/app/auth0-management-client-id.txt:/run/secrets/auth0-management-client-id:ro + - ./secrets/app/auth0-management-client-secret.txt:/run/secrets/auth0-management-client-secret:ro + - ./data/documents:/app/data/documents + - ./data/backups:/app/data/backups + networks: + - backend + - database + depends_on: + - mvp-postgres + - mvp-redis + healthcheck: + test: + - CMD-SHELL + - node -e "require('http').get('http://localhost:3001/health', r => process.exit(r.statusCode===200?0:1)).on('error', () => process.exit(1))" + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + memory: 1G + labels: + - "traefik.enable=true" + - "traefik.docker.network=motovaultpro_backend" + - "com.motovaultpro.stack=green" + - "com.motovaultpro.service=backend" + + # ======================================== + # Override Traefik to add dynamic config + # ======================================== + mvp-traefik: + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./config/traefik/traefik.yml:/etc/traefik/traefik.yml:ro + - ./config/traefik/middleware.yml:/etc/traefik/middleware.yml:ro + - ./config/traefik/dynamic:/etc/traefik/dynamic:ro + - ./certs:/certs:ro + - traefik_data:/data + - ./secrets/app/cloudflare-dns-token.txt:/run/secrets/cloudflare-dns-token:ro diff --git a/docker-compose.yml b/docker-compose.yml index b3ce799..79c131f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,11 @@ +# Base registry for mirrored images (override with environment variable) +x-registry: ®istry + REGISTRY_MIRRORS: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors} + services: # Traefik - Service Discovery and Load Balancing mvp-traefik: - image: traefik:v3.6 + image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/traefik:v3.6 container_name: mvp-traefik restart: unless-stopped command: @@ -15,7 +19,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ./config/traefik/traefik.yml:/etc/traefik/traefik.yml:ro - - ./config/traefik/middleware.yml:/etc/traefik/middleware.yml:ro + - ./config/traefik/dynamic:/etc/traefik/dynamic:ro - ./certs:/certs:ro - traefik_data:/data - ./secrets/app/cloudflare-dns-token.txt:/run/secrets/cloudflare-dns-token:ro @@ -154,7 +158,7 @@ services: # Database Services - Application PostgreSQL mvp-postgres: - image: postgres:18-alpine + image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/postgres:18-alpine container_name: mvp-postgres restart: unless-stopped environment: @@ -179,7 +183,7 @@ services: # Database Services - Application Redis mvp-redis: - image: redis:8.4-alpine + image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/redis:8.4-alpine container_name: mvp-redis restart: unless-stopped command: redis-server --appendonly yes diff --git a/docs/BUILD-SERVER-SETUP.md b/docs/BUILD-SERVER-SETUP.md new file mode 100644 index 0000000..300cc04 --- /dev/null +++ b/docs/BUILD-SERVER-SETUP.md @@ -0,0 +1,315 @@ +# Build Server Setup Guide + +Complete guide for setting up a dedicated build VPS for MotoVaultPro CI/CD pipeline. + +## Overview + +The build server isolates resource-intensive Docker builds from the production server, ensuring deployments don't impact application performance. + +``` ++-------------------+ +--------------------+ +| GitLab Server | | Production Server | +| (CI/CD + Registry)| | (Shell Runner) | ++--------+----------+ +----------+---------+ + | | + v v ++--------+----------+ +----------+---------+ +| Build VPS | | Blue-Green Stacks | +| (Docker Runner) |---->| + Shared Data | ++-------------------+ +--------------------+ +``` + +## Server Requirements + +### Minimum Specifications + +| Resource | Requirement | +|----------|-------------| +| CPU | 2 cores | +| RAM | 4GB | +| Storage | 50GB SSD | +| Network | 100Mbps+ | +| OS | Ubuntu 22.04 LTS / Debian 12 | + +### Network Requirements + +- Outbound HTTPS to GitLab instance +- Outbound HTTPS to Docker registries (for fallback) +- SSH access for administration + +--- + +## Installation Steps + +### 1. Update System + +```bash +sudo apt update && sudo apt upgrade -y +sudo apt install -y curl git ca-certificates gnupg +``` + +### 2. Install Docker Engine + +```bash +# Add Docker's official GPG key +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg + +# Add the repository to Apt sources +echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Install Docker +sudo apt update +sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Verify installation +docker --version +docker compose version +``` + +### 3. Install GitLab Runner + +```bash +# Add GitLab Runner repository +curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash + +# Install GitLab Runner +sudo apt install gitlab-runner + +# Verify installation +gitlab-runner --version +``` + +### 4. Register Runner with Shell Executor + +```bash +sudo gitlab-runner register \ + --non-interactive \ + --url "https://git.motovaultpro.com" \ + --registration-token "YOUR_REGISTRATION_TOKEN" \ + --executor "shell" \ + --description "Build Server - Shell Executor" \ + --tag-list "build" \ + --run-untagged="false" \ + --locked="true" +``` + +**Notes:** +- Replace `YOUR_REGISTRATION_TOKEN` with the token from GitLab Admin > CI/CD > Runners +- Shell executor runs jobs directly on the host with access to Docker +- Tag `build` is used in `.gitlab-ci.yml` to route build jobs to this server + +### 5. Add gitlab-runner to Docker Group + +The gitlab-runner user needs access to Docker: + +```bash +sudo usermod -aG docker gitlab-runner + +# Verify access +sudo -u gitlab-runner docker info +sudo -u gitlab-runner docker compose version +``` + +### 6. Configure Docker Registry Authentication + +Create credentials file for GitLab Container Registry: + +```bash +# Login to GitLab Container Registry (creates ~/.docker/config.json) +docker login registry.motovaultpro.com -u -p +``` + +**Creating Deploy Token:** +1. Go to GitLab Project > Settings > Repository > Deploy Tokens +2. Create token with `read_registry` and `write_registry` scopes +3. Use the token username/password for Docker login + +--- + +## Verification + +### Test Runner Registration + +```bash +sudo gitlab-runner verify +``` + +Expected output: +``` +Verifying runner... is alive runner=XXXXXX +``` + +### Test Docker Access + +```bash +sudo gitlab-runner exec docker --docker-privileged test-job +``` + +### Test Registry Push + +```bash +# Build and push a test image +docker build -t registry.motovaultpro.com/motovaultpro/test:latest -f- . < /dev/null <<'EOF' +#!/bin/bash +# Remove unused Docker resources older than 7 days +docker system prune -af --filter "until=168h" +docker volume prune -f +EOF + +sudo chmod +x /usr/local/bin/docker-cleanup.sh + +# Add to crontab (run daily at 3 AM) +echo "0 3 * * * /usr/local/bin/docker-cleanup.sh >> /var/log/docker-cleanup.log 2>&1" | sudo crontab - +``` + +### Log Rotation + +Configure log rotation for GitLab Runner: + +```bash +sudo tee /etc/logrotate.d/gitlab-runner > /dev/null < build -> deploy-prepare -> deploy-switch -> verify -> [rollback] -> notify ``` -Expected output should show your runner as active with shell executor. +| Stage | Runner | Purpose | +|-------|--------|---------| +| `validate` | prod | Check prerequisites, determine target stack | +| `build` | build | Build and push images to GitLab registry | +| `deploy-prepare` | prod | Pull images, start inactive stack, health check | +| `deploy-switch` | prod | Switch Traefik traffic weights | +| `verify` | prod | Production health verification | +| `rollback` | prod | Auto-triggered on verify failure | +| `notify` | prod | Email success/failure notifications | -### 2. Verify Docker Permissions +### Pipeline Flow -The `gitlab-runner` user must have Docker access: - -```bash -# Add gitlab-runner to docker group (if not already done) -sudo usermod -aG docker gitlab-runner - -# Verify access -sudo -u gitlab-runner docker info -sudo -u gitlab-runner docker compose version +``` +[Push to main] + | + v +[validate] - Checks Docker, paths, registry + | + v +[build] - Builds backend + frontend images + | Pushes to registry.motovaultpro.com + v +[deploy-prepare] - Pulls new images + | Starts inactive stack (blue or green) + | Runs health checks + v +[deploy-switch] - Updates Traefik weights + | Switches traffic instantly + v +[verify] - External health check + | Container status verification + | + +--[SUCCESS]--> [notify-success] - Sends success email + | + +--[FAILURE]--> [rollback] - Switches back to previous stack + | + v + [notify-failure] - Sends failure email ``` -### 3. Verify Deployment Directory +--- -Ensure the deployment directory exists and is accessible: +## Blue-Green Deployment -```bash -# Create deployment directory -sudo mkdir -p /opt/motovaultpro -sudo chown gitlab-runner:gitlab-runner /opt/motovaultpro +### Stack Configuration -# Clone repository (first time only) -sudo -u gitlab-runner git clone /opt/motovaultpro +Both stacks share the same database layer: + +| Component | Blue Stack | Green Stack | Shared | +|-----------|------------|-------------|--------| +| Frontend | `mvp-frontend-blue` | `mvp-frontend-green` | - | +| Backend | `mvp-backend-blue` | `mvp-backend-green` | - | +| PostgreSQL | - | - | `mvp-postgres` | +| Redis | - | - | `mvp-redis` | +| Traefik | - | - | `mvp-traefik` | + +### Traffic Routing + +Traefik uses weighted services for traffic distribution: + +```yaml +# config/traefik/dynamic/blue-green.yml +services: + mvp-frontend-weighted: + weighted: + services: + - name: mvp-frontend-blue-svc + weight: 100 # Active + - name: mvp-frontend-green-svc + weight: 0 # Standby +``` + +### Deployment State + +State is tracked in `config/deployment/state.json`: + +```json +{ + "active_stack": "blue", + "inactive_stack": "green", + "last_deployment": "2024-01-15T10:30:00Z", + "last_deployment_commit": "abc123", + "rollback_available": true +} ``` --- @@ -76,410 +191,316 @@ sudo -u gitlab-runner git clone /opt/motovaultpro Navigate to **Settings > CI/CD > Variables** in your GitLab project. -### Secrets (File Type Variables) +### Required Variables -These variables use GitLab's **File** type, which writes the value to a temporary file and provides the path as the environment variable. This replicates the Kubernetes secrets pattern used by the application. - -| Variable Name | Type | Protected | Masked | Description | -|--------------|------|-----------|--------|-------------| -| `POSTGRES_PASSWORD` | File | Yes | Yes | PostgreSQL database password | -| `AUTH0_CLIENT_SECRET` | File | Yes | Yes | Auth0 client secret for backend | -| `GOOGLE_MAPS_API_KEY` | File | Yes | Yes | Google Maps API key | -| `GOOGLE_MAPS_MAP_ID` | File | Yes | No | Google Maps Map ID | -| `CF_DNS_API_TOKEN` | File | Yes | Yes | Cloudflare API token for Let's Encrypt DNS challenge | -| `RESEND_API_KEY` | File | Yes | Yes | Resend API key for email notifications | - -### Configuration Variables - -| Variable Name | Type | Protected | Masked | Value | -|--------------|------|-----------|--------|-------| -| `VITE_AUTH0_DOMAIN` | Variable | No | No | `motovaultpro.us.auth0.com` | -| `VITE_AUTH0_CLIENT_ID` | Variable | No | No | Your Auth0 client ID | -| `VITE_AUTH0_AUDIENCE` | Variable | No | No | `https://api.motovaultpro.com` | - -Note: `DEPLOY_PATH` is automatically set in `.gitlab-ci.yml` using `GIT_CLONE_PATH` for a stable path. - -### Creating Cloudflare API Token - -The `CF_DNS_API_TOKEN` is required for automatic SSL certificate generation via Let's Encrypt DNS-01 challenge. - -1. Go to [Cloudflare Dashboard](https://dash.cloudflare.com/profile/api-tokens) -2. Click **Create Token** -3. Use template: **Edit zone DNS** -4. Configure permissions: - - **Permissions**: Zone > DNS > Edit - - **Zone Resources**: Include > Specific zone > `motovaultpro.com` -5. Click **Continue to summary** then **Create Token** -6. Copy the token value immediately (it won't be shown again) -7. Add as `CF_DNS_API_TOKEN` File variable in GitLab - -### Setting Up a File Type Variable - -1. Go to **Settings > CI/CD > Variables** -2. Click **Add variable** -3. Enter the variable key (e.g., `POSTGRES_PASSWORD`) -4. Enter the secret value in the **Value** field -5. Set **Type** to **File** -6. Enable **Protect variable** (recommended) -7. Enable **Mask variable** (for sensitive data) -8. Click **Add variable** - ---- - -## Secrets Architecture - -MotoVaultPro uses a Kubernetes-style secrets pattern where secrets are mounted as files at `/run/secrets/` inside containers. - -### How It Works - -1. **GitLab stores secrets** as File type CI/CD variables -2. **During pipeline execution**, GitLab writes each secret to a temporary file -3. **The `inject-secrets.sh` script** copies these files to `secrets/app/` directory -4. **Docker Compose** mounts these files to `/run/secrets/` in containers -5. **Application code** reads secrets from the filesystem (not environment variables) +| Variable | Type | Protected | Purpose | +|----------|------|-----------|---------| +| `DEPLOY_NOTIFY_EMAIL` | Variable | Yes | Notification recipient | +| `VITE_AUTH0_DOMAIN` | Variable | No | Auth0 domain | +| `VITE_AUTH0_CLIENT_ID` | Variable | No | Auth0 client ID | +| `VITE_AUTH0_AUDIENCE` | Variable | No | Auth0 audience | ### Secret Files -``` -secrets/app/ - postgres-password.txt -> /run/secrets/postgres-password - auth0-client-secret.txt -> /run/secrets/auth0-client-secret - google-maps-api-key.txt -> /run/secrets/google-maps-api-key - google-maps-map-id.txt -> /run/secrets/google-maps-map-id - cloudflare-dns-token.txt -> /run/secrets/cloudflare-dns-token - resend-api-key.txt -> /run/secrets/resend-api-key -``` +These use GitLab's **File** type and are injected via `scripts/inject-secrets.sh`: -### Security Benefits +| Variable | Type | Protected | Masked | +|----------|------|-----------|--------| +| `POSTGRES_PASSWORD` | File | Yes | Yes | +| `AUTH0_CLIENT_SECRET` | File | Yes | Yes | +| `GOOGLE_MAPS_API_KEY` | File | Yes | Yes | +| `GOOGLE_MAPS_MAP_ID` | File | Yes | No | +| `CF_DNS_API_TOKEN` | File | Yes | Yes | +| `RESEND_API_KEY` | File | Yes | Yes | -- Secrets never appear as environment variables (not visible in `env` or `printenv`) -- File permissions (600) restrict access -- Masked variables prevent accidental log exposure -- Protected variables only available on protected branches +### Registry Authentication + +GitLab provides these automatically: +- `CI_REGISTRY_USER` - Registry username +- `CI_REGISTRY_PASSWORD` - Registry token +- `CI_REGISTRY` - Registry URL --- -## Pipeline Overview +## Container Registry -The CI/CD pipeline consists of four stages: +All images are hosted on the GitLab Container Registry to avoid Docker Hub rate limits. -### Stage 1: Validate - -Verifies deployment prerequisites: -- Docker is accessible -- Docker Compose is available -- Deployment directory exists - -### Stage 2: Build - -Builds Docker images: -- Pulls latest code from repository -- Builds all service images with `--no-cache` - -### Stage 3: Deploy - -Deploys the application: -1. Injects secrets from GitLab variables -2. Stops existing services gracefully -3. Pulls base images -4. Starts database services (PostgreSQL, Redis) -5. Runs database migrations -6. Starts all services - -### Stage 4: Verify - -Validates deployment health: -- Checks all containers are running -- Tests backend health endpoint -- Reports deployment status - -### Pipeline Diagram +### Registry URL ``` -[Validate] -> [Build] -> [Deploy] -> [Verify] - | | | | - Check Build Inject Health - prereqs images secrets checks - | - Migrate - | - Start - services +registry.motovaultpro.com ``` +### Image Paths + +| Image | Path | +|-------|------| +| Backend | `registry.motovaultpro.com/motovaultpro/backend:$TAG` | +| Frontend | `registry.motovaultpro.com/motovaultpro/frontend:$TAG` | +| Mirrors | `registry.motovaultpro.com/mirrors/` | + +### Base Image Mirrors + +Mirror upstream images to avoid rate limits: + +```bash +# Run manually or via scheduled pipeline +./scripts/ci/mirror-base-images.sh +``` + +Mirrored images: +- `node:20-alpine` +- `nginx:alpine` +- `postgres:18-alpine` +- `redis:8.4-alpine` +- `traefik:v3.6` +- `docker:24.0` +- `docker:24.0-dind` + --- ## Deployment Process ### Automatic Deployment -Deployments are triggered automatically when: -- Code is pushed to the `main` branch -- A merge request is merged into `main` +Deployments trigger automatically on push to `main`: + +1. **Validate**: Check prerequisites, determine target stack +2. **Build**: Build images on dedicated build server +3. **Prepare**: Start inactive stack, run health checks +4. **Switch**: Update Traefik weights (instant) +5. **Verify**: External health check +6. **Notify**: Send email notification ### Manual Deployment -To trigger a manual deployment: - 1. Go to **CI/CD > Pipelines** 2. Click **Run pipeline** -3. Select the `main` branch +3. Select `main` branch 4. Click **Run pipeline** -### Deployment Steps (What Happens) +### Deployment Timeline -1. **Secrets Injection** - - `inject-secrets.sh` copies GitLab File variables to `secrets/app/` - - Permissions are set to 600 for security - -2. **Service Shutdown** - - Existing containers are stopped gracefully (30s timeout) - - Volumes are preserved - -3. **Database Startup** - - PostgreSQL and Redis start first - - 15-second wait for database readiness - -4. **Migrations** - - Backend container runs database migrations - - Ensures schema is up-to-date - -5. **Full Service Startup** - - All services start via `docker compose up -d` - - Traefik routes traffic automatically - -6. **Health Verification** - - Container status checks - - Backend health endpoint validation +| Phase | Duration | +|-------|----------| +| Validate | ~5s | +| Build | ~2 min | +| Deploy-prepare | ~30s | +| Deploy-switch | ~3s | +| Verify | ~30s | +| **Total** | ~3 min | --- -## Rollback Procedure +## Rollback Procedures ### Automatic Rollback -If the verify stage fails, the pipeline will report failure but services remain running. Manual intervention is required. +Triggers automatically when: +- Health check fails in `deploy-prepare` +- `verify` stage fails after switch +- Container becomes unhealthy within verification period + +The pipeline runs `scripts/ci/auto-rollback.sh` which: +1. Verifies previous stack is healthy +2. Switches traffic back +3. Sends notification ### Manual Rollback -Use the rollback script: +SSH to production server: ```bash -# SSH to server -ssh user@server - -# Run rollback to previous commit cd /opt/motovaultpro -./scripts/rollback.sh HEAD~1 -# Or rollback to specific tag/commit -./scripts/rollback.sh v1.0.0 +# Check current state +cat config/deployment/state.json | jq . + +# Switch to other stack +./scripts/ci/switch-traffic.sh blue # or green ``` -### Rollback Script Details - -The script performs: -1. Stops all current services -2. Checks out the specified version -3. Rebuilds images -4. Starts services - ### Emergency Recovery -If rollback fails: +If both stacks are unhealthy: + +```bash +# Stop everything +docker compose -f docker-compose.yml -f docker-compose.blue-green.yml down + +# Restart shared services +docker compose up -d mvp-postgres mvp-redis mvp-traefik + +# Wait for database +sleep 15 + +# Start one stack +export BACKEND_IMAGE=registry.motovaultpro.com/motovaultpro/backend:latest +export FRONTEND_IMAGE=registry.motovaultpro.com/motovaultpro/frontend:latest +docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d \ + mvp-frontend-blue mvp-backend-blue + +# Switch traffic +./scripts/ci/switch-traffic.sh blue +``` + +--- + +## Maintenance Migrations + +For breaking database changes requiring downtime: + +### Via Pipeline (Recommended) + +1. Go to **CI/CD > Pipelines** +2. Find the `maintenance-migration` job +3. Click **Play** to trigger manually + +### Via Script ```bash cd /opt/motovaultpro -# Stop everything -docker compose down +# With backup +./scripts/ci/maintenance-migrate.sh backup -# Check git history -git log --oneline -10 +# Without backup +./scripts/ci/maintenance-migrate.sh +``` -# Checkout known working version -git checkout +### What Happens -# Rebuild and start -docker compose build -docker compose up -d +1. Sends maintenance notification +2. Enables maintenance mode (stops traffic) +3. Creates database backup (if requested) +4. Runs migrations +5. Restarts backends +6. Restores traffic +7. Sends completion notification -# Verify -docker compose ps +--- + +## Notifications + +Email notifications via Resend API for: + +| Event | Subject | +|-------|---------| +| `success` | Deployment Successful | +| `failure` | Deployment Failed | +| `rollback` | Auto-Rollback Executed | +| `rollback_failed` | CRITICAL: Rollback Failed | +| `maintenance_start` | Maintenance Mode Started | +| `maintenance_end` | Maintenance Complete | + +Configure recipient in GitLab CI/CD variables: +``` +DEPLOY_NOTIFY_EMAIL = admin@example.com ``` --- ## Troubleshooting -### Pipeline Fails at Validate Stage - -**Symptom**: `DEPLOY_PATH not found` - -**Solution**: -```bash -# Create directory on runner server -sudo mkdir -p /opt/motovaultpro -sudo chown gitlab-runner:gitlab-runner /opt/motovaultpro -``` - ### Pipeline Fails at Build Stage -**Symptom**: Docker build errors - -**Solutions**: -1. Check Dockerfile syntax -2. Verify network connectivity for npm/package downloads -3. Check disk space: `df -h` -4. Clear Docker cache: `docker system prune -a` - -### Pipeline Fails at Deploy Stage - -**Symptom**: Secrets injection fails - -**Solutions**: -1. Verify CI/CD variables are configured correctly -2. Check variable types are set to **File** for secrets -3. Ensure variables are not restricted to specific environments - -**Symptom**: Migration fails - -**Solutions**: -1. Check database connectivity -2. Verify PostgreSQL is healthy: `docker logs mvp-postgres` -3. Run migrations manually: - ```bash - docker compose exec mvp-backend npm run migrate - ``` - -### Pipeline Fails at Verify Stage - -**Symptom**: Container not running - -**Solutions**: -1. Check container logs: `docker logs ` -2. Verify secrets are correctly mounted -3. Check for port conflicts - -**Symptom**: Health check fails - -**Solutions**: -1. Wait longer (service might be starting) -2. Check backend logs: `docker logs mvp-backend` -3. Verify database connection - -### Services Start But Application Doesn't Work - -**Check secrets are mounted**: +**Check build server connectivity:** ```bash -docker compose exec mvp-backend ls -la /run/secrets/ +# On build server +sudo gitlab-runner verify +docker login registry.motovaultpro.com ``` -**Check configuration**: +**Check disk space:** ```bash -docker compose exec mvp-backend cat /app/config/production.yml +df -h +docker system prune -af ``` -**Check network connectivity**: +### Pipeline Fails at Deploy-Prepare + +**Container won't start:** ```bash -docker network ls -docker network inspect motovaultpro_backend +docker logs mvp-backend-blue --tail 100 +docker logs mvp-frontend-blue --tail 100 ``` -### Viewing Logs - +**Health check timeout:** ```bash -# All services -docker compose logs -f - -# Specific service -docker compose logs -f mvp-backend - -# Last 100 lines -docker compose logs --tail 100 mvp-backend +# Increase timeout in .gitlab-ci.yml +HEALTH_CHECK_TIMEOUT: "90" ``` ---- - -## Maintenance - -### Updating Secrets - -1. Update the CI/CD variable in GitLab -2. Trigger a new pipeline (push or manual) -3. The new secrets will be injected during deployment - -### Database Backups - -Backups should be configured separately. Recommended approach: +### Traffic Not Switching +**Check Traefik config:** ```bash -# Manual backup -docker compose exec mvp-postgres pg_dump -U postgres motovaultpro > backup.sql - -# Automated backup (add to cron) -0 2 * * * cd /opt/motovaultpro && docker compose exec -T mvp-postgres pg_dump -U postgres motovaultpro > /backups/mvp-$(date +\%Y\%m\%d).sql +cat config/traefik/dynamic/blue-green.yml +docker exec mvp-traefik traefik healthcheck ``` -### Monitoring +**Check routing:** +```bash +curl -I https://motovaultpro.com/api/health +``` -Consider adding: -- Prometheus metrics (Traefik already configured) -- Health check alerts -- Log aggregation +### Verify Stage Fails + +**Check external connectivity:** +```bash +curl -sf https://motovaultpro.com/api/health +``` + +**Check container health:** +```bash +docker inspect --format='{{.State.Health.Status}}' mvp-backend-blue +``` --- ## Quick Reference -### Common Commands - -```bash -# View pipeline status -# GitLab UI: CI/CD > Pipelines - -# SSH to server -ssh user@your-server - -# Navigate to project -cd /opt/motovaultpro - -# View running containers -docker compose ps - -# View logs -docker compose logs -f - -# Restart a service -docker compose restart mvp-backend - -# Run migrations manually -docker compose exec mvp-backend npm run migrate - -# Access database -docker compose exec mvp-postgres psql -U postgres motovaultpro - -# Health check -curl http://localhost:3001/health -``` - ### Important Paths | Path | Description | |------|-------------| -| `$CI_BUILDS_DIR/motovaultpro` | Application root (stable clone path) | -| `$CI_BUILDS_DIR/motovaultpro/secrets/app/` | Secrets directory | -| `$CI_BUILDS_DIR/motovaultpro/data/documents/` | Document storage | -| `$CI_BUILDS_DIR/motovaultpro/config/` | Configuration files | +| `config/deployment/state.json` | Deployment state | +| `config/traefik/dynamic/blue-green.yml` | Traffic routing | +| `scripts/ci/` | Deployment scripts | -Note: `CI_BUILDS_DIR` is typically `/opt/gitlab-runner/builds` for shell executors. +### Common Commands -### Container Names +```bash +# View current state +cat config/deployment/state.json | jq . -| Container | Purpose | -|-----------|---------| -| `mvp-traefik` | Reverse proxy, TLS termination | -| `mvp-frontend` | React SPA | -| `mvp-backend` | Node.js API | -| `mvp-postgres` | PostgreSQL database | -| `mvp-redis` | Redis cache | +# Check container status +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Health}}" + +# View logs +docker logs mvp-backend-blue -f + +# Manual traffic switch +./scripts/ci/switch-traffic.sh green + +# Run health check +./scripts/ci/health-check.sh blue + +# Send test notification +./scripts/ci/notify.sh success "Test message" +``` + +### Memory Budget (8GB Server) + +| Component | RAM | +|-----------|-----| +| Blue frontend | 512MB | +| Blue backend | 1GB | +| Green frontend | 512MB | +| Green backend | 1GB | +| PostgreSQL | 2GB | +| Redis | 512MB | +| Traefik | 128MB | +| System | 1.3GB | +| **Total** | ~7GB | diff --git a/frontend/Dockerfile b/frontend/Dockerfile index c7959d7..2dcd775 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,7 +1,11 @@ # Production Dockerfile for MotoVaultPro Frontend +# Uses mirrored base images from GitLab Container Registry -# Stage 1: Base with dependencies -FROM node:lts-alpine AS base +# Build argument for registry (defaults to GitLab mirrors, falls back to Docker Hub) +ARG REGISTRY_MIRRORS=registry.motovaultpro.com/mirrors + +# Stage 1: Base with dependencies +FROM ${REGISTRY_MIRRORS}/node:20-alpine AS base RUN apk add --no-cache dumb-init curl WORKDIR /app COPY package*.json ./ @@ -29,7 +33,7 @@ COPY . . RUN npm run build # Stage 4: Production stage with nginx -FROM nginx:alpine AS production +FROM ${REGISTRY_MIRRORS}/nginx:alpine AS production # Add curl for healthchecks RUN apk add --no-cache curl diff --git a/scripts/ci/auto-rollback.sh b/scripts/ci/auto-rollback.sh new file mode 100755 index 0000000..44e72ca --- /dev/null +++ b/scripts/ci/auto-rollback.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# Auto-rollback script for blue-green deployment +# Reverts traffic to the previous healthy stack +# +# Usage: ./auto-rollback.sh [reason] +# reason: Optional description of why rollback is happening +# +# Exit codes: +# 0 - Rollback successful +# 1 - Rollback failed + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +REASON="${1:-Automatic rollback triggered}" + +STATE_FILE="$PROJECT_ROOT/config/deployment/state.json" +SWITCH_SCRIPT="$SCRIPT_DIR/switch-traffic.sh" +HEALTH_SCRIPT="$SCRIPT_DIR/health-check.sh" +NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh" + +echo "========================================" +echo "AUTO-ROLLBACK INITIATED" +echo "Reason: $REASON" +echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" +echo "========================================" + +# Determine current and rollback stacks +if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + CURRENT_STACK=$(jq -r '.active_stack // "unknown"' "$STATE_FILE") + ROLLBACK_STACK=$(jq -r '.inactive_stack // "unknown"' "$STATE_FILE") +else + echo "ERROR: Cannot determine current stack state" + echo "State file: $STATE_FILE" + exit 1 +fi + +if [[ "$CURRENT_STACK" == "unknown" ]] || [[ "$ROLLBACK_STACK" == "unknown" ]]; then + echo "ERROR: Invalid stack state" + echo " Current: $CURRENT_STACK" + echo " Rollback target: $ROLLBACK_STACK" + exit 1 +fi + +echo "" +echo "Stack Status:" +echo " Currently active: $CURRENT_STACK" +echo " Rollback target: $ROLLBACK_STACK" +echo "" + +# Verify rollback stack is healthy before switching +echo "Step 1/3: Verifying rollback stack health..." +echo "----------------------------------------" + +if [[ -x "$HEALTH_SCRIPT" ]]; then + if ! "$HEALTH_SCRIPT" "$ROLLBACK_STACK" 30; then + echo "" + echo "CRITICAL: Rollback stack ($ROLLBACK_STACK) is NOT healthy!" + echo "Manual intervention required." + echo "" + echo "Troubleshooting steps:" + echo " 1. Check container logs: docker logs mvp-backend-$ROLLBACK_STACK" + echo " 2. Check container status: docker ps -a" + echo " 3. Consider restarting rollback stack" + echo "" + + # Send critical notification + if [[ -x "$NOTIFY_SCRIPT" ]]; then + "$NOTIFY_SCRIPT" "rollback_failed" \ + "Rollback to $ROLLBACK_STACK failed - stack unhealthy. Manual intervention required. Reason: $REASON" \ + || true + fi + + exit 1 + fi + echo " OK: Rollback stack is healthy" +else + echo " WARNING: Health check script not found, proceeding anyway" +fi + +# Switch traffic to rollback stack +echo "" +echo "Step 2/3: Switching traffic to $ROLLBACK_STACK..." +echo "----------------------------------------" + +if [[ -x "$SWITCH_SCRIPT" ]]; then + if ! "$SWITCH_SCRIPT" "$ROLLBACK_STACK" instant; then + echo "ERROR: Traffic switch failed" + + if [[ -x "$NOTIFY_SCRIPT" ]]; then + "$NOTIFY_SCRIPT" "rollback_failed" \ + "Rollback traffic switch failed. Manual intervention required. Reason: $REASON" \ + || true + fi + + exit 1 + fi +else + echo "ERROR: Traffic switch script not found: $SWITCH_SCRIPT" + exit 1 +fi + +# Update state file with rollback info +echo "" +echo "Step 3/3: Updating deployment state..." +echo "----------------------------------------" + +if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + jq --arg stack "$ROLLBACK_STACK" \ + --arg reason "$REASON" \ + --arg ts "$TIMESTAMP" \ + --arg failed "$CURRENT_STACK" \ + '.active_stack = $stack | + .inactive_stack = $failed | + .last_rollback = $ts | + .last_rollback_reason = $reason | + .rollback_available = false' \ + "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" + + echo " State updated" +fi + +# Send notification +if [[ -x "$NOTIFY_SCRIPT" ]]; then + echo "" + echo "Sending rollback notification..." + "$NOTIFY_SCRIPT" "rollback" \ + "Rollback executed. Traffic switched from $CURRENT_STACK to $ROLLBACK_STACK. Reason: $REASON" \ + || echo " WARNING: Notification failed" +fi + +echo "" +echo "========================================" +echo "ROLLBACK COMPLETE" +echo "========================================" +echo "" +echo "Summary:" +echo " Previous stack: $CURRENT_STACK (now inactive)" +echo " Current stack: $ROLLBACK_STACK (now active)" +echo " Reason: $REASON" +echo "" +echo "Next steps:" +echo " 1. Investigate why $CURRENT_STACK failed" +echo " 2. Check logs: docker logs mvp-backend-$CURRENT_STACK" +echo " 3. Fix issues before next deployment" +echo "" + +exit 0 diff --git a/scripts/ci/health-check.sh b/scripts/ci/health-check.sh new file mode 100755 index 0000000..45d2ee1 --- /dev/null +++ b/scripts/ci/health-check.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# Health check script for blue-green deployment +# Verifies container health and HTTP endpoints +# +# Usage: ./health-check.sh [timeout_seconds] +# stack: blue or green +# timeout_seconds: max wait time (default: 60) +# +# Exit codes: +# 0 - All health checks passed +# 1 - Health check failed + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +STACK="${1:-}" +TIMEOUT="${2:-60}" + +if [[ -z "$STACK" ]] || [[ ! "$STACK" =~ ^(blue|green)$ ]]; then + echo "Usage: $0 [timeout_seconds]" + exit 1 +fi + +FRONTEND_CONTAINER="mvp-frontend-$STACK" +BACKEND_CONTAINER="mvp-backend-$STACK" + +echo "========================================" +echo "Health Check - $STACK Stack" +echo "Timeout: ${TIMEOUT}s" +echo "========================================" + +# Function to check Docker container health +check_container_health() { + local container="$1" + local status + + status=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "not found") + + case "$status" in + "healthy") + return 0 + ;; + "starting") + return 2 # Still starting + ;; + "unhealthy"|"not found"|"") + return 1 + ;; + *) + return 1 + ;; + esac +} + +# Function to check HTTP endpoint +check_http_endpoint() { + local container="$1" + local port="$2" + local path="$3" + + if docker exec "$container" curl -sf "http://localhost:${port}${path}" > /dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# Function to check database connectivity via backend +check_database_connectivity() { + local container="$1" + + # The /health endpoint should verify database connectivity + if docker exec "$container" curl -sf "http://localhost:3001/health" 2>/dev/null | grep -q '"database"'; then + return 0 + else + return 1 + fi +} + +# Wait for containers to be healthy +wait_for_health() { + local container="$1" + local elapsed=0 + + while [[ $elapsed -lt $TIMEOUT ]]; do + check_container_health "$container" + local status=$? + + if [[ $status -eq 0 ]]; then + return 0 + elif [[ $status -eq 1 ]]; then + echo " ERROR: Container $container is unhealthy" + docker logs "$container" --tail 20 2>/dev/null || true + return 1 + fi + + # Still starting, wait + sleep 2 + elapsed=$((elapsed + 2)) + echo " Waiting for $container... (${elapsed}s/${TIMEOUT}s)" + done + + echo " ERROR: Timeout waiting for $container" + return 1 +} + +# Main health check sequence +echo "" +echo "Step 1/4: Checking container status..." +echo "----------------------------------------" + +for container in "$FRONTEND_CONTAINER" "$BACKEND_CONTAINER"; do + running=$(docker inspect --format='{{.State.Running}}' "$container" 2>/dev/null || echo "false") + if [[ "$running" != "true" ]]; then + echo " ERROR: Container $container is not running" + docker ps -a --filter "name=$container" --format "table {{.Names}}\t{{.Status}}" + exit 1 + fi + echo " OK: $container is running" +done + +echo "" +echo "Step 2/4: Waiting for Docker health checks..." +echo "----------------------------------------" + +for container in "$FRONTEND_CONTAINER" "$BACKEND_CONTAINER"; do + echo " Checking $container..." + if ! wait_for_health "$container"; then + echo " FAILED: $container health check" + exit 1 + fi + echo " OK: $container is healthy" +done + +echo "" +echo "Step 3/4: Verifying HTTP endpoints..." +echo "----------------------------------------" + +# Check frontend +echo " Checking frontend HTTP..." +if ! check_http_endpoint "$FRONTEND_CONTAINER" 3000 "/"; then + echo " FAILED: Frontend HTTP check" + exit 1 +fi +echo " OK: Frontend responds on port 3000" + +# Check backend health endpoint +echo " Checking backend HTTP..." +if ! check_http_endpoint "$BACKEND_CONTAINER" 3001 "/health"; then + echo " FAILED: Backend HTTP check" + exit 1 +fi +echo " OK: Backend responds on port 3001" + +echo "" +echo "Step 4/4: Verifying database connectivity..." +echo "----------------------------------------" + +if ! check_database_connectivity "$BACKEND_CONTAINER"; then + echo " WARNING: Could not verify database connectivity" + echo " (Backend may not expose database status in /health)" +else + echo " OK: Database connectivity verified" +fi + +echo "" +echo "========================================" +echo "Health Check PASSED - $STACK Stack" +echo "========================================" + +# Update state file +STATE_FILE="$PROJECT_ROOT/config/deployment/state.json" +if [[ -f "$STATE_FILE" ]]; then + # Update stack health status using jq if available + if command -v jq &> /dev/null; then + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + jq --arg stack "$STACK" --arg ts "$TIMESTAMP" \ + '.[$stack].healthy = true | .[$stack].last_health_check = $ts' \ + "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" + fi +fi + +exit 0 diff --git a/scripts/ci/maintenance-migrate.sh b/scripts/ci/maintenance-migrate.sh new file mode 100755 index 0000000..a80302d --- /dev/null +++ b/scripts/ci/maintenance-migrate.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# Maintenance mode migration script +# Enables maintenance mode, runs migrations, then restores service +# +# Usage: ./maintenance-migrate.sh [backup] +# backup: If set, creates a database backup before migration +# +# This script is for BREAKING migrations that require downtime. +# Non-breaking migrations run automatically on container start. +# +# Exit codes: +# 0 - Migration successful +# 1 - Migration failed (maintenance mode will remain active) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +CREATE_BACKUP="${1:-}" + +COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yml" +COMPOSE_BLUE_GREEN="$PROJECT_ROOT/docker-compose.blue-green.yml" +STATE_FILE="$PROJECT_ROOT/config/deployment/state.json" +NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh" +TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml" +BACKUP_DIR="$PROJECT_ROOT/data/backups" + +echo "========================================" +echo "MAINTENANCE MODE MIGRATION" +echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" +echo "========================================" + +# Determine active stack +if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE") +else + ACTIVE_STACK="blue" +fi + +BACKEND_CONTAINER="mvp-backend-$ACTIVE_STACK" + +echo "" +echo "Configuration:" +echo " Active stack: $ACTIVE_STACK" +echo " Backend container: $BACKEND_CONTAINER" +echo " Create backup: ${CREATE_BACKUP:-no}" +echo "" + +# Step 1: Send maintenance notification +echo "Step 1/6: Sending maintenance notification..." +echo "----------------------------------------" + +if [[ -x "$NOTIFY_SCRIPT" ]]; then + "$NOTIFY_SCRIPT" "maintenance_start" "Starting maintenance window for database migration" || true +fi +echo " OK" + +# Step 2: Enable maintenance mode +echo "" +echo "Step 2/6: Enabling maintenance mode..." +echo "----------------------------------------" + +# Update state file +if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + jq '.maintenance_mode = true' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" +fi + +# Stop traffic to both stacks (weight 0) +echo " Stopping traffic to application stacks..." + +# Set both stacks to weight 0 - Traefik will return 503 +sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" +sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" +sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" +sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG" +rm -f "${TRAEFIK_CONFIG}.bak" + +# Wait for Traefik to pick up changes +sleep 3 +echo " OK: Maintenance mode active" + +# Step 3: Create database backup (if requested) +if [[ -n "$CREATE_BACKUP" ]]; then + echo "" + echo "Step 3/6: Creating database backup..." + echo "----------------------------------------" + + mkdir -p "$BACKUP_DIR" + BACKUP_FILE="$BACKUP_DIR/pre-migration-$(date +%Y%m%d-%H%M%S).sql" + + if docker exec mvp-postgres pg_dump -U postgres motovaultpro > "$BACKUP_FILE"; then + echo " OK: Backup created: $BACKUP_FILE" + ls -lh "$BACKUP_FILE" + else + echo " ERROR: Backup failed" + echo "" + echo "Aborting migration. Restoring traffic..." + + # Restore traffic + if [[ "$ACTIVE_STACK" == "blue" ]]; then + sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" + sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" + else + sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" + sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" + fi + + if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + jq '.maintenance_mode = false' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" + fi + + exit 1 + fi +else + echo "" + echo "Step 3/6: Skipping backup (not requested)..." + echo "----------------------------------------" + echo " OK" +fi + +# Step 4: Run migrations +echo "" +echo "Step 4/6: Running database migrations..." +echo "----------------------------------------" + +# Flush Redis cache before migration +echo " Flushing Redis cache..." +docker exec mvp-redis redis-cli FLUSHALL > /dev/null 2>&1 || true + +# Run migrations +echo " Running migrations..." +if docker exec "$BACKEND_CONTAINER" npm run migrate; then + echo " OK: Migrations completed" +else + echo " ERROR: Migration failed" + echo "" + echo "Migration failed. Maintenance mode remains active." + echo "To restore:" + echo " 1. Fix migration issues" + echo " 2. Re-run migrations: docker exec $BACKEND_CONTAINER npm run migrate" + echo " 3. Run: $0 to retry, or manually restore traffic" + + if [[ -x "$NOTIFY_SCRIPT" ]]; then + "$NOTIFY_SCRIPT" "failure" "Database migration failed during maintenance window. Manual intervention required." || true + fi + + exit 1 +fi + +# Step 5: Restart backend containers +echo "" +echo "Step 5/6: Restarting backend containers..." +echo "----------------------------------------" + +# Restart both backends to pick up any schema changes +docker restart mvp-backend-blue mvp-backend-green 2>/dev/null || true + +# Wait for backends to be healthy +echo " Waiting for backends to be healthy..." +sleep 10 + +for container in mvp-backend-blue mvp-backend-green; do + for i in {1..12}; do + health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown") + if [[ "$health" == "healthy" ]]; then + echo " OK: $container is healthy" + break + fi + if [[ $i -eq 12 ]]; then + echo " WARNING: $container health check timeout" + fi + sleep 5 + done +done + +# Step 6: Disable maintenance mode +echo "" +echo "Step 6/6: Restoring traffic..." +echo "----------------------------------------" + +# Restore traffic to active stack +if [[ "$ACTIVE_STACK" == "blue" ]]; then + sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" + sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" +else + sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" + sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG" +fi + +# Update state file +if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + jq --arg ts "$TIMESTAMP" '.maintenance_mode = false | .last_migration = $ts' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" +fi + +# Wait for Traefik to pick up changes +sleep 3 +echo " OK: Traffic restored" + +# Send completion notification +if [[ -x "$NOTIFY_SCRIPT" ]]; then + "$NOTIFY_SCRIPT" "maintenance_end" "Maintenance window complete. Database migration successful." || true +fi + +echo "" +echo "========================================" +echo "MAINTENANCE MIGRATION COMPLETE" +echo "========================================" +echo "" +echo "Summary:" +echo " - Migrations: Successful" +echo " - Active stack: $ACTIVE_STACK" +echo " - Maintenance mode: Disabled" +if [[ -n "$CREATE_BACKUP" ]]; then + echo " - Backup: $BACKUP_FILE" +fi +echo "" + +exit 0 diff --git a/scripts/ci/mirror-base-images.sh b/scripts/ci/mirror-base-images.sh new file mode 100755 index 0000000..0a8c2ca --- /dev/null +++ b/scripts/ci/mirror-base-images.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Mirror upstream Docker images to GitLab Container Registry +# Run manually or via scheduled GitLab pipeline +# This avoids Docker Hub rate limits and ensures build reliability + +set -euo pipefail + +REGISTRY="${REGISTRY:-registry.motovaultpro.com/mirrors}" + +# Base images required by MotoVaultPro +IMAGES=( + "node:20-alpine" + "nginx:alpine" + "postgres:18-alpine" + "redis:8.4-alpine" + "traefik:v3.6" + "docker:24.0" + "docker:24.0-dind" +) + +echo "========================================" +echo "Base Image Mirroring Script" +echo "Registry: $REGISTRY" +echo "========================================" + +# Check if logged into registry +if ! docker info 2>/dev/null | grep -q "Username"; then + echo "WARNING: Not logged into Docker registry" + echo "Run: docker login registry.motovaultpro.com" +fi + +FAILED=() +SUCCESS=() + +for img in "${IMAGES[@]}"; do + echo "" + echo "Processing: $img" + echo "----------------------------------------" + + # Pull from upstream + echo " Pulling from upstream..." + if ! docker pull "$img"; then + echo " ERROR: Failed to pull $img" + FAILED+=("$img") + continue + fi + + # Tag for local registry + local_tag="$REGISTRY/$img" + echo " Tagging as: $local_tag" + docker tag "$img" "$local_tag" + + # Push to local registry + echo " Pushing to registry..." + if ! docker push "$local_tag"; then + echo " ERROR: Failed to push $local_tag" + FAILED+=("$img") + continue + fi + + SUCCESS+=("$img") + echo " OK: $img mirrored successfully" +done + +echo "" +echo "========================================" +echo "Summary" +echo "========================================" +echo "Successful: ${#SUCCESS[@]}" +for img in "${SUCCESS[@]}"; do + echo " - $img" +done + +if [ ${#FAILED[@]} -gt 0 ]; then + echo "" + echo "Failed: ${#FAILED[@]}" + for img in "${FAILED[@]}"; do + echo " - $img" + done + exit 1 +fi + +echo "" +echo "All images mirrored successfully" diff --git a/scripts/ci/notify.sh b/scripts/ci/notify.sh new file mode 100755 index 0000000..54d49f4 --- /dev/null +++ b/scripts/ci/notify.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Deployment notification script using Resend API +# Sends email notifications for deployment events +# +# Usage: ./notify.sh [message] [commit_sha] +# event_type: success, failure, rollback, rollback_failed, maintenance_start, maintenance_end +# message: Optional custom message +# commit_sha: Optional commit SHA for context +# +# Required environment variables: +# DEPLOY_NOTIFY_EMAIL - Recipient email address +# +# Reads Resend API key from: +# /run/secrets/resend-api-key (container) +# ./secrets/app/resend-api-key.txt (local) +# +# Exit codes: +# 0 - Notification sent (or skipped if not configured) +# 1 - Notification failed + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +EVENT_TYPE="${1:-}" +MESSAGE="${2:-}" +COMMIT_SHA="${3:-${CI_COMMIT_SHORT_SHA:-unknown}}" + +if [[ -z "$EVENT_TYPE" ]]; then + echo "Usage: $0 [message] [commit_sha]" + exit 1 +fi + +# Get recipient email +NOTIFY_EMAIL="${DEPLOY_NOTIFY_EMAIL:-}" +if [[ -z "$NOTIFY_EMAIL" ]]; then + echo "DEPLOY_NOTIFY_EMAIL not set, skipping notification" + exit 0 +fi + +# Get Resend API key +RESEND_API_KEY="" +if [[ -f "/run/secrets/resend-api-key" ]]; then + RESEND_API_KEY=$(cat /run/secrets/resend-api-key) +elif [[ -f "$PROJECT_ROOT/secrets/app/resend-api-key.txt" ]]; then + RESEND_API_KEY=$(cat "$PROJECT_ROOT/secrets/app/resend-api-key.txt") +fi + +if [[ -z "$RESEND_API_KEY" ]]; then + echo "WARNING: Resend API key not found, skipping notification" + exit 0 +fi + +# Determine subject and styling based on event type +case "$EVENT_TYPE" in + "success") + SUBJECT="Deployment Successful - MotoVaultPro" + STATUS_COLOR="#22c55e" + STATUS_EMOJI="[OK]" + STATUS_TEXT="Deployment Successful" + DEFAULT_MESSAGE="Version $COMMIT_SHA is now live." + ;; + "failure") + SUBJECT="Deployment Failed - MotoVaultPro" + STATUS_COLOR="#ef4444" + STATUS_EMOJI="[FAILED]" + STATUS_TEXT="Deployment Failed" + DEFAULT_MESSAGE="Deployment of $COMMIT_SHA failed. Check pipeline logs." + ;; + "rollback") + SUBJECT="Auto-Rollback Executed - MotoVaultPro" + STATUS_COLOR="#f59e0b" + STATUS_EMOJI="[ROLLBACK]" + STATUS_TEXT="Rollback Executed" + DEFAULT_MESSAGE="Automatic rollback was triggered. Previous version restored." + ;; + "rollback_failed") + SUBJECT="CRITICAL: Rollback Failed - MotoVaultPro" + STATUS_COLOR="#dc2626" + STATUS_EMOJI="[CRITICAL]" + STATUS_TEXT="Rollback Failed" + DEFAULT_MESSAGE="Rollback attempt failed. Manual intervention required immediately." + ;; + "maintenance_start") + SUBJECT="Maintenance Mode Started - MotoVaultPro" + STATUS_COLOR="#6366f1" + STATUS_EMOJI="[MAINTENANCE]" + STATUS_TEXT="Maintenance Mode Active" + DEFAULT_MESSAGE="Application is in maintenance mode for database migration." + ;; + "maintenance_end") + SUBJECT="Maintenance Mode Ended - MotoVaultPro" + STATUS_COLOR="#22c55e" + STATUS_EMOJI="[ONLINE]" + STATUS_TEXT="Maintenance Complete" + DEFAULT_MESSAGE="Maintenance window complete. Application is online." + ;; + *) + echo "Unknown event type: $EVENT_TYPE" + exit 1 + ;; +esac + +# Use custom message or default +FINAL_MESSAGE="${MESSAGE:-$DEFAULT_MESSAGE}" +TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + +# Build HTML email +HTML_BODY=$(cat < + + + + + +
+
+

MotoVaultPro

+
+
+
${STATUS_EMOJI} ${STATUS_TEXT}
+
${FINAL_MESSAGE}
+
+
+ Environment: + Production +
+
+ Commit: + ${COMMIT_SHA} +
+
+ Time: + ${TIMESTAMP} +
+
+
+
+ MotoVaultPro CI/CD Notification System +
+
+ + +EOF +) + +# Send email via Resend API +echo "Sending notification: $EVENT_TYPE" +echo " To: $NOTIFY_EMAIL" +echo " Subject: $SUBJECT" + +# Build JSON payload +JSON_PAYLOAD=$(cat <", + "to": ["$NOTIFY_EMAIL"], + "subject": "$SUBJECT", + "html": $(echo "$HTML_BODY" | jq -Rs .) +} +EOF +) + +# Send via Resend API +RESPONSE=$(curl -s -w "\n%{http_code}" \ + -X POST \ + -H "Authorization: Bearer $RESEND_API_KEY" \ + -H "Content-Type: application/json" \ + -d "$JSON_PAYLOAD" \ + "https://api.resend.com/emails") + +HTTP_CODE=$(echo "$RESPONSE" | tail -n1) +BODY=$(echo "$RESPONSE" | sed '$d') + +if [[ "$HTTP_CODE" == "200" ]] || [[ "$HTTP_CODE" == "201" ]]; then + echo " OK: Notification sent successfully" + exit 0 +else + echo " ERROR: Failed to send notification (HTTP $HTTP_CODE)" + echo " Response: $BODY" + exit 1 +fi diff --git a/scripts/ci/switch-traffic.sh b/scripts/ci/switch-traffic.sh new file mode 100755 index 0000000..2c9546f --- /dev/null +++ b/scripts/ci/switch-traffic.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Traffic switching script for blue-green deployment +# Updates Traefik weighted routing configuration +# +# Usage: ./switch-traffic.sh [mode] +# target_stack: blue or green (the stack to switch TO) +# mode: instant (default) or gradual +# +# Gradual mode: 25% -> 50% -> 75% -> 100% with 3s intervals +# +# Exit codes: +# 0 - Traffic switch successful +# 1 - Switch failed + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +TARGET_STACK="${1:-}" +MODE="${2:-instant}" + +if [[ -z "$TARGET_STACK" ]] || [[ ! "$TARGET_STACK" =~ ^(blue|green)$ ]]; then + echo "Usage: $0 [instant|gradual]" + exit 1 +fi + +TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml" +STATE_FILE="$PROJECT_ROOT/config/deployment/state.json" + +if [[ ! -f "$TRAEFIK_CONFIG" ]]; then + echo "ERROR: Traefik config not found: $TRAEFIK_CONFIG" + exit 1 +fi + +echo "========================================" +echo "Traffic Switch" +echo "Target Stack: $TARGET_STACK" +echo "Mode: $MODE" +echo "========================================" + +# Determine source and target weights +if [[ "$TARGET_STACK" == "blue" ]]; then + SOURCE_STACK="green" +else + SOURCE_STACK="blue" +fi + +# Function to update weights in Traefik config +update_weights() { + local blue_weight="$1" + local green_weight="$2" + + echo " Setting weights: blue=$blue_weight, green=$green_weight" + + # Use sed to update weights in the YAML file + # Frontend blue weight + sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG" + # Frontend green weight + sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG" + # Backend blue weight + sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG" + # Backend green weight + sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG" + + # Clean up backup files + rm -f "${TRAEFIK_CONFIG}.bak" + + # Traefik watches the file and reloads automatically + # Give it a moment to pick up changes + sleep 1 +} + +# Verify Traefik has picked up the changes +verify_traefik_reload() { + # Give Traefik time to reload config + sleep 2 + + # Check if Traefik is still healthy + if docker exec mvp-traefik traefik healthcheck > /dev/null 2>&1; then + echo " OK: Traefik config reloaded" + return 0 + else + echo " WARNING: Could not verify Traefik health" + return 0 # Don't fail on this, file watcher is reliable + fi +} + +if [[ "$MODE" == "gradual" ]]; then + echo "" + echo "Gradual traffic switch starting..." + echo "----------------------------------------" + + if [[ "$TARGET_STACK" == "blue" ]]; then + STEPS=("25 75" "50 50" "75 25" "100 0") + else + STEPS=("75 25" "50 50" "25 75" "0 100") + fi + + step=1 + for weights in "${STEPS[@]}"; do + blue_w=$(echo "$weights" | cut -d' ' -f1) + green_w=$(echo "$weights" | cut -d' ' -f2) + + echo "" + echo "Step $step/4: blue=$blue_w%, green=$green_w%" + update_weights "$blue_w" "$green_w" + verify_traefik_reload + + if [[ $step -lt 4 ]]; then + echo " Waiting 3 seconds before next step..." + sleep 3 + fi + + step=$((step + 1)) + done + +else + # Instant switch + echo "" + echo "Instant traffic switch..." + echo "----------------------------------------" + + if [[ "$TARGET_STACK" == "blue" ]]; then + update_weights 100 0 + else + update_weights 0 100 + fi + + verify_traefik_reload +fi + +# Update state file +echo "" +echo "Updating deployment state..." +echo "----------------------------------------" + +if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + jq --arg active "$TARGET_STACK" \ + --arg inactive "$SOURCE_STACK" \ + --arg ts "$TIMESTAMP" \ + '.active_stack = $active | .inactive_stack = $inactive | .last_switch = $ts' \ + "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" + + echo " Active stack: $TARGET_STACK" + echo " Inactive stack: $SOURCE_STACK" +fi + +echo "" +echo "========================================" +echo "Traffic Switch COMPLETE" +echo "All traffic now routed to: $TARGET_STACK" +echo "========================================" + +exit 0