fix: I dunno, I'm making git server changes

2025-12-29 08:44:49 -06:00
parent 57d2c43da7
commit 9b0de6a5b8
18 changed files with 2584 additions and 512 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,32 +1,50 @@
-# MotoVaultPro GitLab CI/CD Pipeline
-# GitLab 18.6+ with shell executor
+# MotoVaultPro GitLab CI/CD Pipeline - Blue-Green Deployment
+# GitLab 18.6+ with separate build and production runners
 # See docs/CICD-DEPLOY.md for complete documentation
-# v1.6
+# v2.0 - Blue-Green with Auto-Rollback

 stages:
  - validate
  - build
-  - deploy
+  - deploy-prepare
+  - deploy-switch
  - verify
+  - rollback
+  - notify

 variables:
-  # Use stable clone path instead of runner-specific path
-  GIT_CLONE_PATH: $CI_BUILDS_DIR/motovaultpro
-  DEPLOY_PATH: $CI_BUILDS_DIR/motovaultpro
-  DOCKER_COMPOSE_FILE: docker-compose.yml
-  DOCKER_COMPOSE_PROD_FILE: docker-compose.prod.yml
+  # Registry configuration
+  REGISTRY: registry.motovaultpro.com
+  REGISTRY_MIRRORS: ${REGISTRY}/mirrors
+  IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}
+  BACKEND_IMAGE: ${REGISTRY}/motovaultpro/backend:${IMAGE_TAG}
+  FRONTEND_IMAGE: ${REGISTRY}/motovaultpro/frontend:${IMAGE_TAG}

-# Fix permissions after every job - docker creates files as root
+  # Deployment configuration
+  GIT_CLONE_PATH: ${CI_BUILDS_DIR}/motovaultpro
+  DEPLOY_PATH: ${CI_BUILDS_DIR}/motovaultpro
+  COMPOSE_FILE: docker-compose.yml
+  COMPOSE_BLUE_GREEN: docker-compose.blue-green.yml
+
+  # Health check configuration
+  HEALTH_CHECK_TIMEOUT: "60"
+
+# Default after_script to fix permissions
 default:
  after_script:
    - echo "Fixing file permissions..."
    - sudo chown -R gitlab-runner:gitlab-runner "$DEPLOY_PATH" 2>/dev/null || true
-    # Keep data directories owned by container user
    - sudo chown -R 1001:1001 "$DEPLOY_PATH/data/backups" "$DEPLOY_PATH/data/documents" 2>/dev/null || true

-# Validate Stage - Check prerequisites
+# ============================================
+# Stage 1: VALIDATE
+# Check prerequisites before starting pipeline
+# ============================================
 validate:
  stage: validate
+  tags:
+    - production
+    - shell
  only:
    - main
  script:
@@ -34,129 +52,385 @@ validate:
    - echo "Validating deployment prerequisites..."
    - echo "=========================================="
    - echo "Checking Docker..."
-    - 'docker info > /dev/null 2>&1 || (echo "ERROR: Docker not accessible" && exit 1)'
-    - echo "OK Docker is accessible"
+    - docker info > /dev/null 2>&1 || (echo "ERROR - Docker not accessible" && exit 1)
+    - echo "OK - Docker is accessible"
    - echo "Checking Docker Compose..."
-    - 'docker compose version > /dev/null 2>&1 || (echo "ERROR: Docker Compose not available" && exit 1)'
-    - echo "OK Docker Compose is available"
+    - docker compose version > /dev/null 2>&1 || (echo "ERROR - Docker Compose not available" && exit 1)
+    - echo "OK - Docker Compose is available"
    - echo "Checking deployment path..."
-    - 'test -d "$DEPLOY_PATH" || (echo "ERROR: DEPLOY_PATH not found" && exit 1)'
-    - echo "OK Deployment path exists"
+    - test -d "$DEPLOY_PATH" || (echo "ERROR - DEPLOY_PATH not found" && exit 1)
+    - echo "OK - Deployment path exists"
+    - echo "Checking registry access..."
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" || true
+    - echo "OK - Registry authentication configured"
+    - echo "Determining target stack..."
+    - |
+      STATE_FILE="$DEPLOY_PATH/config/deployment/state.json"
+      if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
+        ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
+        if [ "$ACTIVE_STACK" = "blue" ]; then
+          echo "TARGET_STACK=green" >> deploy.env
+        else
+          echo "TARGET_STACK=blue" >> deploy.env
+        fi
+      else
+        echo "TARGET_STACK=green" >> deploy.env
+      fi
+      cat deploy.env
    - echo "=========================================="
    - echo "Validation complete"
    - echo "=========================================="
+  artifacts:
+    reports:
+      dotenv: deploy.env

-# Build Stage - Build Docker images
+# ============================================
+# Stage 2: BUILD
+# Build and push images to GitLab Container Registry
+# Runs on dedicated build server (shell executor)
+# ============================================
 build:
  stage: build
+  tags:
+    - build
  only:
    - main
  script:
+    - echo "Authenticating with GitLab Container Registry..."
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
    - echo "=========================================="
    - echo "Building Docker images..."
+    - echo "Commit - ${CI_COMMIT_SHORT_SHA}"
+    - echo "Backend - ${BACKEND_IMAGE}"
+    - echo "Frontend - ${FRONTEND_IMAGE}"
    - echo "=========================================="
-    - cd "$DEPLOY_PATH"
-    - echo "Building images..."
-    - docker compose -f $DOCKER_COMPOSE_FILE build --no-cache
+
+    # Build backend
+    - echo "Building backend..."
+    - |
+      docker build \
+        --build-arg BUILDKIT_INLINE_CACHE=1 \
+        --cache-from ${REGISTRY}/motovaultpro/backend:latest \
+        -t ${BACKEND_IMAGE} \
+        -t ${REGISTRY}/motovaultpro/backend:latest \
+        -f backend/Dockerfile \
+        .
+
+    # Build frontend
+    - echo "Building frontend..."
+    - |
+      docker build \
+        --build-arg BUILDKIT_INLINE_CACHE=1 \
+        --build-arg VITE_AUTH0_DOMAIN=${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com} \
+        --build-arg VITE_AUTH0_CLIENT_ID=${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3} \
+        --build-arg VITE_AUTH0_AUDIENCE=${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com} \
+        --build-arg VITE_API_BASE_URL=/api \
+        --cache-from ${REGISTRY}/motovaultpro/frontend:latest \
+        -t ${FRONTEND_IMAGE} \
+        -t ${REGISTRY}/motovaultpro/frontend:latest \
+        -f frontend/Dockerfile \
+        frontend
+
+    # Push images
+    - echo "Pushing images to registry..."
+    - docker push ${BACKEND_IMAGE}
+    - docker push ${FRONTEND_IMAGE}
+    - docker push ${REGISTRY}/motovaultpro/backend:latest
+    - docker push ${REGISTRY}/motovaultpro/frontend:latest
+
    - echo "=========================================="
    - echo "Build complete"
    - echo "=========================================="

-# Deploy Stage - Inject secrets and deploy services
-deploy:
-  stage: deploy
+# ============================================
+# Stage 3: DEPLOY-PREPARE
+# Pull images, start inactive stack, run health checks
+# ============================================
+deploy-prepare:
+  stage: deploy-prepare
+  tags:
+    - production
+    - shell
  only:
    - main
+  needs:
+    - job: validate
+      artifacts: true
+    - job: build
  environment:
    name: production
    url: https://motovaultpro.com
  script:
    - echo "=========================================="
-    - echo "Deploying MotoVaultPro..."
+    - echo "Preparing deployment to ${TARGET_STACK} stack..."
    - echo "=========================================="
    - cd "$DEPLOY_PATH"
-    - echo "Step 1/8 Initializing data directories..."
+
+    # Authenticate with registry
+    - echo "Authenticating with registry..."
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
+
+    # Inject secrets
+    - echo "Step 1/5 - Injecting secrets..."
+    - chmod +x scripts/inject-secrets.sh
+    - ./scripts/inject-secrets.sh
+
+    # Initialize data directories
+    - echo "Step 2/5 - Initializing data directories..."
    - sudo mkdir -p data/backups data/documents
    - sudo chown -R 1001:1001 data/backups data/documents
    - sudo chmod 755 data/backups data/documents
-    - echo "Step 2/8 Injecting secrets..."
-    - chmod +x scripts/inject-secrets.sh
-    - ./scripts/inject-secrets.sh
-    - echo "Step 3/8 Stopping existing services..."
-    - docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE down --timeout 30 || true
-    - echo "Step 4/8 Pulling base images..."
-    - docker compose -f $DOCKER_COMPOSE_FILE pull
-    - echo "Step 5/8 Starting database services..."
-    - docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d mvp-postgres mvp-redis
-    - echo "Waiting for database to be ready..."
-    - sleep 15
-    - echo "Step 6/8 Running database migrations..."
-    - docker compose -f $DOCKER_COMPOSE_FILE run --rm mvp-backend npm run migrate || echo "Migration skipped"
-    - echo "Step 7/8 Vehicle catalog data..."
-    # Schema and data now loaded via standard migration system
-    # Migration runner handles table creation and data loading automatically
-    - echo "Vehicle catalog loaded via platform feature migration"
-    - echo "Flushing Redis cache..."
-    - docker exec mvp-redis redis-cli FLUSHALL
-    - echo "Step 8/8 Starting all services..."
-    - docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d
-    - echo "Waiting for services to initialize..."
-    - sleep 30
+
+    # Pull new images
+    - echo "Step 3/5 - Pulling images..."
+    - docker pull ${BACKEND_IMAGE}
+    - docker pull ${FRONTEND_IMAGE}
+
+    # Start inactive stack
+    - echo "Step 4/5 - Starting ${TARGET_STACK} stack..."
+    - |
+      export BACKEND_IMAGE=${BACKEND_IMAGE}
+      export FRONTEND_IMAGE=${FRONTEND_IMAGE}
+      docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \
+        mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}
+
+    # Wait for stack to be ready
+    - echo "Step 5/5 - Waiting for stack health..."
+    - sleep 10
+
+    # Run health check
+    - echo "Running health check on ${TARGET_STACK} stack..."
+    - chmod +x scripts/ci/health-check.sh
+    - ./scripts/ci/health-check.sh ${TARGET_STACK} ${HEALTH_CHECK_TIMEOUT}
+
+    # Update state with deployment info
+    - |
+      STATE_FILE="config/deployment/state.json"
+      if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
+        TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+        jq --arg stack "$TARGET_STACK" \
+           --arg commit "$CI_COMMIT_SHORT_SHA" \
+           --arg ts "$TIMESTAMP" \
+           '.[$stack].version = $commit | .[$stack].commit = $commit | .[$stack].deployed_at = $ts | .[$stack].healthy = true' \
+           "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
+      fi
+
    - echo "=========================================="
-    - echo "Deployment complete"
+    - echo "Deploy preparation complete"
    - echo "=========================================="

-# Verify Stage - Health checks
-verify:
-  stage: verify
+# ============================================
+# Stage 4: DEPLOY-SWITCH
+# Switch traffic to new stack
+# ============================================
+deploy-switch:
+  stage: deploy-switch
+  tags:
+    - production
+    - shell
  only:
    - main
+  needs:
+    - job: validate
+      artifacts: true
+    - job: deploy-prepare
  script:
    - echo "=========================================="
-    - echo "Verifying deployment..."
+    - echo "Switching traffic to ${TARGET_STACK} stack..."
    - echo "=========================================="
    - cd "$DEPLOY_PATH"
-    - echo "Checking container status..."
+
+    # Switch traffic
+    - chmod +x scripts/ci/switch-traffic.sh
+    - ./scripts/ci/switch-traffic.sh ${TARGET_STACK} instant
+
+    # Update state
    - |
-      FAILED=0
-      for service in mvp-traefik mvp-frontend mvp-backend mvp-postgres mvp-redis; do
-        status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
-        if [ "$status" != "running" ]; then
-          echo "ERROR: $service is not running (status: $status)"
-          docker logs $service --tail 50 2>/dev/null || true
-          FAILED=1
-        else
-          echo "OK: $service is running"
-        fi
-      done
-      if [ $FAILED -eq 1 ]; then
-        echo "One or more services failed to start"
-        exit 1
+      STATE_FILE="config/deployment/state.json"
+      if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
+        TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+        jq --arg commit "$CI_COMMIT_SHORT_SHA" \
+           --arg ts "$TIMESTAMP" \
+           '.last_deployment = $ts | .last_deployment_commit = $commit | .last_deployment_status = "success" | .rollback_available = true' \
+           "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
      fi
-    - echo "Checking backend health..."
+
+    - echo "=========================================="
+    - echo "Traffic switch complete"
+    - echo "=========================================="
+
+# ============================================
+# Stage 5: VERIFY
+# Production health verification after switch
+# ============================================
+verify:
+  stage: verify
+  tags:
+    - production
+    - shell
+  only:
+    - main
+  needs:
+    - job: validate
+      artifacts: true
+    - job: deploy-switch
+  script:
+    - echo "=========================================="
+    - echo "Verifying production deployment..."
+    - echo "=========================================="
+    - cd "$DEPLOY_PATH"
+
+    # Wait for Traefik to propagate routing
+    - echo "Waiting for traffic routing to stabilize..."
+    - sleep 5
+
+    # Verify via external endpoint
+    - echo "Checking external endpoint..."
    - |
-      HEALTH_OK=0
      for i in 1 2 3 4 5 6; do
-        if docker exec mvp-backend curl -sf http://localhost:3001/health > /dev/null 2>&1; then
-          echo "OK: Backend health check passed"
-          HEALTH_OK=1
+        if curl -sf https://motovaultpro.com/api/health > /dev/null 2>&1; then
+          echo "OK - External health check passed"
          break
        fi
-        echo "Attempt $i/6: Backend not ready, waiting 10s..."
+        if [ $i -eq 6 ]; then
+          echo "ERROR - External health check failed after 6 attempts"
+          exit 1
+        fi
+        echo "Attempt $i/6 - Waiting 10s..."
        sleep 10
      done
-      if [ $HEALTH_OK -eq 0 ]; then
-        echo "ERROR: Backend health check failed after 6 attempts"
-        docker logs mvp-backend --tail 100
-        exit 1
-      fi
-    - echo "Checking frontend..."
+
+    # Verify container status
+    - echo "Checking container status..."
    - |
-      if docker compose -f $DOCKER_COMPOSE_FILE exec -T mvp-frontend curl -sf http://localhost:3000 > /dev/null 2>&1; then
-        echo "OK: Frontend is accessible"
-      else
-        echo "WARNING: Frontend check failed (might need Traefik routing)"
-      fi
+      for service in mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}; do
+        status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
+        health=$(docker inspect --format='{{.State.Health.Status}}' $service 2>/dev/null || echo "unknown")
+        if [ "$status" != "running" ] || [ "$health" != "healthy" ]; then
+          echo "ERROR - $service is not healthy (status: $status, health: $health)"
+          docker logs $service --tail 50 2>/dev/null || true
+          exit 1
+        fi
+        echo "OK - $service is running and healthy"
+      done
+
    - echo "=========================================="
    - echo "Deployment verified successfully!"
+    - echo "Version ${CI_COMMIT_SHORT_SHA} is now live"
    - echo "=========================================="
+
+# ============================================
+# Stage 6: ROLLBACK (on failure)
+# Automatic rollback if verify stage fails
+# ============================================
+rollback:
+  stage: rollback
+  tags:
+    - production
+    - shell
+  only:
+    - main
+  when: on_failure
+  needs:
+    - job: validate
+      artifacts: true
+    - job: deploy-switch
+    - job: verify
+  script:
+    - echo "=========================================="
+    - echo "INITIATING AUTO-ROLLBACK"
+    - echo "=========================================="
+    - cd "$DEPLOY_PATH"
+
+    # Run rollback script
+    - chmod +x scripts/ci/auto-rollback.sh
+    - ./scripts/ci/auto-rollback.sh "Verify stage failed - automatic rollback"
+
+    # Update state
+    - |
+      STATE_FILE="config/deployment/state.json"
+      if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
+        jq '.last_deployment_status = "rolled_back"' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
+      fi
+
+    - echo "=========================================="
+    - echo "Rollback complete"
+    - echo "=========================================="
+
+# ============================================
+# Stage 7: NOTIFY
+# Send deployment notifications
+# ============================================
+notify-success:
+  stage: notify
+  tags:
+    - production
+    - shell
+  only:
+    - main
+  needs:
+    - job: verify
+  script:
+    - echo "Sending success notification..."
+    - cd "$DEPLOY_PATH"
+    - chmod +x scripts/ci/notify.sh
+    - ./scripts/ci/notify.sh success "Version ${CI_COMMIT_SHORT_SHA} deployed successfully" ${CI_COMMIT_SHORT_SHA}
+
+notify-failure:
+  stage: notify
+  tags:
+    - production
+    - shell
+  only:
+    - main
+  when: on_failure
+  needs:
+    - job: build
+      optional: true
+    - job: deploy-prepare
+      optional: true
+    - job: deploy-switch
+      optional: true
+    - job: verify
+      optional: true
+  script:
+    - echo "Sending failure notification..."
+    - cd "$DEPLOY_PATH"
+    - chmod +x scripts/ci/notify.sh
+    - ./scripts/ci/notify.sh failure "Deployment of ${CI_COMMIT_SHORT_SHA} failed" ${CI_COMMIT_SHORT_SHA}
+
+# ============================================
+# Manual Jobs
+# ============================================
+
+# Manual maintenance migration job
+maintenance-migration:
+  stage: deploy-prepare
+  tags:
+    - production
+    - shell
+  only:
+    - main
+  when: manual
+  script:
+    - echo "=========================================="
+    - echo "MAINTENANCE MODE MIGRATION"
+    - echo "=========================================="
+    - cd "$DEPLOY_PATH"
+    - chmod +x scripts/ci/maintenance-migrate.sh
+    - ./scripts/ci/maintenance-migrate.sh backup
+
+# Mirror base images (scheduled or manual)
+mirror-images:
+  stage: build
+  tags:
+    - build
+  only:
+    - schedules
+    - web
+  when: manual
+  script:
+    - echo "Mirroring base images to GitLab Container Registry..."
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
+    - chmod +x scripts/ci/mirror-base-images.sh
+    - REGISTRY=${REGISTRY}/mirrors ./scripts/ci/mirror-base-images.sh