egullickson/motovaultpro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: I dunno, I'm making git server changes

Browse Source
This commit is contained in:
Eric Gullickson
2025-12-29 08:44:49 -06:00
parent 57d2c43da7
commit 9b0de6a5b8
18 changed files with 2584 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
.github/workflows/build-and-push-image.yml vendored
View File

@@ -1,57 +0,0 @@
name: Build and Push Images to Dockerhub and GHCR
on:
push:
branches: ["main"]
release:
types: ["published"]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
service: [backend, frontend]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: "${{ secrets.DH_USER }}"
password: "${{ secrets.DH_PASS }}"
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: "${{ secrets.GHCR_USER }}"
password: "${{ secrets.GHCR_PAT }}"
- name: Docker Metadata
id: meta
uses: docker/metadata-action@v5
with:
context: workflow
images: |
ericgullickson/motovaultpro-${{ matrix.service }}
ghcr.io/ericgullickson/motovaultpro-${{ matrix.service }}
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=ref,event=tag
labels: |
org.opencontainers.image.title=MotoVaultPro ${{ matrix.service }}
org.opencontainers.image.description=MotoVaultPro ${{ matrix.service }} service
- name: Build and push
uses: docker/build-push-action@v5
with:
context: ./${{ matrix.service }}
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

446
.gitlab-ci.yml
View File

@@ -1,32 +1,50 @@
# MotoVaultPro GitLab CI/CD Pipeline # MotoVaultPro GitLab CI/CD Pipeline - Blue-Green Deployment
# GitLab 18.6+ with shell executor # GitLab 18.6+ with separate build and production runners
# See docs/CICD-DEPLOY.md for complete documentation # See docs/CICD-DEPLOY.md for complete documentation
# v1.6 # v2.0 - Blue-Green with Auto-Rollback
stages: stages:
- validate - validate
- build - build
- deploy - deploy-prepare
- deploy-switch
- verify - verify
- rollback
- notify
variables: variables:
# Use stable clone path instead of runner-specific path # Registry configuration
GIT_CLONE_PATH: $CI_BUILDS_DIR/motovaultpro REGISTRY: registry.motovaultpro.com
DEPLOY_PATH: $CI_BUILDS_DIR/motovaultpro REGISTRY_MIRRORS: ${REGISTRY}/mirrors
DOCKER_COMPOSE_FILE: docker-compose.yml IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}
DOCKER_COMPOSE_PROD_FILE: docker-compose.prod.yml BACKEND_IMAGE: ${REGISTRY}/motovaultpro/backend:${IMAGE_TAG}
FRONTEND_IMAGE: ${REGISTRY}/motovaultpro/frontend:${IMAGE_TAG}
# Fix permissions after every job - docker creates files as root # Deployment configuration
GIT_CLONE_PATH: ${CI_BUILDS_DIR}/motovaultpro
DEPLOY_PATH: ${CI_BUILDS_DIR}/motovaultpro
COMPOSE_FILE: docker-compose.yml
COMPOSE_BLUE_GREEN: docker-compose.blue-green.yml
# Health check configuration
HEALTH_CHECK_TIMEOUT: "60"
# Default after_script to fix permissions
default: default:
after_script: after_script:
- echo "Fixing file permissions..." - echo "Fixing file permissions..."
- sudo chown -R gitlab-runner:gitlab-runner "$DEPLOY_PATH" 2>/dev/null || true - sudo chown -R gitlab-runner:gitlab-runner "$DEPLOY_PATH" 2>/dev/null || true
# Keep data directories owned by container user
- sudo chown -R 1001:1001 "$DEPLOY_PATH/data/backups" "$DEPLOY_PATH/data/documents" 2>/dev/null || true - sudo chown -R 1001:1001 "$DEPLOY_PATH/data/backups" "$DEPLOY_PATH/data/documents" 2>/dev/null || true
# Validate Stage - Check prerequisites # ============================================
# Stage 1: VALIDATE
# Check prerequisites before starting pipeline
# ============================================
validate: validate:
stage: validate stage: validate
tags:
- production
- shell
only: only:
- main - main
script: script:
@@ -34,129 +52,385 @@ validate:
- echo "Validating deployment prerequisites..." - echo "Validating deployment prerequisites..."
- echo "==========================================" - echo "=========================================="
- echo "Checking Docker..." - echo "Checking Docker..."
- 'docker info > /dev/null 2>&1 || (echo "ERROR: Docker not accessible" && exit 1)' - docker info > /dev/null 2>&1 || (echo "ERROR - Docker not accessible" && exit 1)
- echo "OK Docker is accessible" - echo "OK - Docker is accessible"
- echo "Checking Docker Compose..." - echo "Checking Docker Compose..."
- 'docker compose version > /dev/null 2>&1 || (echo "ERROR: Docker Compose not available" && exit 1)' - docker compose version > /dev/null 2>&1 || (echo "ERROR - Docker Compose not available" && exit 1)
- echo "OK Docker Compose is available" - echo "OK - Docker Compose is available"
- echo "Checking deployment path..." - echo "Checking deployment path..."
- 'test -d "$DEPLOY_PATH" || (echo "ERROR: DEPLOY_PATH not found" && exit 1)' - test -d "$DEPLOY_PATH" || (echo "ERROR - DEPLOY_PATH not found" && exit 1)
- echo "OK Deployment path exists" - echo "OK - Deployment path exists"
- echo "Checking registry access..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" || true
- echo "OK - Registry authentication configured"
- echo "Determining target stack..."
- |
STATE_FILE="$DEPLOY_PATH/config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
if [ "$ACTIVE_STACK" = "blue" ]; then
echo "TARGET_STACK=green" >> deploy.env
else
echo "TARGET_STACK=blue" >> deploy.env
fi
else
echo "TARGET_STACK=green" >> deploy.env
fi
cat deploy.env
- echo "==========================================" - echo "=========================================="
- echo "Validation complete" - echo "Validation complete"
- echo "==========================================" - echo "=========================================="
artifacts:
reports:
dotenv: deploy.env
# Build Stage - Build Docker images # ============================================
# Stage 2: BUILD
# Build and push images to GitLab Container Registry
# Runs on dedicated build server (shell executor)
# ============================================
build: build:
stage: build stage: build
tags:
- build
only: only:
- main - main
script: script:
- echo "Authenticating with GitLab Container Registry..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
- echo "==========================================" - echo "=========================================="
- echo "Building Docker images..." - echo "Building Docker images..."
- echo "Commit - ${CI_COMMIT_SHORT_SHA}"
- echo "Backend - ${BACKEND_IMAGE}"
- echo "Frontend - ${FRONTEND_IMAGE}"
- echo "==========================================" - echo "=========================================="
- cd "$DEPLOY_PATH"
- echo "Building images..." # Build backend
- docker compose -f $DOCKER_COMPOSE_FILE build --no-cache - echo "Building backend..."
- |
docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--cache-from ${REGISTRY}/motovaultpro/backend:latest \
-t ${BACKEND_IMAGE} \
-t ${REGISTRY}/motovaultpro/backend:latest \
-f backend/Dockerfile \
.
# Build frontend
- echo "Building frontend..."
- |
docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg VITE_AUTH0_DOMAIN=${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com} \
--build-arg VITE_AUTH0_CLIENT_ID=${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3} \
--build-arg VITE_AUTH0_AUDIENCE=${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com} \
--build-arg VITE_API_BASE_URL=/api \
--cache-from ${REGISTRY}/motovaultpro/frontend:latest \
-t ${FRONTEND_IMAGE} \
-t ${REGISTRY}/motovaultpro/frontend:latest \
-f frontend/Dockerfile \
frontend
# Push images
- echo "Pushing images to registry..."
- docker push ${BACKEND_IMAGE}
- docker push ${FRONTEND_IMAGE}
- docker push ${REGISTRY}/motovaultpro/backend:latest
- docker push ${REGISTRY}/motovaultpro/frontend:latest
- echo "==========================================" - echo "=========================================="
- echo "Build complete" - echo "Build complete"
- echo "==========================================" - echo "=========================================="
# Deploy Stage - Inject secrets and deploy services # ============================================
deploy: # Stage 3: DEPLOY-PREPARE
stage: deploy # Pull images, start inactive stack, run health checks
# ============================================
deploy-prepare:
stage: deploy-prepare
tags:
- production
- shell
only: only:
- main - main
needs:
- job: validate
artifacts: true
- job: build
environment: environment:
name: production name: production
url: https://motovaultpro.com url: https://motovaultpro.com
script: script:
- echo "==========================================" - echo "=========================================="
- echo "Deploying MotoVaultPro..." - echo "Preparing deployment to ${TARGET_STACK} stack..."
- echo "==========================================" - echo "=========================================="
- cd "$DEPLOY_PATH" - cd "$DEPLOY_PATH"
- echo "Step 1/8 Initializing data directories..."
# Authenticate with registry
- echo "Authenticating with registry..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
# Inject secrets
- echo "Step 1/5 - Injecting secrets..."
- chmod +x scripts/inject-secrets.sh
- ./scripts/inject-secrets.sh
# Initialize data directories
- echo "Step 2/5 - Initializing data directories..."
- sudo mkdir -p data/backups data/documents - sudo mkdir -p data/backups data/documents
- sudo chown -R 1001:1001 data/backups data/documents - sudo chown -R 1001:1001 data/backups data/documents
- sudo chmod 755 data/backups data/documents - sudo chmod 755 data/backups data/documents
- echo "Step 2/8 Injecting secrets..."
- chmod +x scripts/inject-secrets.sh # Pull new images
- ./scripts/inject-secrets.sh - echo "Step 3/5 - Pulling images..."
- echo "Step 3/8 Stopping existing services..." - docker pull ${BACKEND_IMAGE}
- docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE down --timeout 30 || true - docker pull ${FRONTEND_IMAGE}
- echo "Step 4/8 Pulling base images..."
- docker compose -f $DOCKER_COMPOSE_FILE pull # Start inactive stack
- echo "Step 5/8 Starting database services..." - echo "Step 4/5 - Starting ${TARGET_STACK} stack..."
- docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d mvp-postgres mvp-redis - |
- echo "Waiting for database to be ready..." export BACKEND_IMAGE=${BACKEND_IMAGE}
- sleep 15 export FRONTEND_IMAGE=${FRONTEND_IMAGE}
- echo "Step 6/8 Running database migrations..." docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \
- docker compose -f $DOCKER_COMPOSE_FILE run --rm mvp-backend npm run migrate || echo "Migration skipped" mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}
- echo "Step 7/8 Vehicle catalog data..."
# Schema and data now loaded via standard migration system # Wait for stack to be ready
# Migration runner handles table creation and data loading automatically - echo "Step 5/5 - Waiting for stack health..."
- echo "Vehicle catalog loaded via platform feature migration" - sleep 10
- echo "Flushing Redis cache..."
- docker exec mvp-redis redis-cli FLUSHALL # Run health check
- echo "Step 8/8 Starting all services..." - echo "Running health check on ${TARGET_STACK} stack..."
- docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d - chmod +x scripts/ci/health-check.sh
- echo "Waiting for services to initialize..." - ./scripts/ci/health-check.sh ${TARGET_STACK} ${HEALTH_CHECK_TIMEOUT}
- sleep 30
# Update state with deployment info
- |
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg stack "$TARGET_STACK" \
--arg commit "$CI_COMMIT_SHORT_SHA" \
--arg ts "$TIMESTAMP" \
'.[$stack].version = $commit | .[$stack].commit = $commit | .[$stack].deployed_at = $ts | .[$stack].healthy = true' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
- echo "==========================================" - echo "=========================================="
- echo "Deployment complete" - echo "Deploy preparation complete"
- echo "==========================================" - echo "=========================================="
# Verify Stage - Health checks # ============================================
verify: # Stage 4: DEPLOY-SWITCH
stage: verify # Switch traffic to new stack
# ============================================
deploy-switch:
stage: deploy-switch
tags:
- production
- shell
only: only:
- main - main
needs:
- job: validate
artifacts: true
- job: deploy-prepare
script: script:
- echo "==========================================" - echo "=========================================="
- echo "Verifying deployment..." - echo "Switching traffic to ${TARGET_STACK} stack..."
- echo "==========================================" - echo "=========================================="
- cd "$DEPLOY_PATH" - cd "$DEPLOY_PATH"
- echo "Checking container status..."
# Switch traffic
- chmod +x scripts/ci/switch-traffic.sh
- ./scripts/ci/switch-traffic.sh ${TARGET_STACK} instant
# Update state
- | - |
FAILED=0 STATE_FILE="config/deployment/state.json"
for service in mvp-traefik mvp-frontend mvp-backend mvp-postgres mvp-redis; do if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found") TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
if [ "$status" != "running" ]; then jq --arg commit "$CI_COMMIT_SHORT_SHA" \
echo "ERROR: $service is not running (status: $status)" --arg ts "$TIMESTAMP" \
docker logs $service --tail 50 2>/dev/null || true '.last_deployment = $ts | .last_deployment_commit = $commit | .last_deployment_status = "success" | .rollback_available = true' \
FAILED=1 "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
else
echo "OK: $service is running"
fi fi
done
if [ $FAILED -eq 1 ]; then - echo "=========================================="
echo "One or more services failed to start" - echo "Traffic switch complete"
exit 1 - echo "=========================================="
fi
- echo "Checking backend health..." # ============================================
# Stage 5: VERIFY
# Production health verification after switch
# ============================================
verify:
stage: verify
tags:
- production
- shell
only:
- main
needs:
- job: validate
artifacts: true
- job: deploy-switch
script:
- echo "=========================================="
- echo "Verifying production deployment..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Wait for Traefik to propagate routing
- echo "Waiting for traffic routing to stabilize..."
- sleep 5
# Verify via external endpoint
- echo "Checking external endpoint..."
- | - |
HEALTH_OK=0
for i in 1 2 3 4 5 6; do for i in 1 2 3 4 5 6; do
if docker exec mvp-backend curl -sf http://localhost:3001/health > /dev/null 2>&1; then if curl -sf https://motovaultpro.com/api/health > /dev/null 2>&1; then
echo "OK: Backend health check passed" echo "OK - External health check passed"
HEALTH_OK=1
break break
fi fi
echo "Attempt $i/6: Backend not ready, waiting 10s..." if [ $i -eq 6 ]; then
sleep 10 echo "ERROR - External health check failed after 6 attempts"
done
if [ $HEALTH_OK -eq 0 ]; then
echo "ERROR: Backend health check failed after 6 attempts"
docker logs mvp-backend --tail 100
exit 1 exit 1
fi fi
- echo "Checking frontend..." echo "Attempt $i/6 - Waiting 10s..."
sleep 10
done
# Verify container status
- echo "Checking container status..."
- | - |
if docker compose -f $DOCKER_COMPOSE_FILE exec -T mvp-frontend curl -sf http://localhost:3000 > /dev/null 2>&1; then for service in mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}; do
echo "OK: Frontend is accessible" status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
else health=$(docker inspect --format='{{.State.Health.Status}}' $service 2>/dev/null || echo "unknown")
echo "WARNING: Frontend check failed (might need Traefik routing)" if [ "$status" != "running" ] || [ "$health" != "healthy" ]; then
echo "ERROR - $service is not healthy (status: $status, health: $health)"
docker logs $service --tail 50 2>/dev/null || true
exit 1
fi fi
echo "OK - $service is running and healthy"
done
- echo "==========================================" - echo "=========================================="
- echo "Deployment verified successfully!" - echo "Deployment verified successfully!"
- echo "Version ${CI_COMMIT_SHORT_SHA} is now live"
- echo "==========================================" - echo "=========================================="
# ============================================
# Stage 6: ROLLBACK (on failure)
# Automatic rollback if verify stage fails
# ============================================
rollback:
stage: rollback
tags:
- production
- shell
only:
- main
when: on_failure
needs:
- job: validate
artifacts: true
- job: deploy-switch
- job: verify
script:
- echo "=========================================="
- echo "INITIATING AUTO-ROLLBACK"
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Run rollback script
- chmod +x scripts/ci/auto-rollback.sh
- ./scripts/ci/auto-rollback.sh "Verify stage failed - automatic rollback"
# Update state
- |
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
jq '.last_deployment_status = "rolled_back"' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
- echo "=========================================="
- echo "Rollback complete"
- echo "=========================================="
# ============================================
# Stage 7: NOTIFY
# Send deployment notifications
# ============================================
notify-success:
stage: notify
tags:
- production
- shell
only:
- main
needs:
- job: verify
script:
- echo "Sending success notification..."
- cd "$DEPLOY_PATH"
- chmod +x scripts/ci/notify.sh
- ./scripts/ci/notify.sh success "Version ${CI_COMMIT_SHORT_SHA} deployed successfully" ${CI_COMMIT_SHORT_SHA}
notify-failure:
stage: notify
tags:
- production
- shell
only:
- main
when: on_failure
needs:
- job: build
optional: true
- job: deploy-prepare
optional: true
- job: deploy-switch
optional: true
- job: verify
optional: true
script:
- echo "Sending failure notification..."
- cd "$DEPLOY_PATH"
- chmod +x scripts/ci/notify.sh
- ./scripts/ci/notify.sh failure "Deployment of ${CI_COMMIT_SHORT_SHA} failed" ${CI_COMMIT_SHORT_SHA}
# ============================================
# Manual Jobs
# ============================================
# Manual maintenance migration job
maintenance-migration:
stage: deploy-prepare
tags:
- production
- shell
only:
- main
when: manual
script:
- echo "=========================================="
- echo "MAINTENANCE MODE MIGRATION"
- echo "=========================================="
- cd "$DEPLOY_PATH"
- chmod +x scripts/ci/maintenance-migrate.sh
- ./scripts/ci/maintenance-migrate.sh backup
# Mirror base images (scheduled or manual)
mirror-images:
stage: build
tags:
- build
only:
- schedules
- web
when: manual
script:
- echo "Mirroring base images to GitLab Container Registry..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
- chmod +x scripts/ci/mirror-base-images.sh
- REGISTRY=${REGISTRY}/mirrors ./scripts/ci/mirror-base-images.sh

8
backend/Dockerfile
View File

@@ -1,7 +1,11 @@
# Production Dockerfile for MotoVaultPro Backend # Production Dockerfile for MotoVaultPro Backend
# Uses mirrored base images from GitLab Container Registry
# Build argument for registry (defaults to GitLab mirrors, falls back to Docker Hub)
ARG REGISTRY_MIRRORS=registry.motovaultpro.com/mirrors
# Stage 1: Build stage # Stage 1: Build stage
FROM node:lts-alpine AS builder FROM ${REGISTRY_MIRRORS}/node:20-alpine AS builder
# Install build dependencies # Install build dependencies
RUN apk add --no-cache dumb-init git curl RUN apk add --no-cache dumb-init git curl
@@ -27,7 +31,7 @@ ENV DOCKER_BUILD=true
RUN npm run build RUN npm run build
# Stage 2: Production runtime # Stage 2: Production runtime
FROM node:lts-alpine AS production FROM ${REGISTRY_MIRRORS}/node:20-alpine AS production
# Install runtime dependencies only (postgresql-client for backup/restore) # Install runtime dependencies only (postgresql-client for backup/restore)
RUN apk add --no-cache dumb-init curl postgresql-client RUN apk add --no-cache dumb-init curl postgresql-client

21
config/deployment/state.json Normal file
View File

@@ -0,0 +1,21 @@
{
"active_stack": "blue",
"inactive_stack": "green",
"last_deployment": null,
"last_deployment_commit": null,
"last_deployment_status": null,
"blue": {
"version": null,
"commit": null,
"deployed_at": null,
"healthy": false
},
"green": {
"version": null,
"commit": null,
"deployed_at": null,
"healthy": false
},
"rollback_available": false,
"maintenance_mode": false
}

116
config/traefik/dynamic/blue-green.yml Normal file
View File

@@ -0,0 +1,116 @@
# Traefik Dynamic Configuration for Blue-Green Deployment
# This file is watched by Traefik and reloaded on changes
# Traffic weights are updated by scripts/ci/switch-traffic.sh
#
# Current active stack is determined by weights:
# - blue=100, green=0 -> Blue is active
# - blue=0, green=100 -> Green is active
# - Gradual: 75/25, 50/50, 25/75 for canary deployments
http:
# ========================================
# Routers - Route traffic to weighted services
# ========================================
routers:
# Frontend router with weighted service
mvp-frontend-bluegreen:
rule: "(Host(`motovaultpro.com`) || Host(`www.motovaultpro.com`)) && !PathPrefix(`/api`)"
entryPoints:
- websecure
tls:
certResolver: letsencrypt
service: mvp-frontend-weighted
priority: 10
# Backend API router with weighted service
mvp-backend-bluegreen:
rule: "(Host(`motovaultpro.com`) || Host(`www.motovaultpro.com`)) && PathPrefix(`/api`)"
entryPoints:
- websecure
tls:
certResolver: letsencrypt
service: mvp-backend-weighted
priority: 20
# Health check router (always routes to active stack)
mvp-backend-health:
rule: "(Host(`motovaultpro.com`) || Host(`www.motovaultpro.com`)) && Path(`/api/health`)"
entryPoints:
- websecure
tls:
certResolver: letsencrypt
service: mvp-backend-weighted
priority: 30
# ========================================
# Services - Weighted load balancers
# ========================================
services:
# Frontend weighted service
# Weights are updated by switch-traffic.sh
mvp-frontend-weighted:
weighted:
services:
- name: mvp-frontend-blue-svc
weight: 100
- name: mvp-frontend-green-svc
weight: 0
healthCheck: {}
# Backend weighted service
# Weights are updated by switch-traffic.sh
mvp-backend-weighted:
weighted:
services:
- name: mvp-backend-blue-svc
weight: 100
- name: mvp-backend-green-svc
weight: 0
healthCheck: {}
# Individual stack services
mvp-frontend-blue-svc:
loadBalancer:
servers:
- url: "http://mvp-frontend-blue:3000"
healthCheck:
path: /
interval: 10s
timeout: 3s
passHostHeader: true
mvp-frontend-green-svc:
loadBalancer:
servers:
- url: "http://mvp-frontend-green:3000"
healthCheck:
path: /
interval: 10s
timeout: 3s
passHostHeader: true
mvp-backend-blue-svc:
loadBalancer:
servers:
- url: "http://mvp-backend-blue:3001"
healthCheck:
path: /health
interval: 10s
timeout: 3s
passHostHeader: true
mvp-backend-green-svc:
loadBalancer:
servers:
- url: "http://mvp-backend-green:3001"
healthCheck:
path: /health
interval: 10s
timeout: 3s
passHostHeader: true
# Maintenance mode service (optional)
mvp-maintenance:
loadBalancer:
servers:
- url: "http://mvp-maintenance:80"

180
config/traefik/dynamic/middleware.yml Executable file
View File

@@ -0,0 +1,180 @@
http:
middlewares:
# Security headers middleware
secure-headers:
headers:
accessControlAllowMethods:
- GET
- OPTIONS
- PUT
- POST
- DELETE
accessControlAllowOriginList:
- "https://admin.motovaultpro.com"
- "https://motovaultpro.com"
accessControlMaxAge: 100
addVaryHeader: true
browserXssFilter: true
contentTypeNosniff: true
forceSTSHeader: true
frameDeny: true
stsIncludeSubdomains: true
stsPreload: true
stsSeconds: 31536000
customRequestHeaders:
X-Forwarded-Proto: https
# CORS middleware for API endpoints
cors:
headers:
accessControlAllowCredentials: true
accessControlAllowHeaders:
- "Authorization"
- "Content-Type"
- "X-Requested-With"
- "X-Tenant-ID"
accessControlAllowMethods:
- "GET"
- "POST"
- "PUT"
- "DELETE"
- "OPTIONS"
accessControlAllowOriginList:
- "https://admin.motovaultpro.com"
- "https://motovaultpro.com"
accessControlMaxAge: 100
# API authentication middleware
api-auth:
forwardAuth:
address: "http://admin-backend:3001/auth/verify"
authResponseHeaders:
- "X-Auth-User"
- "X-Auth-Roles"
- "X-Tenant-ID"
authRequestHeaders:
- "Authorization"
- "X-Tenant-ID"
trustForwardHeader: true
# Platform API authentication middleware
platform-auth:
forwardAuth:
address: "http://admin-backend:3001/auth/verify-platform"
authResponseHeaders:
- "X-Service-Name"
- "X-Auth-Scope"
authRequestHeaders:
- "X-API-Key"
- "Authorization"
trustForwardHeader: true
# Rate limiting middleware
rate-limit:
rateLimit:
burst: 100
average: 50
period: 1m
# Request/response size limits
size-limit:
buffering:
maxRequestBodyBytes: 26214400 # 25MB
maxResponseBodyBytes: 26214400 # 25MB
# IP whitelist for development (optional)
local-ips:
ipWhiteList:
sourceRange:
- "127.0.0.1/32"
- "10.0.0.0/8"
- "172.16.0.0/12"
- "192.168.0.0/16"
# Advanced security headers for production
security-headers-strict:
headers:
accessControlAllowCredentials: false
accessControlAllowMethods:
- GET
- POST
- OPTIONS
accessControlAllowOriginList:
- "https://admin.motovaultpro.com"
- "https://motovaultpro.com"
browserXssFilter: true
contentTypeNosniff: true
customRequestHeaders:
X-Forwarded-Proto: https
customResponseHeaders:
X-Frame-Options: DENY
X-Content-Type-Options: nosniff
Referrer-Policy: strict-origin-when-cross-origin
Permissions-Policy: "geolocation=(), microphone=(), camera=()"
forceSTSHeader: true
frameDeny: true
stsIncludeSubdomains: true
stsPreload: true
stsSeconds: 31536000
# Circuit breaker for reliability
circuit-breaker:
circuitBreaker:
expression: "NetworkErrorRatio() > 0.3 || ResponseCodeRatio(500, 600, 0, 600) > 0.3"
checkPeriod: 30s
fallbackDuration: 10s
recoveryDuration: 30s
# Request retry for resilience
retry-policy:
retry:
attempts: 3
initialInterval: 100ms
# Timeout middleware
timeout:
timeout: 30s
# Compress responses for performance
compression:
compress: {}
# Health check middleware chain
health-check-chain:
chain:
middlewares:
- compression
- secure-headers
- timeout
# API middleware chain
api-chain:
chain:
middlewares:
- compression
- security-headers-strict
- cors
- rate-limit
- api-auth
- retry-policy
- timeout
# Platform API middleware chain
platform-chain:
chain:
middlewares:
- compression
- security-headers-strict
- rate-limit
- platform-auth
- circuit-breaker
- retry-policy
- timeout
# Public frontend middleware chain
frontend-chain:
chain:
middlewares:
- compression
- secure-headers
- timeout

3
config/traefik/traefik.yml
View File

@@ -21,7 +21,8 @@ providers:
exposedByDefault: false exposedByDefault: false
# Network auto-discovery - Traefik will use the networks it's connected to # Network auto-discovery - Traefik will use the networks it's connected to
file: file:
filename: /etc/traefik/middleware.yml # Watch directory for dynamic configuration (blue-green routing, middleware)
directory: /etc/traefik/dynamic
watch: true watch: true
certificatesResolvers: certificatesResolvers:

196
docker-compose.blue-green.yml Normal file
View File

@@ -0,0 +1,196 @@
# Blue-Green Deployment Overlay for MotoVaultPro
# Usage: docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d
#
# This overlay defines blue and green stacks that share the same database layer.
# Traffic routing is handled by Traefik's weighted load balancer.
#
# Stack naming:
# BLUE: mvp-frontend-blue, mvp-backend-blue
# GREEN: mvp-frontend-green, mvp-backend-green
#
# Shared services (from base compose):
# mvp-traefik, mvp-postgres, mvp-redis
services:
# ========================================
# BLUE Stack - Frontend
# ========================================
mvp-frontend-blue:
image: ${FRONTEND_IMAGE:-registry.motovaultpro.com/motovaultpro/frontend:latest}
container_name: mvp-frontend-blue
restart: unless-stopped
environment:
VITE_API_BASE_URL: /api
VITE_AUTH0_DOMAIN: ${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com}
VITE_AUTH0_CLIENT_ID: ${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3}
VITE_AUTH0_AUDIENCE: ${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com}
SECRETS_DIR: /run/secrets
volumes:
- ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro
- ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro
networks:
- frontend
depends_on:
- mvp-backend-blue
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3000 || exit 1"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
deploy:
resources:
limits:
memory: 512M
labels:
- "traefik.enable=true"
- "traefik.docker.network=motovaultpro_frontend"
- "com.motovaultpro.stack=blue"
- "com.motovaultpro.service=frontend"
# ========================================
# BLUE Stack - Backend
# ========================================
mvp-backend-blue:
image: ${BACKEND_IMAGE:-registry.motovaultpro.com/motovaultpro/backend:latest}
container_name: mvp-backend-blue
restart: unless-stopped
environment:
NODE_ENV: production
CONFIG_PATH: /app/config/production.yml
SECRETS_DIR: /run/secrets
DATABASE_HOST: mvp-postgres
REDIS_HOST: mvp-redis
volumes:
- ./config/app/production.yml:/app/config/production.yml:ro
- ./config/shared/production.yml:/app/config/shared.yml:ro
- ./secrets/app/postgres-password.txt:/run/secrets/postgres-password:ro
- ./secrets/app/auth0-client-secret.txt:/run/secrets/auth0-client-secret:ro
- ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro
- ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro
- ./secrets/app/resend-api-key.txt:/run/secrets/resend-api-key:ro
- ./secrets/app/auth0-management-client-id.txt:/run/secrets/auth0-management-client-id:ro
- ./secrets/app/auth0-management-client-secret.txt:/run/secrets/auth0-management-client-secret:ro
- ./data/documents:/app/data/documents
- ./data/backups:/app/data/backups
networks:
- backend
- database
depends_on:
- mvp-postgres
- mvp-redis
healthcheck:
test:
- CMD-SHELL
- node -e "require('http').get('http://localhost:3001/health', r => process.exit(r.statusCode===200?0:1)).on('error', () => process.exit(1))"
interval: 10s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
memory: 1G
labels:
- "traefik.enable=true"
- "traefik.docker.network=motovaultpro_backend"
- "com.motovaultpro.stack=blue"
- "com.motovaultpro.service=backend"
# ========================================
# GREEN Stack - Frontend
# ========================================
mvp-frontend-green:
image: ${FRONTEND_IMAGE:-registry.motovaultpro.com/motovaultpro/frontend:latest}
container_name: mvp-frontend-green
restart: unless-stopped
environment:
VITE_API_BASE_URL: /api
VITE_AUTH0_DOMAIN: ${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com}
VITE_AUTH0_CLIENT_ID: ${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3}
VITE_AUTH0_AUDIENCE: ${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com}
SECRETS_DIR: /run/secrets
volumes:
- ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro
- ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro
networks:
- frontend
depends_on:
- mvp-backend-green
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:3000 || exit 1"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
deploy:
resources:
limits:
memory: 512M
labels:
- "traefik.enable=true"
- "traefik.docker.network=motovaultpro_frontend"
- "com.motovaultpro.stack=green"
- "com.motovaultpro.service=frontend"
# ========================================
# GREEN Stack - Backend
# ========================================
mvp-backend-green:
image: ${BACKEND_IMAGE:-registry.motovaultpro.com/motovaultpro/backend:latest}
container_name: mvp-backend-green
restart: unless-stopped
environment:
NODE_ENV: production
CONFIG_PATH: /app/config/production.yml
SECRETS_DIR: /run/secrets
DATABASE_HOST: mvp-postgres
REDIS_HOST: mvp-redis
volumes:
- ./config/app/production.yml:/app/config/production.yml:ro
- ./config/shared/production.yml:/app/config/shared.yml:ro
- ./secrets/app/postgres-password.txt:/run/secrets/postgres-password:ro
- ./secrets/app/auth0-client-secret.txt:/run/secrets/auth0-client-secret:ro
- ./secrets/app/google-maps-api-key.txt:/run/secrets/google-maps-api-key:ro
- ./secrets/app/google-maps-map-id.txt:/run/secrets/google-maps-map-id:ro
- ./secrets/app/resend-api-key.txt:/run/secrets/resend-api-key:ro
- ./secrets/app/auth0-management-client-id.txt:/run/secrets/auth0-management-client-id:ro
- ./secrets/app/auth0-management-client-secret.txt:/run/secrets/auth0-management-client-secret:ro
- ./data/documents:/app/data/documents
- ./data/backups:/app/data/backups
networks:
- backend
- database
depends_on:
- mvp-postgres
- mvp-redis
healthcheck:
test:
- CMD-SHELL
- node -e "require('http').get('http://localhost:3001/health', r => process.exit(r.statusCode===200?0:1)).on('error', () => process.exit(1))"
interval: 10s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
memory: 1G
labels:
- "traefik.enable=true"
- "traefik.docker.network=motovaultpro_backend"
- "com.motovaultpro.stack=green"
- "com.motovaultpro.service=backend"
# ========================================
# Override Traefik to add dynamic config
# ========================================
mvp-traefik:
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- ./config/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
- ./config/traefik/middleware.yml:/etc/traefik/middleware.yml:ro
- ./config/traefik/dynamic:/etc/traefik/dynamic:ro
- ./certs:/certs:ro
- traefik_data:/data
- ./secrets/app/cloudflare-dns-token.txt:/run/secrets/cloudflare-dns-token:ro

12
docker-compose.yml
View File

@@ -1,7 +1,11 @@
# Base registry for mirrored images (override with environment variable)
x-registry: &registry
REGISTRY_MIRRORS: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}
services: services:
# Traefik - Service Discovery and Load Balancing # Traefik - Service Discovery and Load Balancing
mvp-traefik: mvp-traefik:
image: traefik:v3.6 image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/traefik:v3.6
container_name: mvp-traefik container_name: mvp-traefik
restart: unless-stopped restart: unless-stopped
command: command:
@@ -15,7 +19,7 @@ services:
volumes: volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro - /var/run/docker.sock:/var/run/docker.sock:ro
- ./config/traefik/traefik.yml:/etc/traefik/traefik.yml:ro - ./config/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
- ./config/traefik/middleware.yml:/etc/traefik/middleware.yml:ro - ./config/traefik/dynamic:/etc/traefik/dynamic:ro
- ./certs:/certs:ro - ./certs:/certs:ro
- traefik_data:/data - traefik_data:/data
- ./secrets/app/cloudflare-dns-token.txt:/run/secrets/cloudflare-dns-token:ro - ./secrets/app/cloudflare-dns-token.txt:/run/secrets/cloudflare-dns-token:ro
@@ -154,7 +158,7 @@ services:
# Database Services - Application PostgreSQL # Database Services - Application PostgreSQL
mvp-postgres: mvp-postgres:
image: postgres:18-alpine image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/postgres:18-alpine
container_name: mvp-postgres container_name: mvp-postgres
restart: unless-stopped restart: unless-stopped
environment: environment:
@@ -179,7 +183,7 @@ services:
# Database Services - Application Redis # Database Services - Application Redis
mvp-redis: mvp-redis:
image: redis:8.4-alpine image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/redis:8.4-alpine
container_name: mvp-redis container_name: mvp-redis
restart: unless-stopped restart: unless-stopped
command: redis-server --appendonly yes command: redis-server --appendonly yes

315
docs/BUILD-SERVER-SETUP.md Normal file
View File

@@ -0,0 +1,315 @@
# Build Server Setup Guide
Complete guide for setting up a dedicated build VPS for MotoVaultPro CI/CD pipeline.
## Overview
The build server isolates resource-intensive Docker builds from the production server, ensuring deployments don't impact application performance.
```
+-------------------+ +--------------------+
| GitLab Server | | Production Server |
| (CI/CD + Registry)| | (Shell Runner) |
+--------+----------+ +----------+---------+
| |
v v
+--------+----------+ +----------+---------+
| Build VPS | | Blue-Green Stacks |
| (Docker Runner) |---->| + Shared Data |
+-------------------+ +--------------------+
```
## Server Requirements
### Minimum Specifications
| Resource | Requirement |
|----------|-------------|
| CPU | 2 cores |
| RAM | 4GB |
| Storage | 50GB SSD |
| Network | 100Mbps+ |
| OS | Ubuntu 22.04 LTS / Debian 12 |
### Network Requirements
- Outbound HTTPS to GitLab instance
- Outbound HTTPS to Docker registries (for fallback)
- SSH access for administration
---
## Installation Steps
### 1. Update System
```bash
sudo apt update && sudo apt upgrade -y
sudo apt install -y curl git ca-certificates gnupg
```
### 2. Install Docker Engine
```bash
# Add Docker's official GPG key
sudo install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg
# Add the repository to Apt sources
echo \
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
# Install Docker
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
# Verify installation
docker --version
docker compose version
```
### 3. Install GitLab Runner
```bash
# Add GitLab Runner repository
curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash
# Install GitLab Runner
sudo apt install gitlab-runner
# Verify installation
gitlab-runner --version
```
### 4. Register Runner with Shell Executor
```bash
sudo gitlab-runner register \
--non-interactive \
--url "https://git.motovaultpro.com" \
--registration-token "YOUR_REGISTRATION_TOKEN" \
--executor "shell" \
--description "Build Server - Shell Executor" \
--tag-list "build" \
--run-untagged="false" \
--locked="true"
```
**Notes:**
- Replace `YOUR_REGISTRATION_TOKEN` with the token from GitLab Admin > CI/CD > Runners
- Shell executor runs jobs directly on the host with access to Docker
- Tag `build` is used in `.gitlab-ci.yml` to route build jobs to this server
### 5. Add gitlab-runner to Docker Group
The gitlab-runner user needs access to Docker:
```bash
sudo usermod -aG docker gitlab-runner
# Verify access
sudo -u gitlab-runner docker info
sudo -u gitlab-runner docker compose version
```
### 6. Configure Docker Registry Authentication
Create credentials file for GitLab Container Registry:
```bash
# Login to GitLab Container Registry (creates ~/.docker/config.json)
docker login registry.motovaultpro.com -u <deploy-token-username> -p <deploy-token>
```
**Creating Deploy Token:**
1. Go to GitLab Project > Settings > Repository > Deploy Tokens
2. Create token with `read_registry` and `write_registry` scopes
3. Use the token username/password for Docker login
---
## Verification
### Test Runner Registration
```bash
sudo gitlab-runner verify
```
Expected output:
```
Verifying runner... is alive runner=XXXXXX
```
### Test Docker Access
```bash
sudo gitlab-runner exec docker --docker-privileged test-job
```
### Test Registry Push
```bash
# Build and push a test image
docker build -t registry.motovaultpro.com/motovaultpro/test:latest -f- . <<EOF
FROM alpine:latest
RUN echo "test"
EOF
docker push registry.motovaultpro.com/motovaultpro/test:latest
```
---
## Maintenance
### Disk Cleanup
Docker builds accumulate disk space. Set up automated cleanup:
```bash
# Create cleanup script
sudo tee /usr/local/bin/docker-cleanup.sh > /dev/null <<'EOF'
#!/bin/bash
# Remove unused Docker resources older than 7 days
docker system prune -af --filter "until=168h"
docker volume prune -f
EOF
sudo chmod +x /usr/local/bin/docker-cleanup.sh
# Add to crontab (run daily at 3 AM)
echo "0 3 * * * /usr/local/bin/docker-cleanup.sh >> /var/log/docker-cleanup.log 2>&1" | sudo crontab -
```
### Log Rotation
Configure log rotation for GitLab Runner:
```bash
sudo tee /etc/logrotate.d/gitlab-runner > /dev/null <<EOF
/var/log/gitlab-runner/*.log {
daily
rotate 7
compress
missingok
notifempty
}
EOF
```
### Update Runner
```bash
# Update GitLab Runner
sudo apt update
sudo apt upgrade gitlab-runner
# Restart runner
sudo gitlab-runner restart
```
---
## Security Considerations
### Firewall Configuration
```bash
# Allow only necessary outbound traffic
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow ssh
sudo ufw enable
```
### Runner Security
- **Locked runner**: Only accepts jobs from the specific project
- **Protected tags**: Only runs on protected branches (main)
- **Docker socket**: Mounted read-only where possible
### Secrets Management
The build server does NOT store application secrets. All secrets are:
- Stored in GitLab CI/CD Variables
- Injected at runtime on the production server
- Never cached in Docker layers
---
## Troubleshooting
### Runner Not Picking Up Jobs
```bash
# Check runner status
sudo gitlab-runner status
# View runner logs
sudo journalctl -u gitlab-runner -f
# Re-register runner if needed
sudo gitlab-runner unregister --all-runners
sudo gitlab-runner register
```
### Docker Build Failures
```bash
# Check Docker daemon
sudo systemctl status docker
# Check available disk space
df -h
# Clear Docker cache
docker system prune -af
```
### Registry Push Failures
```bash
# Verify registry login
docker login registry.motovaultpro.com
# Check network connectivity
curl -v https://registry.motovaultpro.com/v2/
# Verify image exists
docker images | grep motovaultpro
```
---
## Quick Reference
### Important Paths
| Path | Description |
|------|-------------|
| `/etc/gitlab-runner/config.toml` | Runner configuration |
| `/var/log/gitlab-runner/` | Runner logs |
| `~/.docker/config.json` | Docker registry credentials |
| `/var/lib/docker/` | Docker data |
### Common Commands
```bash
# Runner management
sudo gitlab-runner status
sudo gitlab-runner restart
sudo gitlab-runner verify
# Docker management
docker system df # Check disk usage
docker system prune -af # Clean all unused resources
docker images # List images
docker ps -a # List containers
# View build logs
sudo journalctl -u gitlab-runner --since "1 hour ago"
```

731
docs/CICD-DEPLOY.md
View File

@@ -1,17 +1,72 @@
# MotoVaultPro GitLab CI/CD Deployment Guide # MotoVaultPro GitLab CI/CD Deployment Guide
Complete guide for deploying MotoVaultPro using GitLab CI/CD with shell executor runners. Complete guide for deploying MotoVaultPro using GitLab CI/CD with blue-green deployment and auto-rollback.
## Table of Contents ## Table of Contents
1. [Prerequisites](#prerequisites) 1. [Architecture Overview](#architecture-overview)
2. [GitLab Runner Setup](#gitlab-runner-setup) 2. [Prerequisites](#prerequisites)
3. [CI/CD Variables Configuration](#cicd-variables-configuration) 3. [Pipeline Stages](#pipeline-stages)
4. [Secrets Architecture](#secrets-architecture) 4. [Blue-Green Deployment](#blue-green-deployment)
5. [Pipeline Overview](#pipeline-overview) 5. [CI/CD Variables Configuration](#cicd-variables-configuration)
6. [Deployment Process](#deployment-process) 6. [Container Registry](#container-registry)
7. [Rollback Procedure](#rollback-procedure) 7. [Deployment Process](#deployment-process)
8. [Troubleshooting](#troubleshooting) 8. [Rollback Procedures](#rollback-procedures)
9. [Maintenance Migrations](#maintenance-migrations)
10. [Notifications](#notifications)
11. [Troubleshooting](#troubleshooting)
---
## Architecture Overview
MotoVaultPro uses a blue-green deployment strategy with automatic rollback:
```
+---------------------------------------------------+
| GitLab (CI/CD + Registry) |
+---------------------------------------------------+
| |
v v
+------------------+ +-----------------------+
| Build VPS | | Production Server |
| (Docker Runner) | | (Shell Runner) |
| Tags: build | | Tags: production |
+------------------+ +-----------+-----------+
| |
| Push images | Pull + Deploy
v v
+---------------------------------------------------+
| GitLab Container Registry |
| registry.motovaultpro.com/motovaultpro/ |
+---------------------------------------------------+
|
+---------------+---------------+
| |
+--------v--------+ +--------v--------+
| BLUE Stack | | GREEN Stack |
| mvp-frontend | | mvp-frontend |
| mvp-backend | | mvp-backend |
+-----------------+ +-----------------+
| |
+----------- Traefik -----------+
(weighted LB)
|
+---------------+---------------+
| |
+--------v--------+ +--------v--------+
| PostgreSQL | | Redis |
| (shared) | | (shared) |
+-----------------+ +-----------------+
```
### Key Features
- **Zero-downtime deployments**: Traffic switches in under 5 seconds
- **Instant rollback**: Previous version remains running
- **Automatic rollback**: On health check failure
- **Email notifications**: Via Resend API
- **Container registry**: Self-hosted on GitLab (no Docker Hub)
--- ---
@@ -19,55 +74,115 @@ Complete guide for deploying MotoVaultPro using GitLab CI/CD with shell executor
### Server Requirements ### Server Requirements
- Linux server with Docker Engine installed | Server | Purpose | Specs | Runner Tags |
- Docker Compose v2 (plugin version) |--------|---------|-------|-------------|
- GitLab Runner installed and registered | Build VPS | Docker image builds | 2 CPU, 4GB RAM | `build` |
- Git installed | Prod Server | Application hosting | 8GB+ RAM | `production` |
- curl installed (for health checks)
### GitLab Requirements See [BUILD-SERVER-SETUP.md](BUILD-SERVER-SETUP.md) for build server setup.
- GitLab 18.6+ (tested with 18.6.2) ### Software Requirements
- Project with CI/CD enabled
- Protected `main` branch - GitLab 18.6+
- Maintainer access for CI/CD variable configuration - Docker Engine 24.0+
- Docker Compose v2
- GitLab Runner (shell executor on both servers)
- `jq` for JSON processing
--- ---
## GitLab Runner Setup ## Pipeline Stages
### 1. Verify Runner Registration The CI/CD pipeline consists of 7 stages:
```bash ```
sudo gitlab-runner verify validate -> build -> deploy-prepare -> deploy-switch -> verify -> [rollback] -> notify
``` ```
Expected output should show your runner as active with shell executor. | Stage | Runner | Purpose |
|-------|--------|---------|
| `validate` | prod | Check prerequisites, determine target stack |
| `build` | build | Build and push images to GitLab registry |
| `deploy-prepare` | prod | Pull images, start inactive stack, health check |
| `deploy-switch` | prod | Switch Traefik traffic weights |
| `verify` | prod | Production health verification |
| `rollback` | prod | Auto-triggered on verify failure |
| `notify` | prod | Email success/failure notifications |
### 2. Verify Docker Permissions ### Pipeline Flow
The `gitlab-runner` user must have Docker access: ```
[Push to main]
```bash |
# Add gitlab-runner to docker group (if not already done) v
sudo usermod -aG docker gitlab-runner [validate] - Checks Docker, paths, registry
|
# Verify access v
sudo -u gitlab-runner docker info [build] - Builds backend + frontend images
sudo -u gitlab-runner docker compose version | Pushes to registry.motovaultpro.com
v
[deploy-prepare] - Pulls new images
| Starts inactive stack (blue or green)
| Runs health checks
v
[deploy-switch] - Updates Traefik weights
| Switches traffic instantly
v
[verify] - External health check
| Container status verification
|
+--[SUCCESS]--> [notify-success] - Sends success email
|
+--[FAILURE]--> [rollback] - Switches back to previous stack
|
v
[notify-failure] - Sends failure email
``` ```
### 3. Verify Deployment Directory ---
Ensure the deployment directory exists and is accessible: ## Blue-Green Deployment
```bash ### Stack Configuration
# Create deployment directory
sudo mkdir -p /opt/motovaultpro
sudo chown gitlab-runner:gitlab-runner /opt/motovaultpro
# Clone repository (first time only) Both stacks share the same database layer:
sudo -u gitlab-runner git clone <repository-url> /opt/motovaultpro
| Component | Blue Stack | Green Stack | Shared |
|-----------|------------|-------------|--------|
| Frontend | `mvp-frontend-blue` | `mvp-frontend-green` | - |
| Backend | `mvp-backend-blue` | `mvp-backend-green` | - |
| PostgreSQL | - | - | `mvp-postgres` |
| Redis | - | - | `mvp-redis` |
| Traefik | - | - | `mvp-traefik` |
### Traffic Routing
Traefik uses weighted services for traffic distribution:
```yaml
# config/traefik/dynamic/blue-green.yml
services:
mvp-frontend-weighted:
weighted:
services:
- name: mvp-frontend-blue-svc
weight: 100 # Active
- name: mvp-frontend-green-svc
weight: 0 # Standby
```
### Deployment State
State is tracked in `config/deployment/state.json`:
```json
{
"active_stack": "blue",
"inactive_stack": "green",
"last_deployment": "2024-01-15T10:30:00Z",
"last_deployment_commit": "abc123",
"rollback_available": true
}
``` ```
--- ---
@@ -76,410 +191,316 @@ sudo -u gitlab-runner git clone <repository-url> /opt/motovaultpro
Navigate to **Settings > CI/CD > Variables** in your GitLab project. Navigate to **Settings > CI/CD > Variables** in your GitLab project.
### Secrets (File Type Variables) ### Required Variables
These variables use GitLab's **File** type, which writes the value to a temporary file and provides the path as the environment variable. This replicates the Kubernetes secrets pattern used by the application. | Variable | Type | Protected | Purpose |
|----------|------|-----------|---------|
| Variable Name | Type | Protected | Masked | Description | | `DEPLOY_NOTIFY_EMAIL` | Variable | Yes | Notification recipient |
|--------------|------|-----------|--------|-------------| | `VITE_AUTH0_DOMAIN` | Variable | No | Auth0 domain |
| `POSTGRES_PASSWORD` | File | Yes | Yes | PostgreSQL database password | | `VITE_AUTH0_CLIENT_ID` | Variable | No | Auth0 client ID |
| `AUTH0_CLIENT_SECRET` | File | Yes | Yes | Auth0 client secret for backend | | `VITE_AUTH0_AUDIENCE` | Variable | No | Auth0 audience |
| `GOOGLE_MAPS_API_KEY` | File | Yes | Yes | Google Maps API key |
| `GOOGLE_MAPS_MAP_ID` | File | Yes | No | Google Maps Map ID |
| `CF_DNS_API_TOKEN` | File | Yes | Yes | Cloudflare API token for Let's Encrypt DNS challenge |
| `RESEND_API_KEY` | File | Yes | Yes | Resend API key for email notifications |
### Configuration Variables
| Variable Name | Type | Protected | Masked | Value |
|--------------|------|-----------|--------|-------|
| `VITE_AUTH0_DOMAIN` | Variable | No | No | `motovaultpro.us.auth0.com` |
| `VITE_AUTH0_CLIENT_ID` | Variable | No | No | Your Auth0 client ID |
| `VITE_AUTH0_AUDIENCE` | Variable | No | No | `https://api.motovaultpro.com` |
Note: `DEPLOY_PATH` is automatically set in `.gitlab-ci.yml` using `GIT_CLONE_PATH` for a stable path.
### Creating Cloudflare API Token
The `CF_DNS_API_TOKEN` is required for automatic SSL certificate generation via Let's Encrypt DNS-01 challenge.
1. Go to [Cloudflare Dashboard](https://dash.cloudflare.com/profile/api-tokens)
2. Click **Create Token**
3. Use template: **Edit zone DNS**
4. Configure permissions:
- **Permissions**: Zone > DNS > Edit
- **Zone Resources**: Include > Specific zone > `motovaultpro.com`
5. Click **Continue to summary** then **Create Token**
6. Copy the token value immediately (it won't be shown again)
7. Add as `CF_DNS_API_TOKEN` File variable in GitLab
### Setting Up a File Type Variable
1. Go to **Settings > CI/CD > Variables**
2. Click **Add variable**
3. Enter the variable key (e.g., `POSTGRES_PASSWORD`)
4. Enter the secret value in the **Value** field
5. Set **Type** to **File**
6. Enable **Protect variable** (recommended)
7. Enable **Mask variable** (for sensitive data)
8. Click **Add variable**
---
## Secrets Architecture
MotoVaultPro uses a Kubernetes-style secrets pattern where secrets are mounted as files at `/run/secrets/` inside containers.
### How It Works
1. **GitLab stores secrets** as File type CI/CD variables
2. **During pipeline execution**, GitLab writes each secret to a temporary file
3. **The `inject-secrets.sh` script** copies these files to `secrets/app/` directory
4. **Docker Compose** mounts these files to `/run/secrets/` in containers
5. **Application code** reads secrets from the filesystem (not environment variables)
### Secret Files ### Secret Files
``` These use GitLab's **File** type and are injected via `scripts/inject-secrets.sh`:
secrets/app/
postgres-password.txt -> /run/secrets/postgres-password
auth0-client-secret.txt -> /run/secrets/auth0-client-secret
google-maps-api-key.txt -> /run/secrets/google-maps-api-key
google-maps-map-id.txt -> /run/secrets/google-maps-map-id
cloudflare-dns-token.txt -> /run/secrets/cloudflare-dns-token
resend-api-key.txt -> /run/secrets/resend-api-key
```
### Security Benefits | Variable | Type | Protected | Masked |
|----------|------|-----------|--------|
| `POSTGRES_PASSWORD` | File | Yes | Yes |
| `AUTH0_CLIENT_SECRET` | File | Yes | Yes |
| `GOOGLE_MAPS_API_KEY` | File | Yes | Yes |
| `GOOGLE_MAPS_MAP_ID` | File | Yes | No |
| `CF_DNS_API_TOKEN` | File | Yes | Yes |
| `RESEND_API_KEY` | File | Yes | Yes |
- Secrets never appear as environment variables (not visible in `env` or `printenv`) ### Registry Authentication
- File permissions (600) restrict access
- Masked variables prevent accidental log exposure GitLab provides these automatically:
- Protected variables only available on protected branches - `CI_REGISTRY_USER` - Registry username
- `CI_REGISTRY_PASSWORD` - Registry token
- `CI_REGISTRY` - Registry URL
--- ---
## Pipeline Overview ## Container Registry
The CI/CD pipeline consists of four stages: All images are hosted on the GitLab Container Registry to avoid Docker Hub rate limits.
### Stage 1: Validate ### Registry URL
Verifies deployment prerequisites:
- Docker is accessible
- Docker Compose is available
- Deployment directory exists
### Stage 2: Build
Builds Docker images:
- Pulls latest code from repository
- Builds all service images with `--no-cache`
### Stage 3: Deploy
Deploys the application:
1. Injects secrets from GitLab variables
2. Stops existing services gracefully
3. Pulls base images
4. Starts database services (PostgreSQL, Redis)
5. Runs database migrations
6. Starts all services
### Stage 4: Verify
Validates deployment health:
- Checks all containers are running
- Tests backend health endpoint
- Reports deployment status
### Pipeline Diagram
``` ```
[Validate] -> [Build] -> [Deploy] -> [Verify] registry.motovaultpro.com
| | | |
Check Build Inject Health
prereqs images secrets checks
|
Migrate
|
Start
services
``` ```
### Image Paths
| Image | Path |
|-------|------|
| Backend | `registry.motovaultpro.com/motovaultpro/backend:$TAG` |
| Frontend | `registry.motovaultpro.com/motovaultpro/frontend:$TAG` |
| Mirrors | `registry.motovaultpro.com/mirrors/` |
### Base Image Mirrors
Mirror upstream images to avoid rate limits:
```bash
# Run manually or via scheduled pipeline
./scripts/ci/mirror-base-images.sh
```
Mirrored images:
- `node:20-alpine`
- `nginx:alpine`
- `postgres:18-alpine`
- `redis:8.4-alpine`
- `traefik:v3.6`
- `docker:24.0`
- `docker:24.0-dind`
--- ---
## Deployment Process ## Deployment Process
### Automatic Deployment ### Automatic Deployment
Deployments are triggered automatically when: Deployments trigger automatically on push to `main`:
- Code is pushed to the `main` branch
- A merge request is merged into `main` 1. **Validate**: Check prerequisites, determine target stack
2. **Build**: Build images on dedicated build server
3. **Prepare**: Start inactive stack, run health checks
4. **Switch**: Update Traefik weights (instant)
5. **Verify**: External health check
6. **Notify**: Send email notification
### Manual Deployment ### Manual Deployment
To trigger a manual deployment:
1. Go to **CI/CD > Pipelines** 1. Go to **CI/CD > Pipelines**
2. Click **Run pipeline** 2. Click **Run pipeline**
3. Select the `main` branch 3. Select `main` branch
4. Click **Run pipeline** 4. Click **Run pipeline**
### Deployment Steps (What Happens) ### Deployment Timeline
1. **Secrets Injection** | Phase | Duration |
- `inject-secrets.sh` copies GitLab File variables to `secrets/app/` |-------|----------|
- Permissions are set to 600 for security | Validate | ~5s |
| Build | ~2 min |
2. **Service Shutdown** | Deploy-prepare | ~30s |
- Existing containers are stopped gracefully (30s timeout) | Deploy-switch | ~3s |
- Volumes are preserved | Verify | ~30s |
| **Total** | ~3 min |
3. **Database Startup**
- PostgreSQL and Redis start first
- 15-second wait for database readiness
4. **Migrations**
- Backend container runs database migrations
- Ensures schema is up-to-date
5. **Full Service Startup**
- All services start via `docker compose up -d`
- Traefik routes traffic automatically
6. **Health Verification**
- Container status checks
- Backend health endpoint validation
--- ---
## Rollback Procedure ## Rollback Procedures
### Automatic Rollback ### Automatic Rollback
If the verify stage fails, the pipeline will report failure but services remain running. Manual intervention is required. Triggers automatically when:
- Health check fails in `deploy-prepare`
- `verify` stage fails after switch
- Container becomes unhealthy within verification period
The pipeline runs `scripts/ci/auto-rollback.sh` which:
1. Verifies previous stack is healthy
2. Switches traffic back
3. Sends notification
### Manual Rollback ### Manual Rollback
Use the rollback script: SSH to production server:
```bash ```bash
# SSH to server
ssh user@server
# Run rollback to previous commit
cd /opt/motovaultpro cd /opt/motovaultpro
./scripts/rollback.sh HEAD~1
# Or rollback to specific tag/commit # Check current state
./scripts/rollback.sh v1.0.0 cat config/deployment/state.json | jq .
# Switch to other stack
./scripts/ci/switch-traffic.sh blue # or green
``` ```
### Rollback Script Details
The script performs:
1. Stops all current services
2. Checks out the specified version
3. Rebuilds images
4. Starts services
### Emergency Recovery ### Emergency Recovery
If rollback fails: If both stacks are unhealthy:
```bash
# Stop everything
docker compose -f docker-compose.yml -f docker-compose.blue-green.yml down
# Restart shared services
docker compose up -d mvp-postgres mvp-redis mvp-traefik
# Wait for database
sleep 15
# Start one stack
export BACKEND_IMAGE=registry.motovaultpro.com/motovaultpro/backend:latest
export FRONTEND_IMAGE=registry.motovaultpro.com/motovaultpro/frontend:latest
docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d \
mvp-frontend-blue mvp-backend-blue
# Switch traffic
./scripts/ci/switch-traffic.sh blue
```
---
## Maintenance Migrations
For breaking database changes requiring downtime:
### Via Pipeline (Recommended)
1. Go to **CI/CD > Pipelines**
2. Find the `maintenance-migration` job
3. Click **Play** to trigger manually
### Via Script
```bash ```bash
cd /opt/motovaultpro cd /opt/motovaultpro
# Stop everything # With backup
docker compose down ./scripts/ci/maintenance-migrate.sh backup
# Check git history # Without backup
git log --oneline -10 ./scripts/ci/maintenance-migrate.sh
```
# Checkout known working version ### What Happens
git checkout <commit-hash>
# Rebuild and start 1. Sends maintenance notification
docker compose build 2. Enables maintenance mode (stops traffic)
docker compose up -d 3. Creates database backup (if requested)
4. Runs migrations
5. Restarts backends
6. Restores traffic
7. Sends completion notification
# Verify ---
docker compose ps
## Notifications
Email notifications via Resend API for:
| Event | Subject |
|-------|---------|
| `success` | Deployment Successful |
| `failure` | Deployment Failed |
| `rollback` | Auto-Rollback Executed |
| `rollback_failed` | CRITICAL: Rollback Failed |
| `maintenance_start` | Maintenance Mode Started |
| `maintenance_end` | Maintenance Complete |
Configure recipient in GitLab CI/CD variables:
```
DEPLOY_NOTIFY_EMAIL = admin@example.com
``` ```
--- ---
## Troubleshooting ## Troubleshooting
### Pipeline Fails at Validate Stage
**Symptom**: `DEPLOY_PATH not found`
**Solution**:
```bash
# Create directory on runner server
sudo mkdir -p /opt/motovaultpro
sudo chown gitlab-runner:gitlab-runner /opt/motovaultpro
```
### Pipeline Fails at Build Stage ### Pipeline Fails at Build Stage
**Symptom**: Docker build errors **Check build server connectivity:**
**Solutions**:
1. Check Dockerfile syntax
2. Verify network connectivity for npm/package downloads
3. Check disk space: `df -h`
4. Clear Docker cache: `docker system prune -a`
### Pipeline Fails at Deploy Stage
**Symptom**: Secrets injection fails
**Solutions**:
1. Verify CI/CD variables are configured correctly
2. Check variable types are set to **File** for secrets
3. Ensure variables are not restricted to specific environments
**Symptom**: Migration fails
**Solutions**:
1. Check database connectivity
2. Verify PostgreSQL is healthy: `docker logs mvp-postgres`
3. Run migrations manually:
```bash ```bash
docker compose exec mvp-backend npm run migrate # On build server
sudo gitlab-runner verify
docker login registry.motovaultpro.com
``` ```
### Pipeline Fails at Verify Stage **Check disk space:**
**Symptom**: Container not running
**Solutions**:
1. Check container logs: `docker logs <container-name>`
2. Verify secrets are correctly mounted
3. Check for port conflicts
**Symptom**: Health check fails
**Solutions**:
1. Wait longer (service might be starting)
2. Check backend logs: `docker logs mvp-backend`
3. Verify database connection
### Services Start But Application Doesn't Work
**Check secrets are mounted**:
```bash ```bash
docker compose exec mvp-backend ls -la /run/secrets/ df -h
docker system prune -af
``` ```
**Check configuration**: ### Pipeline Fails at Deploy-Prepare
**Container won't start:**
```bash ```bash
docker compose exec mvp-backend cat /app/config/production.yml docker logs mvp-backend-blue --tail 100
docker logs mvp-frontend-blue --tail 100
``` ```
**Check network connectivity**: **Health check timeout:**
```bash ```bash
docker network ls # Increase timeout in .gitlab-ci.yml
docker network inspect motovaultpro_backend HEALTH_CHECK_TIMEOUT: "90"
``` ```
### Viewing Logs ### Traffic Not Switching
**Check Traefik config:**
```bash ```bash
# All services cat config/traefik/dynamic/blue-green.yml
docker compose logs -f docker exec mvp-traefik traefik healthcheck
# Specific service
docker compose logs -f mvp-backend
# Last 100 lines
docker compose logs --tail 100 mvp-backend
``` ```
--- **Check routing:**
## Maintenance
### Updating Secrets
1. Update the CI/CD variable in GitLab
2. Trigger a new pipeline (push or manual)
3. The new secrets will be injected during deployment
### Database Backups
Backups should be configured separately. Recommended approach:
```bash ```bash
# Manual backup curl -I https://motovaultpro.com/api/health
docker compose exec mvp-postgres pg_dump -U postgres motovaultpro > backup.sql
# Automated backup (add to cron)
0 2 * * * cd /opt/motovaultpro && docker compose exec -T mvp-postgres pg_dump -U postgres motovaultpro > /backups/mvp-$(date +\%Y\%m\%d).sql
``` ```
### Monitoring ### Verify Stage Fails
Consider adding: **Check external connectivity:**
- Prometheus metrics (Traefik already configured) ```bash
- Health check alerts curl -sf https://motovaultpro.com/api/health
- Log aggregation ```
**Check container health:**
```bash
docker inspect --format='{{.State.Health.Status}}' mvp-backend-blue
```
--- ---
## Quick Reference ## Quick Reference
### Common Commands
```bash
# View pipeline status
# GitLab UI: CI/CD > Pipelines
# SSH to server
ssh user@your-server
# Navigate to project
cd /opt/motovaultpro
# View running containers
docker compose ps
# View logs
docker compose logs -f
# Restart a service
docker compose restart mvp-backend
# Run migrations manually
docker compose exec mvp-backend npm run migrate
# Access database
docker compose exec mvp-postgres psql -U postgres motovaultpro
# Health check
curl http://localhost:3001/health
```
### Important Paths ### Important Paths
| Path | Description | | Path | Description |
|------|-------------| |------|-------------|
| `$CI_BUILDS_DIR/motovaultpro` | Application root (stable clone path) | | `config/deployment/state.json` | Deployment state |
| `$CI_BUILDS_DIR/motovaultpro/secrets/app/` | Secrets directory | | `config/traefik/dynamic/blue-green.yml` | Traffic routing |
| `$CI_BUILDS_DIR/motovaultpro/data/documents/` | Document storage | | `scripts/ci/` | Deployment scripts |
| `$CI_BUILDS_DIR/motovaultpro/config/` | Configuration files |
Note: `CI_BUILDS_DIR` is typically `/opt/gitlab-runner/builds` for shell executors. ### Common Commands
### Container Names ```bash
# View current state
cat config/deployment/state.json | jq .
| Container | Purpose | # Check container status
|-----------|---------| docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Health}}"
| `mvp-traefik` | Reverse proxy, TLS termination |
| `mvp-frontend` | React SPA | # View logs
| `mvp-backend` | Node.js API | docker logs mvp-backend-blue -f
| `mvp-postgres` | PostgreSQL database |
| `mvp-redis` | Redis cache | # Manual traffic switch
./scripts/ci/switch-traffic.sh green
# Run health check
./scripts/ci/health-check.sh blue
# Send test notification
./scripts/ci/notify.sh success "Test message"
```
### Memory Budget (8GB Server)
| Component | RAM |
|-----------|-----|
| Blue frontend | 512MB |
| Blue backend | 1GB |
| Green frontend | 512MB |
| Green backend | 1GB |
| PostgreSQL | 2GB |
| Redis | 512MB |
| Traefik | 128MB |
| System | 1.3GB |
| **Total** | ~7GB |

8
frontend/Dockerfile
View File

@@ -1,7 +1,11 @@
# Production Dockerfile for MotoVaultPro Frontend # Production Dockerfile for MotoVaultPro Frontend
# Uses mirrored base images from GitLab Container Registry
# Build argument for registry (defaults to GitLab mirrors, falls back to Docker Hub)
ARG REGISTRY_MIRRORS=registry.motovaultpro.com/mirrors
# Stage 1: Base with dependencies # Stage 1: Base with dependencies
FROM node:lts-alpine AS base FROM ${REGISTRY_MIRRORS}/node:20-alpine AS base
RUN apk add --no-cache dumb-init curl RUN apk add --no-cache dumb-init curl
WORKDIR /app WORKDIR /app
COPY package*.json ./ COPY package*.json ./
@@ -29,7 +33,7 @@ COPY . .
RUN npm run build RUN npm run build
# Stage 4: Production stage with nginx # Stage 4: Production stage with nginx
FROM nginx:alpine AS production FROM ${REGISTRY_MIRRORS}/nginx:alpine AS production
# Add curl for healthchecks # Add curl for healthchecks
RUN apk add --no-cache curl RUN apk add --no-cache curl

152
scripts/ci/auto-rollback.sh Executable file
View File

@@ -0,0 +1,152 @@
#!/bin/bash
# Auto-rollback script for blue-green deployment
# Reverts traffic to the previous healthy stack
#
# Usage: ./auto-rollback.sh [reason]
# reason: Optional description of why rollback is happening
#
# Exit codes:
# 0 - Rollback successful
# 1 - Rollback failed
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
REASON="${1:-Automatic rollback triggered}"
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
SWITCH_SCRIPT="$SCRIPT_DIR/switch-traffic.sh"
HEALTH_SCRIPT="$SCRIPT_DIR/health-check.sh"
NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh"
echo "========================================"
echo "AUTO-ROLLBACK INITIATED"
echo "Reason: $REASON"
echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
echo "========================================"
# Determine current and rollback stacks
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
CURRENT_STACK=$(jq -r '.active_stack // "unknown"' "$STATE_FILE")
ROLLBACK_STACK=$(jq -r '.inactive_stack // "unknown"' "$STATE_FILE")
else
echo "ERROR: Cannot determine current stack state"
echo "State file: $STATE_FILE"
exit 1
fi
if [[ "$CURRENT_STACK" == "unknown" ]] || [[ "$ROLLBACK_STACK" == "unknown" ]]; then
echo "ERROR: Invalid stack state"
echo " Current: $CURRENT_STACK"
echo " Rollback target: $ROLLBACK_STACK"
exit 1
fi
echo ""
echo "Stack Status:"
echo " Currently active: $CURRENT_STACK"
echo " Rollback target: $ROLLBACK_STACK"
echo ""
# Verify rollback stack is healthy before switching
echo "Step 1/3: Verifying rollback stack health..."
echo "----------------------------------------"
if [[ -x "$HEALTH_SCRIPT" ]]; then
if ! "$HEALTH_SCRIPT" "$ROLLBACK_STACK" 30; then
echo ""
echo "CRITICAL: Rollback stack ($ROLLBACK_STACK) is NOT healthy!"
echo "Manual intervention required."
echo ""
echo "Troubleshooting steps:"
echo " 1. Check container logs: docker logs mvp-backend-$ROLLBACK_STACK"
echo " 2. Check container status: docker ps -a"
echo " 3. Consider restarting rollback stack"
echo ""
# Send critical notification
if [[ -x "$NOTIFY_SCRIPT" ]]; then
"$NOTIFY_SCRIPT" "rollback_failed" \
"Rollback to $ROLLBACK_STACK failed - stack unhealthy. Manual intervention required. Reason: $REASON" \
|| true
fi
exit 1
fi
echo " OK: Rollback stack is healthy"
else
echo " WARNING: Health check script not found, proceeding anyway"
fi
# Switch traffic to rollback stack
echo ""
echo "Step 2/3: Switching traffic to $ROLLBACK_STACK..."
echo "----------------------------------------"
if [[ -x "$SWITCH_SCRIPT" ]]; then
if ! "$SWITCH_SCRIPT" "$ROLLBACK_STACK" instant; then
echo "ERROR: Traffic switch failed"
if [[ -x "$NOTIFY_SCRIPT" ]]; then
"$NOTIFY_SCRIPT" "rollback_failed" \
"Rollback traffic switch failed. Manual intervention required. Reason: $REASON" \
|| true
fi
exit 1
fi
else
echo "ERROR: Traffic switch script not found: $SWITCH_SCRIPT"
exit 1
fi
# Update state file with rollback info
echo ""
echo "Step 3/3: Updating deployment state..."
echo "----------------------------------------"
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg stack "$ROLLBACK_STACK" \
--arg reason "$REASON" \
--arg ts "$TIMESTAMP" \
--arg failed "$CURRENT_STACK" \
'.active_stack = $stack |
.inactive_stack = $failed |
.last_rollback = $ts |
.last_rollback_reason = $reason |
.rollback_available = false' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
echo " State updated"
fi
# Send notification
if [[ -x "$NOTIFY_SCRIPT" ]]; then
echo ""
echo "Sending rollback notification..."
"$NOTIFY_SCRIPT" "rollback" \
"Rollback executed. Traffic switched from $CURRENT_STACK to $ROLLBACK_STACK. Reason: $REASON" \
|| echo " WARNING: Notification failed"
fi
echo ""
echo "========================================"
echo "ROLLBACK COMPLETE"
echo "========================================"
echo ""
echo "Summary:"
echo " Previous stack: $CURRENT_STACK (now inactive)"
echo " Current stack: $ROLLBACK_STACK (now active)"
echo " Reason: $REASON"
echo ""
echo "Next steps:"
echo " 1. Investigate why $CURRENT_STACK failed"
echo " 2. Check logs: docker logs mvp-backend-$CURRENT_STACK"
echo " 3. Fix issues before next deployment"
echo ""
exit 0

185
scripts/ci/health-check.sh Executable file
View File

@@ -0,0 +1,185 @@
#!/bin/bash
# Health check script for blue-green deployment
# Verifies container health and HTTP endpoints
#
# Usage: ./health-check.sh <stack> [timeout_seconds]
# stack: blue or green
# timeout_seconds: max wait time (default: 60)
#
# Exit codes:
# 0 - All health checks passed
# 1 - Health check failed
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
STACK="${1:-}"
TIMEOUT="${2:-60}"
if [[ -z "$STACK" ]] || [[ ! "$STACK" =~ ^(blue|green)$ ]]; then
echo "Usage: $0 <blue|green> [timeout_seconds]"
exit 1
fi
FRONTEND_CONTAINER="mvp-frontend-$STACK"
BACKEND_CONTAINER="mvp-backend-$STACK"
echo "========================================"
echo "Health Check - $STACK Stack"
echo "Timeout: ${TIMEOUT}s"
echo "========================================"
# Function to check Docker container health
check_container_health() {
local container="$1"
local status
status=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "not found")
case "$status" in
"healthy")
return 0
;;
"starting")
return 2 # Still starting
;;
"unhealthy"|"not found"|"")
return 1
;;
*)
return 1
;;
esac
}
# Function to check HTTP endpoint
check_http_endpoint() {
local container="$1"
local port="$2"
local path="$3"
if docker exec "$container" curl -sf "http://localhost:${port}${path}" > /dev/null 2>&1; then
return 0
else
return 1
fi
}
# Function to check database connectivity via backend
check_database_connectivity() {
local container="$1"
# The /health endpoint should verify database connectivity
if docker exec "$container" curl -sf "http://localhost:3001/health" 2>/dev/null | grep -q '"database"'; then
return 0
else
return 1
fi
}
# Wait for containers to be healthy
wait_for_health() {
local container="$1"
local elapsed=0
while [[ $elapsed -lt $TIMEOUT ]]; do
check_container_health "$container"
local status=$?
if [[ $status -eq 0 ]]; then
return 0
elif [[ $status -eq 1 ]]; then
echo " ERROR: Container $container is unhealthy"
docker logs "$container" --tail 20 2>/dev/null || true
return 1
fi
# Still starting, wait
sleep 2
elapsed=$((elapsed + 2))
echo " Waiting for $container... (${elapsed}s/${TIMEOUT}s)"
done
echo " ERROR: Timeout waiting for $container"
return 1
}
# Main health check sequence
echo ""
echo "Step 1/4: Checking container status..."
echo "----------------------------------------"
for container in "$FRONTEND_CONTAINER" "$BACKEND_CONTAINER"; do
running=$(docker inspect --format='{{.State.Running}}' "$container" 2>/dev/null || echo "false")
if [[ "$running" != "true" ]]; then
echo " ERROR: Container $container is not running"
docker ps -a --filter "name=$container" --format "table {{.Names}}\t{{.Status}}"
exit 1
fi
echo " OK: $container is running"
done
echo ""
echo "Step 2/4: Waiting for Docker health checks..."
echo "----------------------------------------"
for container in "$FRONTEND_CONTAINER" "$BACKEND_CONTAINER"; do
echo " Checking $container..."
if ! wait_for_health "$container"; then
echo " FAILED: $container health check"
exit 1
fi
echo " OK: $container is healthy"
done
echo ""
echo "Step 3/4: Verifying HTTP endpoints..."
echo "----------------------------------------"
# Check frontend
echo " Checking frontend HTTP..."
if ! check_http_endpoint "$FRONTEND_CONTAINER" 3000 "/"; then
echo " FAILED: Frontend HTTP check"
exit 1
fi
echo " OK: Frontend responds on port 3000"
# Check backend health endpoint
echo " Checking backend HTTP..."
if ! check_http_endpoint "$BACKEND_CONTAINER" 3001 "/health"; then
echo " FAILED: Backend HTTP check"
exit 1
fi
echo " OK: Backend responds on port 3001"
echo ""
echo "Step 4/4: Verifying database connectivity..."
echo "----------------------------------------"
if ! check_database_connectivity "$BACKEND_CONTAINER"; then
echo " WARNING: Could not verify database connectivity"
echo " (Backend may not expose database status in /health)"
else
echo " OK: Database connectivity verified"
fi
echo ""
echo "========================================"
echo "Health Check PASSED - $STACK Stack"
echo "========================================"
# Update state file
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
if [[ -f "$STATE_FILE" ]]; then
# Update stack health status using jq if available
if command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg stack "$STACK" --arg ts "$TIMESTAMP" \
'.[$stack].healthy = true | .[$stack].last_health_check = $ts' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
fi
exit 0

220
scripts/ci/maintenance-migrate.sh Executable file
View File

@@ -0,0 +1,220 @@
#!/bin/bash
# Maintenance mode migration script
# Enables maintenance mode, runs migrations, then restores service
#
# Usage: ./maintenance-migrate.sh [backup]
# backup: If set, creates a database backup before migration
#
# This script is for BREAKING migrations that require downtime.
# Non-breaking migrations run automatically on container start.
#
# Exit codes:
# 0 - Migration successful
# 1 - Migration failed (maintenance mode will remain active)
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
CREATE_BACKUP="${1:-}"
COMPOSE_FILE="$PROJECT_ROOT/docker-compose.yml"
COMPOSE_BLUE_GREEN="$PROJECT_ROOT/docker-compose.blue-green.yml"
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
NOTIFY_SCRIPT="$SCRIPT_DIR/notify.sh"
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
BACKUP_DIR="$PROJECT_ROOT/data/backups"
echo "========================================"
echo "MAINTENANCE MODE MIGRATION"
echo "Time: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
echo "========================================"
# Determine active stack
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
else
ACTIVE_STACK="blue"
fi
BACKEND_CONTAINER="mvp-backend-$ACTIVE_STACK"
echo ""
echo "Configuration:"
echo " Active stack: $ACTIVE_STACK"
echo " Backend container: $BACKEND_CONTAINER"
echo " Create backup: ${CREATE_BACKUP:-no}"
echo ""
# Step 1: Send maintenance notification
echo "Step 1/6: Sending maintenance notification..."
echo "----------------------------------------"
if [[ -x "$NOTIFY_SCRIPT" ]]; then
"$NOTIFY_SCRIPT" "maintenance_start" "Starting maintenance window for database migration" || true
fi
echo " OK"
# Step 2: Enable maintenance mode
echo ""
echo "Step 2/6: Enabling maintenance mode..."
echo "----------------------------------------"
# Update state file
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
jq '.maintenance_mode = true' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
# Stop traffic to both stacks (weight 0)
echo " Stopping traffic to application stacks..."
# Set both stacks to weight 0 - Traefik will return 503
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 0/" "$TRAEFIK_CONFIG"
rm -f "${TRAEFIK_CONFIG}.bak"
# Wait for Traefik to pick up changes
sleep 3
echo " OK: Maintenance mode active"
# Step 3: Create database backup (if requested)
if [[ -n "$CREATE_BACKUP" ]]; then
echo ""
echo "Step 3/6: Creating database backup..."
echo "----------------------------------------"
mkdir -p "$BACKUP_DIR"
BACKUP_FILE="$BACKUP_DIR/pre-migration-$(date +%Y%m%d-%H%M%S).sql"
if docker exec mvp-postgres pg_dump -U postgres motovaultpro > "$BACKUP_FILE"; then
echo " OK: Backup created: $BACKUP_FILE"
ls -lh "$BACKUP_FILE"
else
echo " ERROR: Backup failed"
echo ""
echo "Aborting migration. Restoring traffic..."
# Restore traffic
if [[ "$ACTIVE_STACK" == "blue" ]]; then
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
else
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
fi
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
jq '.maintenance_mode = false' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
exit 1
fi
else
echo ""
echo "Step 3/6: Skipping backup (not requested)..."
echo "----------------------------------------"
echo " OK"
fi
# Step 4: Run migrations
echo ""
echo "Step 4/6: Running database migrations..."
echo "----------------------------------------"
# Flush Redis cache before migration
echo " Flushing Redis cache..."
docker exec mvp-redis redis-cli FLUSHALL > /dev/null 2>&1 || true
# Run migrations
echo " Running migrations..."
if docker exec "$BACKEND_CONTAINER" npm run migrate; then
echo " OK: Migrations completed"
else
echo " ERROR: Migration failed"
echo ""
echo "Migration failed. Maintenance mode remains active."
echo "To restore:"
echo " 1. Fix migration issues"
echo " 2. Re-run migrations: docker exec $BACKEND_CONTAINER npm run migrate"
echo " 3. Run: $0 to retry, or manually restore traffic"
if [[ -x "$NOTIFY_SCRIPT" ]]; then
"$NOTIFY_SCRIPT" "failure" "Database migration failed during maintenance window. Manual intervention required." || true
fi
exit 1
fi
# Step 5: Restart backend containers
echo ""
echo "Step 5/6: Restarting backend containers..."
echo "----------------------------------------"
# Restart both backends to pick up any schema changes
docker restart mvp-backend-blue mvp-backend-green 2>/dev/null || true
# Wait for backends to be healthy
echo " Waiting for backends to be healthy..."
sleep 10
for container in mvp-backend-blue mvp-backend-green; do
for i in {1..12}; do
health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown")
if [[ "$health" == "healthy" ]]; then
echo " OK: $container is healthy"
break
fi
if [[ $i -eq 12 ]]; then
echo " WARNING: $container health check timeout"
fi
sleep 5
done
done
# Step 6: Disable maintenance mode
echo ""
echo "Step 6/6: Restoring traffic..."
echo "----------------------------------------"
# Restore traffic to active stack
if [[ "$ACTIVE_STACK" == "blue" ]]; then
sed -i -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
sed -i -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
else
sed -i -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
sed -i -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 100/" "$TRAEFIK_CONFIG"
fi
# Update state file
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg ts "$TIMESTAMP" '.maintenance_mode = false | .last_migration = $ts' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
# Wait for Traefik to pick up changes
sleep 3
echo " OK: Traffic restored"
# Send completion notification
if [[ -x "$NOTIFY_SCRIPT" ]]; then
"$NOTIFY_SCRIPT" "maintenance_end" "Maintenance window complete. Database migration successful." || true
fi
echo ""
echo "========================================"
echo "MAINTENANCE MIGRATION COMPLETE"
echo "========================================"
echo ""
echo "Summary:"
echo " - Migrations: Successful"
echo " - Active stack: $ACTIVE_STACK"
echo " - Maintenance mode: Disabled"
if [[ -n "$CREATE_BACKUP" ]]; then
echo " - Backup: $BACKUP_FILE"
fi
echo ""
exit 0

84
scripts/ci/mirror-base-images.sh Executable file
View File

@@ -0,0 +1,84 @@
#!/bin/bash
# Mirror upstream Docker images to GitLab Container Registry
# Run manually or via scheduled GitLab pipeline
# This avoids Docker Hub rate limits and ensures build reliability
set -euo pipefail
REGISTRY="${REGISTRY:-registry.motovaultpro.com/mirrors}"
# Base images required by MotoVaultPro
IMAGES=(
"node:20-alpine"
"nginx:alpine"
"postgres:18-alpine"
"redis:8.4-alpine"
"traefik:v3.6"
"docker:24.0"
"docker:24.0-dind"
)
echo "========================================"
echo "Base Image Mirroring Script"
echo "Registry: $REGISTRY"
echo "========================================"
# Check if logged into registry
if ! docker info 2>/dev/null | grep -q "Username"; then
echo "WARNING: Not logged into Docker registry"
echo "Run: docker login registry.motovaultpro.com"
fi
FAILED=()
SUCCESS=()
for img in "${IMAGES[@]}"; do
echo ""
echo "Processing: $img"
echo "----------------------------------------"
# Pull from upstream
echo " Pulling from upstream..."
if ! docker pull "$img"; then
echo " ERROR: Failed to pull $img"
FAILED+=("$img")
continue
fi
# Tag for local registry
local_tag="$REGISTRY/$img"
echo " Tagging as: $local_tag"
docker tag "$img" "$local_tag"
# Push to local registry
echo " Pushing to registry..."
if ! docker push "$local_tag"; then
echo " ERROR: Failed to push $local_tag"
FAILED+=("$img")
continue
fi
SUCCESS+=("$img")
echo " OK: $img mirrored successfully"
done
echo ""
echo "========================================"
echo "Summary"
echo "========================================"
echo "Successful: ${#SUCCESS[@]}"
for img in "${SUCCESS[@]}"; do
echo " - $img"
done
if [ ${#FAILED[@]} -gt 0 ]; then
echo ""
echo "Failed: ${#FAILED[@]}"
for img in "${FAILED[@]}"; do
echo " - $img"
done
exit 1
fi
echo ""
echo "All images mirrored successfully"

195
scripts/ci/notify.sh Executable file
View File

@@ -0,0 +1,195 @@
#!/bin/bash
# Deployment notification script using Resend API
# Sends email notifications for deployment events
#
# Usage: ./notify.sh <event_type> [message] [commit_sha]
# event_type: success, failure, rollback, rollback_failed, maintenance_start, maintenance_end
# message: Optional custom message
# commit_sha: Optional commit SHA for context
#
# Required environment variables:
# DEPLOY_NOTIFY_EMAIL - Recipient email address
#
# Reads Resend API key from:
# /run/secrets/resend-api-key (container)
# ./secrets/app/resend-api-key.txt (local)
#
# Exit codes:
# 0 - Notification sent (or skipped if not configured)
# 1 - Notification failed
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
EVENT_TYPE="${1:-}"
MESSAGE="${2:-}"
COMMIT_SHA="${3:-${CI_COMMIT_SHORT_SHA:-unknown}}"
if [[ -z "$EVENT_TYPE" ]]; then
echo "Usage: $0 <success|failure|rollback|rollback_failed|maintenance_start|maintenance_end> [message] [commit_sha]"
exit 1
fi
# Get recipient email
NOTIFY_EMAIL="${DEPLOY_NOTIFY_EMAIL:-}"
if [[ -z "$NOTIFY_EMAIL" ]]; then
echo "DEPLOY_NOTIFY_EMAIL not set, skipping notification"
exit 0
fi
# Get Resend API key
RESEND_API_KEY=""
if [[ -f "/run/secrets/resend-api-key" ]]; then
RESEND_API_KEY=$(cat /run/secrets/resend-api-key)
elif [[ -f "$PROJECT_ROOT/secrets/app/resend-api-key.txt" ]]; then
RESEND_API_KEY=$(cat "$PROJECT_ROOT/secrets/app/resend-api-key.txt")
fi
if [[ -z "$RESEND_API_KEY" ]]; then
echo "WARNING: Resend API key not found, skipping notification"
exit 0
fi
# Determine subject and styling based on event type
case "$EVENT_TYPE" in
"success")
SUBJECT="Deployment Successful - MotoVaultPro"
STATUS_COLOR="#22c55e"
STATUS_EMOJI="[OK]"
STATUS_TEXT="Deployment Successful"
DEFAULT_MESSAGE="Version $COMMIT_SHA is now live."
;;
"failure")
SUBJECT="Deployment Failed - MotoVaultPro"
STATUS_COLOR="#ef4444"
STATUS_EMOJI="[FAILED]"
STATUS_TEXT="Deployment Failed"
DEFAULT_MESSAGE="Deployment of $COMMIT_SHA failed. Check pipeline logs."
;;
"rollback")
SUBJECT="Auto-Rollback Executed - MotoVaultPro"
STATUS_COLOR="#f59e0b"
STATUS_EMOJI="[ROLLBACK]"
STATUS_TEXT="Rollback Executed"
DEFAULT_MESSAGE="Automatic rollback was triggered. Previous version restored."
;;
"rollback_failed")
SUBJECT="CRITICAL: Rollback Failed - MotoVaultPro"
STATUS_COLOR="#dc2626"
STATUS_EMOJI="[CRITICAL]"
STATUS_TEXT="Rollback Failed"
DEFAULT_MESSAGE="Rollback attempt failed. Manual intervention required immediately."
;;
"maintenance_start")
SUBJECT="Maintenance Mode Started - MotoVaultPro"
STATUS_COLOR="#6366f1"
STATUS_EMOJI="[MAINTENANCE]"
STATUS_TEXT="Maintenance Mode Active"
DEFAULT_MESSAGE="Application is in maintenance mode for database migration."
;;
"maintenance_end")
SUBJECT="Maintenance Mode Ended - MotoVaultPro"
STATUS_COLOR="#22c55e"
STATUS_EMOJI="[ONLINE]"
STATUS_TEXT="Maintenance Complete"
DEFAULT_MESSAGE="Maintenance window complete. Application is online."
;;
*)
echo "Unknown event type: $EVENT_TYPE"
exit 1
;;
esac
# Use custom message or default
FINAL_MESSAGE="${MESSAGE:-$DEFAULT_MESSAGE}"
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
# Build HTML email
HTML_BODY=$(cat <<EOF
<!DOCTYPE html>
<html>
<head>
<style>
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #f3f4f6; padding: 20px; }
.container { max-width: 600px; margin: 0 auto; background: white; border-radius: 8px; overflow: hidden; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
.header { background: ${STATUS_COLOR}; color: white; padding: 20px; text-align: center; }
.header h1 { margin: 0; font-size: 24px; }
.content { padding: 30px; }
.status { font-size: 18px; font-weight: 600; margin-bottom: 15px; }
.message { color: #374151; line-height: 1.6; margin-bottom: 20px; }
.details { background: #f9fafb; border-radius: 6px; padding: 15px; font-size: 14px; }
.details-row { display: flex; justify-content: space-between; margin-bottom: 8px; }
.details-label { color: #6b7280; }
.details-value { color: #111827; font-weight: 500; }
.footer { padding: 20px; text-align: center; color: #6b7280; font-size: 12px; border-top: 1px solid #e5e7eb; }
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>MotoVaultPro</h1>
</div>
<div class="content">
<div class="status">${STATUS_EMOJI} ${STATUS_TEXT}</div>
<div class="message">${FINAL_MESSAGE}</div>
<div class="details">
<div class="details-row">
<span class="details-label">Environment:</span>
<span class="details-value">Production</span>
</div>
<div class="details-row">
<span class="details-label">Commit:</span>
<span class="details-value">${COMMIT_SHA}</span>
</div>
<div class="details-row">
<span class="details-label">Time:</span>
<span class="details-value">${TIMESTAMP}</span>
</div>
</div>
</div>
<div class="footer">
MotoVaultPro CI/CD Notification System
</div>
</div>
</body>
</html>
EOF
)
# Send email via Resend API
echo "Sending notification: $EVENT_TYPE"
echo " To: $NOTIFY_EMAIL"
echo " Subject: $SUBJECT"
# Build JSON payload
JSON_PAYLOAD=$(cat <<EOF
{
"from": "MotoVaultPro <deploy@motovaultpro.com>",
"to": ["$NOTIFY_EMAIL"],
"subject": "$SUBJECT",
"html": $(echo "$HTML_BODY" | jq -Rs .)
}
EOF
)
# Send via Resend API
RESPONSE=$(curl -s -w "\n%{http_code}" \
-X POST \
-H "Authorization: Bearer $RESEND_API_KEY" \
-H "Content-Type: application/json" \
-d "$JSON_PAYLOAD" \
"https://api.resend.com/emails")
HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
BODY=$(echo "$RESPONSE" | sed '$d')
if [[ "$HTTP_CODE" == "200" ]] || [[ "$HTTP_CODE" == "201" ]]; then
echo " OK: Notification sent successfully"
exit 0
else
echo " ERROR: Failed to send notification (HTTP $HTTP_CODE)"
echo " Response: $BODY"
exit 1
fi

157
scripts/ci/switch-traffic.sh Executable file
View File

@@ -0,0 +1,157 @@
#!/bin/bash
# Traffic switching script for blue-green deployment
# Updates Traefik weighted routing configuration
#
# Usage: ./switch-traffic.sh <target_stack> [mode]
# target_stack: blue or green (the stack to switch TO)
# mode: instant (default) or gradual
#
# Gradual mode: 25% -> 50% -> 75% -> 100% with 3s intervals
#
# Exit codes:
# 0 - Traffic switch successful
# 1 - Switch failed
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
TARGET_STACK="${1:-}"
MODE="${2:-instant}"
if [[ -z "$TARGET_STACK" ]] || [[ ! "$TARGET_STACK" =~ ^(blue|green)$ ]]; then
echo "Usage: $0 <blue|green> [instant|gradual]"
exit 1
fi
TRAEFIK_CONFIG="$PROJECT_ROOT/config/traefik/dynamic/blue-green.yml"
STATE_FILE="$PROJECT_ROOT/config/deployment/state.json"
if [[ ! -f "$TRAEFIK_CONFIG" ]]; then
echo "ERROR: Traefik config not found: $TRAEFIK_CONFIG"
exit 1
fi
echo "========================================"
echo "Traffic Switch"
echo "Target Stack: $TARGET_STACK"
echo "Mode: $MODE"
echo "========================================"
# Determine source and target weights
if [[ "$TARGET_STACK" == "blue" ]]; then
SOURCE_STACK="green"
else
SOURCE_STACK="blue"
fi
# Function to update weights in Traefik config
update_weights() {
local blue_weight="$1"
local green_weight="$2"
echo " Setting weights: blue=$blue_weight, green=$green_weight"
# Use sed to update weights in the YAML file
# Frontend blue weight
sed -i.bak -E "s/(name: mvp-frontend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG"
# Frontend green weight
sed -i.bak -E "s/(name: mvp-frontend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG"
# Backend blue weight
sed -i.bak -E "s/(name: mvp-backend-blue-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $blue_weight/" "$TRAEFIK_CONFIG"
# Backend green weight
sed -i.bak -E "s/(name: mvp-backend-green-svc[[:space:]]+weight:)[[:space:]]+[0-9]+/\1 $green_weight/" "$TRAEFIK_CONFIG"
# Clean up backup files
rm -f "${TRAEFIK_CONFIG}.bak"
# Traefik watches the file and reloads automatically
# Give it a moment to pick up changes
sleep 1
}
# Verify Traefik has picked up the changes
verify_traefik_reload() {
# Give Traefik time to reload config
sleep 2
# Check if Traefik is still healthy
if docker exec mvp-traefik traefik healthcheck > /dev/null 2>&1; then
echo " OK: Traefik config reloaded"
return 0
else
echo " WARNING: Could not verify Traefik health"
return 0 # Don't fail on this, file watcher is reliable
fi
}
if [[ "$MODE" == "gradual" ]]; then
echo ""
echo "Gradual traffic switch starting..."
echo "----------------------------------------"
if [[ "$TARGET_STACK" == "blue" ]]; then
STEPS=("25 75" "50 50" "75 25" "100 0")
else
STEPS=("75 25" "50 50" "25 75" "0 100")
fi
step=1
for weights in "${STEPS[@]}"; do
blue_w=$(echo "$weights" | cut -d' ' -f1)
green_w=$(echo "$weights" | cut -d' ' -f2)
echo ""
echo "Step $step/4: blue=$blue_w%, green=$green_w%"
update_weights "$blue_w" "$green_w"
verify_traefik_reload
if [[ $step -lt 4 ]]; then
echo " Waiting 3 seconds before next step..."
sleep 3
fi
step=$((step + 1))
done
else
# Instant switch
echo ""
echo "Instant traffic switch..."
echo "----------------------------------------"
if [[ "$TARGET_STACK" == "blue" ]]; then
update_weights 100 0
else
update_weights 0 100
fi
verify_traefik_reload
fi
# Update state file
echo ""
echo "Updating deployment state..."
echo "----------------------------------------"
if [[ -f "$STATE_FILE" ]] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg active "$TARGET_STACK" \
--arg inactive "$SOURCE_STACK" \
--arg ts "$TIMESTAMP" \
'.active_stack = $active | .inactive_stack = $inactive | .last_switch = $ts' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
echo " Active stack: $TARGET_STACK"
echo " Inactive stack: $SOURCE_STACK"
fi
echo ""
echo "========================================"
echo "Traffic Switch COMPLETE"
echo "All traffic now routed to: $TARGET_STACK"
echo "========================================"
exit 0