CI/CD Gitea v1.0
Some checks failed
Deploy to Staging / Build Images (push) Failing after 7s
Deploy to Staging / Deploy to Staging (push) Has been skipped
Deploy to Staging / Verify Staging (push) Has been skipped
Deploy to Staging / Notify Staging Ready (push) Has been skipped
Deploy to Staging / Notify Staging Failure (push) Failing after 6s

This commit is contained in:
Eric Gullickson
2025-12-29 18:51:41 -06:00
parent 9b0de6a5b8
commit 83d79da3aa
15 changed files with 1101 additions and 929 deletions

View File

@@ -0,0 +1,67 @@
# MotoVaultPro Maintenance Migration Workflow
# Manual trigger for breaking database migrations requiring downtime
name: Maintenance Migration
run-name: Maintenance Migration
on:
workflow_dispatch:
inputs:
create_backup:
description: 'Create database backup before migration'
required: true
default: 'yes'
type: choice
options:
- 'yes'
- 'no'
env:
DEPLOY_PATH: /opt/motovaultpro
jobs:
maintenance-migration:
name: Run Maintenance Migration
runs-on: mvp-prod
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Send maintenance start notification
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
./scripts/ci/notify.sh maintenance_start "Starting maintenance window for database migration"
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
- name: Run maintenance migration
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/maintenance-migrate.sh
if [ "${{ inputs.create_backup }}" = "yes" ]; then
./scripts/ci/maintenance-migrate.sh backup
else
./scripts/ci/maintenance-migrate.sh
fi
- name: Send maintenance complete notification
if: success()
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
./scripts/ci/notify.sh maintenance_end "Maintenance window complete. Database migration successful."
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
- name: Send maintenance failure notification
if: failure()
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
./scripts/ci/notify.sh failure "Maintenance migration FAILED. Manual intervention required."
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}

View File

@@ -0,0 +1,38 @@
# MotoVaultPro Base Image Mirroring Workflow
# Mirrors upstream Docker images to Gitea Package Registry
# Runs weekly on schedule or manual trigger
name: Mirror Base Images
run-name: Mirror Base Images to Registry
on:
schedule:
# Run every Sunday at 3:00 AM UTC
- cron: '0 3 * * 0'
workflow_dispatch:
env:
REGISTRY: git.motovaultpro.com
jobs:
mirror:
name: Mirror Base Images
runs-on: mvp-build
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to Gitea Container Registry
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USER }}" --password-stdin "$REGISTRY"
- name: Run mirror script
run: |
chmod +x scripts/ci/mirror-base-images.sh
REGISTRY=$REGISTRY/egullickson/mirrors ./scripts/ci/mirror-base-images.sh
- name: Report results
if: always()
run: |
echo "Base image mirroring complete"
echo "Mirrored images available at: $REGISTRY/egullickson/mirrors/"

View File

@@ -0,0 +1,260 @@
# MotoVaultPro Production Deployment Workflow
# Manual trigger only - run after verifying staging
# Blue-green deployment with auto-rollback
name: Deploy to Production
run-name: Production Deploy - ${{ inputs.image_tag || 'latest' }}
on:
workflow_dispatch:
inputs:
image_tag:
description: 'Image tag to deploy (defaults to latest)'
required: false
default: 'latest'
env:
REGISTRY: git.motovaultpro.com
DEPLOY_PATH: /opt/motovaultpro
COMPOSE_FILE: docker-compose.yml
COMPOSE_BLUE_GREEN: docker-compose.blue-green.yml
HEALTH_CHECK_TIMEOUT: "60"
jobs:
# ============================================
# VALIDATE - Determine target stack
# ============================================
validate:
name: Validate Prerequisites
runs-on: mvp-prod
outputs:
target_stack: ${{ steps.determine-stack.outputs.target_stack }}
backend_image: ${{ steps.set-images.outputs.backend_image }}
frontend_image: ${{ steps.set-images.outputs.frontend_image }}
steps:
- name: Check Docker availability
run: |
docker info > /dev/null 2>&1 || (echo "ERROR - Docker not accessible" && exit 1)
docker compose version > /dev/null 2>&1 || (echo "ERROR - Docker Compose not available" && exit 1)
- name: Check deployment path
run: test -d "$DEPLOY_PATH" || (echo "ERROR - DEPLOY_PATH not found" && exit 1)
- name: Login to Gitea Container Registry
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USER }}" --password-stdin "$REGISTRY"
- name: Set image tags
id: set-images
run: |
TAG="${{ inputs.image_tag }}"
echo "backend_image=$REGISTRY/egullickson/backend:$TAG" >> $GITHUB_OUTPUT
echo "frontend_image=$REGISTRY/egullickson/frontend:$TAG" >> $GITHUB_OUTPUT
- name: Determine target stack
id: determine-stack
run: |
STATE_FILE="$DEPLOY_PATH/config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
if [ "$ACTIVE_STACK" = "blue" ]; then
echo "target_stack=green" >> $GITHUB_OUTPUT
echo "Deploying to GREEN stack (BLUE is currently active)"
else
echo "target_stack=blue" >> $GITHUB_OUTPUT
echo "Deploying to BLUE stack (GREEN is currently active)"
fi
else
echo "target_stack=green" >> $GITHUB_OUTPUT
echo "No state file found, defaulting to GREEN stack"
fi
# ============================================
# DEPLOY PROD - Blue-green deployment
# ============================================
deploy-prod:
name: Deploy to Production
runs-on: mvp-prod
needs: validate
env:
TARGET_STACK: ${{ needs.validate.outputs.target_stack }}
BACKEND_IMAGE: ${{ needs.validate.outputs.backend_image }}
FRONTEND_IMAGE: ${{ needs.validate.outputs.frontend_image }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to registry
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USER }}" --password-stdin "$REGISTRY"
- name: Inject secrets
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/inject-secrets.sh
./scripts/inject-secrets.sh
env:
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
AUTH0_CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }}
AUTH0_MANAGEMENT_CLIENT_ID: ${{ secrets.AUTH0_MANAGEMENT_CLIENT_ID }}
AUTH0_MANAGEMENT_CLIENT_SECRET: ${{ secrets.AUTH0_MANAGEMENT_CLIENT_SECRET }}
GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
GOOGLE_MAPS_MAP_ID: ${{ secrets.GOOGLE_MAPS_MAP_ID }}
CF_DNS_API_TOKEN: ${{ secrets.CF_DNS_API_TOKEN }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
- name: Initialize data directories
run: |
cd "$DEPLOY_PATH"
sudo mkdir -p data/backups data/documents
sudo chown -R 1001:1001 data/backups data/documents
sudo chmod 755 data/backups data/documents
- name: Pull new images
run: |
docker pull $BACKEND_IMAGE
docker pull $FRONTEND_IMAGE
- name: Start target stack
run: |
cd "$DEPLOY_PATH"
export BACKEND_IMAGE=$BACKEND_IMAGE
export FRONTEND_IMAGE=$FRONTEND_IMAGE
docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \
mvp-frontend-$TARGET_STACK mvp-backend-$TARGET_STACK
- name: Wait for stack initialization
run: sleep 10
- name: Run health check
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/health-check.sh
./scripts/ci/health-check.sh $TARGET_STACK $HEALTH_CHECK_TIMEOUT
- name: Switch traffic
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/switch-traffic.sh
./scripts/ci/switch-traffic.sh $TARGET_STACK instant
- name: Update deployment state
run: |
cd "$DEPLOY_PATH"
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg commit "${{ inputs.image_tag }}" \
--arg ts "$TIMESTAMP" \
'.last_deployment = $ts | .last_deployment_commit = $commit | .last_deployment_status = "success" | .rollback_available = true' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
# ============================================
# VERIFY PROD - External health check
# ============================================
verify-prod:
name: Verify Production
runs-on: mvp-prod
needs: [validate, deploy-prod]
env:
TARGET_STACK: ${{ needs.validate.outputs.target_stack }}
steps:
- name: Wait for routing propagation
run: sleep 5
- name: External health check
run: |
for i in 1 2 3 4 5 6; do
if curl -sf https://motovaultpro.com/api/health > /dev/null 2>&1; then
echo "OK: Production external health check passed"
exit 0
fi
if [ $i -eq 6 ]; then
echo "ERROR: Production external health check failed after 6 attempts"
exit 1
fi
echo "Attempt $i/6: Waiting 10s..."
sleep 10
done
- name: Verify container status
run: |
for service in mvp-frontend-$TARGET_STACK mvp-backend-$TARGET_STACK; do
status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
health=$(docker inspect --format='{{.State.Health.Status}}' $service 2>/dev/null || echo "unknown")
if [ "$status" != "running" ] || [ "$health" != "healthy" ]; then
echo "ERROR: $service is not healthy (status: $status, health: $health)"
docker logs $service --tail 50 2>/dev/null || true
exit 1
fi
echo "OK: $service is running and healthy"
done
# ============================================
# ROLLBACK - Auto-rollback on failure
# ============================================
rollback:
name: Auto Rollback
runs-on: mvp-prod
needs: [validate, deploy-prod, verify-prod]
if: failure()
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Execute rollback
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/auto-rollback.sh
./scripts/ci/auto-rollback.sh "Production verification failed - automatic rollback"
- name: Update state
run: |
cd "$DEPLOY_PATH"
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
jq '.last_deployment_status = "rolled_back"' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
# ============================================
# NOTIFY SUCCESS
# ============================================
notify-success:
name: Notify Success
runs-on: mvp-prod
needs: [validate, verify-prod]
if: success()
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Send success notification
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
./scripts/ci/notify.sh success "Production deployment successful - ${{ inputs.image_tag }} is now live" ${{ inputs.image_tag }}
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
# ============================================
# NOTIFY FAILURE
# ============================================
notify-failure:
name: Notify Failure
runs-on: mvp-prod
needs: [validate, deploy-prod, verify-prod, rollback]
if: failure()
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Send failure notification
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
./scripts/ci/notify.sh failure "Production deployment failed for ${{ inputs.image_tag }}" ${{ inputs.image_tag }}
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}

View File

@@ -0,0 +1,234 @@
# MotoVaultPro Staging Deployment Workflow
# Triggers on push to main, builds and deploys to staging.motovaultpro.com
# After verification, sends notification with link to trigger production deploy
name: Deploy to Staging
run-name: Staging Deploy - ${{ gitea.sha }}
on:
push:
branches:
- main
env:
REGISTRY: git.motovaultpro.com
DEPLOY_PATH: /opt/motovaultpro
COMPOSE_FILE: docker-compose.yml
COMPOSE_STAGING: docker-compose.staging.yml
HEALTH_CHECK_TIMEOUT: "60"
jobs:
# ============================================
# BUILD - Build and push images
# ============================================
build:
name: Build Images
runs-on: mvp-build
outputs:
backend_image: ${{ steps.tags.outputs.backend_image }}
frontend_image: ${{ steps.tags.outputs.frontend_image }}
short_sha: ${{ steps.tags.outputs.short_sha }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to Gitea Container Registry
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USER }}" --password-stdin "$REGISTRY"
- name: Set image tags
id: tags
run: |
SHORT_SHA="${{ gitea.sha }}"
SHORT_SHA="${SHORT_SHA:0:7}"
echo "backend_image=$REGISTRY/egullickson/backend:$SHORT_SHA" >> $GITHUB_OUTPUT
echo "frontend_image=$REGISTRY/egullickson/frontend:$SHORT_SHA" >> $GITHUB_OUTPUT
echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
- name: Build backend image
run: |
docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg REGISTRY_MIRRORS=$REGISTRY/egullickson/mirrors \
--cache-from $REGISTRY/egullickson/backend:latest \
-t ${{ steps.tags.outputs.backend_image }} \
-t $REGISTRY/egullickson/backend:latest \
-f backend/Dockerfile \
.
- name: Build frontend image
run: |
docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg REGISTRY_MIRRORS=$REGISTRY/egullickson/mirrors \
--build-arg VITE_AUTH0_DOMAIN=${{ vars.VITE_AUTH0_DOMAIN }} \
--build-arg VITE_AUTH0_CLIENT_ID=${{ vars.VITE_AUTH0_CLIENT_ID }} \
--build-arg VITE_AUTH0_AUDIENCE=${{ vars.VITE_AUTH0_AUDIENCE }} \
--build-arg VITE_API_BASE_URL=/api \
--cache-from $REGISTRY/egullickson/frontend:latest \
-t ${{ steps.tags.outputs.frontend_image }} \
-t $REGISTRY/egullickson/frontend:latest \
-f frontend/Dockerfile \
frontend
- name: Push images
run: |
docker push ${{ steps.tags.outputs.backend_image }}
docker push ${{ steps.tags.outputs.frontend_image }}
docker push $REGISTRY/egullickson/backend:latest
docker push $REGISTRY/egullickson/frontend:latest
# ============================================
# DEPLOY STAGING - Deploy to staging server
# ============================================
deploy-staging:
name: Deploy to Staging
runs-on: mvp-build
needs: build
env:
BACKEND_IMAGE: ${{ needs.build.outputs.backend_image }}
FRONTEND_IMAGE: ${{ needs.build.outputs.frontend_image }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to registry
run: |
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login -u "${{ secrets.REGISTRY_USER }}" --password-stdin "$REGISTRY"
- name: Inject secrets
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/inject-secrets.sh
SECRETS_DIR="$DEPLOY_PATH/secrets/staging" ./scripts/inject-secrets.sh
env:
POSTGRES_PASSWORD: ${{ secrets.STAGING_POSTGRES_PASSWORD }}
AUTH0_CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }}
AUTH0_MANAGEMENT_CLIENT_ID: ${{ secrets.AUTH0_MANAGEMENT_CLIENT_ID }}
AUTH0_MANAGEMENT_CLIENT_SECRET: ${{ secrets.AUTH0_MANAGEMENT_CLIENT_SECRET }}
GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
GOOGLE_MAPS_MAP_ID: ${{ secrets.GOOGLE_MAPS_MAP_ID }}
CF_DNS_API_TOKEN: ${{ secrets.CF_DNS_API_TOKEN }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
- name: Initialize data directories
run: |
cd "$DEPLOY_PATH"
sudo mkdir -p data/backups data/documents
sudo chown -R 1001:1001 data/backups data/documents
sudo chmod 755 data/backups data/documents
- name: Pull new images
run: |
docker pull $BACKEND_IMAGE
docker pull $FRONTEND_IMAGE
- name: Deploy staging stack
run: |
cd "$DEPLOY_PATH"
export BACKEND_IMAGE=$BACKEND_IMAGE
export FRONTEND_IMAGE=$FRONTEND_IMAGE
docker compose -f $COMPOSE_FILE -f $COMPOSE_STAGING down --timeout 30 || true
docker compose -f $COMPOSE_FILE -f $COMPOSE_STAGING up -d
- name: Wait for services
run: sleep 15
# ============================================
# VERIFY STAGING - Health checks
# ============================================
verify-staging:
name: Verify Staging
runs-on: mvp-build
needs: [build, deploy-staging]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Check container status
run: |
for service in mvp-frontend-staging mvp-backend-staging mvp-postgres-staging mvp-redis-staging; do
status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
if [ "$status" != "running" ]; then
echo "ERROR: $service is not running (status: $status)"
docker logs $service --tail 50 2>/dev/null || true
exit 1
fi
echo "OK: $service is running"
done
- name: Wait for backend health
run: |
for i in 1 2 3 4 5 6; do
if docker exec mvp-backend-staging curl -sf http://localhost:3001/health > /dev/null 2>&1; then
echo "OK: Backend health check passed"
exit 0
fi
if [ $i -eq 6 ]; then
echo "ERROR: Backend health check failed after 6 attempts"
docker logs mvp-backend-staging --tail 100
exit 1
fi
echo "Attempt $i/6: Backend not ready, waiting 10s..."
sleep 10
done
- name: Check external endpoint
run: |
for i in 1 2 3 4 5 6; do
if curl -sf https://staging.motovaultpro.com/api/health > /dev/null 2>&1; then
echo "OK: Staging external health check passed"
exit 0
fi
if [ $i -eq 6 ]; then
echo "ERROR: Staging external health check failed after 6 attempts"
exit 1
fi
echo "Attempt $i/6: Waiting 10s..."
sleep 10
done
# ============================================
# NOTIFY - Staging ready for production
# ============================================
notify-staging-ready:
name: Notify Staging Ready
runs-on: mvp-build
needs: [build, verify-staging]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Send staging ready notification
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
./scripts/ci/notify.sh staging_ready \
"Staging verified for commit ${{ needs.build.outputs.short_sha }}. Review at https://staging.motovaultpro.com then trigger production deploy at https://git.motovaultpro.com/egullickson/motovaultpro/actions" \
${{ needs.build.outputs.short_sha }}
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
# ============================================
# NOTIFY FAILURE - Staging failed
# ============================================
notify-staging-failure:
name: Notify Staging Failure
runs-on: mvp-build
needs: [build, deploy-staging, verify-staging]
if: failure()
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Send failure notification
run: |
cd "$DEPLOY_PATH"
chmod +x scripts/ci/notify.sh
SHORT_SHA="${{ gitea.sha }}"
SHORT_SHA="${SHORT_SHA:0:7}"
./scripts/ci/notify.sh failure "Staging deployment failed for commit $SHORT_SHA" $SHORT_SHA
env:
DEPLOY_NOTIFY_EMAIL: ${{ vars.DEPLOY_NOTIFY_EMAIL }}
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}

View File

@@ -1,436 +0,0 @@
# MotoVaultPro GitLab CI/CD Pipeline - Blue-Green Deployment
# GitLab 18.6+ with separate build and production runners
# See docs/CICD-DEPLOY.md for complete documentation
# v2.0 - Blue-Green with Auto-Rollback
stages:
- validate
- build
- deploy-prepare
- deploy-switch
- verify
- rollback
- notify
variables:
# Registry configuration
REGISTRY: registry.motovaultpro.com
REGISTRY_MIRRORS: ${REGISTRY}/mirrors
IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}
BACKEND_IMAGE: ${REGISTRY}/motovaultpro/backend:${IMAGE_TAG}
FRONTEND_IMAGE: ${REGISTRY}/motovaultpro/frontend:${IMAGE_TAG}
# Deployment configuration
GIT_CLONE_PATH: ${CI_BUILDS_DIR}/motovaultpro
DEPLOY_PATH: ${CI_BUILDS_DIR}/motovaultpro
COMPOSE_FILE: docker-compose.yml
COMPOSE_BLUE_GREEN: docker-compose.blue-green.yml
# Health check configuration
HEALTH_CHECK_TIMEOUT: "60"
# Default after_script to fix permissions
default:
after_script:
- echo "Fixing file permissions..."
- sudo chown -R gitlab-runner:gitlab-runner "$DEPLOY_PATH" 2>/dev/null || true
- sudo chown -R 1001:1001 "$DEPLOY_PATH/data/backups" "$DEPLOY_PATH/data/documents" 2>/dev/null || true
# ============================================
# Stage 1: VALIDATE
# Check prerequisites before starting pipeline
# ============================================
validate:
stage: validate
tags:
- production
- shell
only:
- main
script:
- echo "=========================================="
- echo "Validating deployment prerequisites..."
- echo "=========================================="
- echo "Checking Docker..."
- docker info > /dev/null 2>&1 || (echo "ERROR - Docker not accessible" && exit 1)
- echo "OK - Docker is accessible"
- echo "Checking Docker Compose..."
- docker compose version > /dev/null 2>&1 || (echo "ERROR - Docker Compose not available" && exit 1)
- echo "OK - Docker Compose is available"
- echo "Checking deployment path..."
- test -d "$DEPLOY_PATH" || (echo "ERROR - DEPLOY_PATH not found" && exit 1)
- echo "OK - Deployment path exists"
- echo "Checking registry access..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY" || true
- echo "OK - Registry authentication configured"
- echo "Determining target stack..."
- |
STATE_FILE="$DEPLOY_PATH/config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
ACTIVE_STACK=$(jq -r '.active_stack // "blue"' "$STATE_FILE")
if [ "$ACTIVE_STACK" = "blue" ]; then
echo "TARGET_STACK=green" >> deploy.env
else
echo "TARGET_STACK=blue" >> deploy.env
fi
else
echo "TARGET_STACK=green" >> deploy.env
fi
cat deploy.env
- echo "=========================================="
- echo "Validation complete"
- echo "=========================================="
artifacts:
reports:
dotenv: deploy.env
# ============================================
# Stage 2: BUILD
# Build and push images to GitLab Container Registry
# Runs on dedicated build server (shell executor)
# ============================================
build:
stage: build
tags:
- build
only:
- main
script:
- echo "Authenticating with GitLab Container Registry..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
- echo "=========================================="
- echo "Building Docker images..."
- echo "Commit - ${CI_COMMIT_SHORT_SHA}"
- echo "Backend - ${BACKEND_IMAGE}"
- echo "Frontend - ${FRONTEND_IMAGE}"
- echo "=========================================="
# Build backend
- echo "Building backend..."
- |
docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--cache-from ${REGISTRY}/motovaultpro/backend:latest \
-t ${BACKEND_IMAGE} \
-t ${REGISTRY}/motovaultpro/backend:latest \
-f backend/Dockerfile \
.
# Build frontend
- echo "Building frontend..."
- |
docker build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg VITE_AUTH0_DOMAIN=${VITE_AUTH0_DOMAIN:-motovaultpro.us.auth0.com} \
--build-arg VITE_AUTH0_CLIENT_ID=${VITE_AUTH0_CLIENT_ID:-yspR8zdnSxmV8wFIghHynQ08iXAPoQJ3} \
--build-arg VITE_AUTH0_AUDIENCE=${VITE_AUTH0_AUDIENCE:-https://api.motovaultpro.com} \
--build-arg VITE_API_BASE_URL=/api \
--cache-from ${REGISTRY}/motovaultpro/frontend:latest \
-t ${FRONTEND_IMAGE} \
-t ${REGISTRY}/motovaultpro/frontend:latest \
-f frontend/Dockerfile \
frontend
# Push images
- echo "Pushing images to registry..."
- docker push ${BACKEND_IMAGE}
- docker push ${FRONTEND_IMAGE}
- docker push ${REGISTRY}/motovaultpro/backend:latest
- docker push ${REGISTRY}/motovaultpro/frontend:latest
- echo "=========================================="
- echo "Build complete"
- echo "=========================================="
# ============================================
# Stage 3: DEPLOY-PREPARE
# Pull images, start inactive stack, run health checks
# ============================================
deploy-prepare:
stage: deploy-prepare
tags:
- production
- shell
only:
- main
needs:
- job: validate
artifacts: true
- job: build
environment:
name: production
url: https://motovaultpro.com
script:
- echo "=========================================="
- echo "Preparing deployment to ${TARGET_STACK} stack..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Authenticate with registry
- echo "Authenticating with registry..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
# Inject secrets
- echo "Step 1/5 - Injecting secrets..."
- chmod +x scripts/inject-secrets.sh
- ./scripts/inject-secrets.sh
# Initialize data directories
- echo "Step 2/5 - Initializing data directories..."
- sudo mkdir -p data/backups data/documents
- sudo chown -R 1001:1001 data/backups data/documents
- sudo chmod 755 data/backups data/documents
# Pull new images
- echo "Step 3/5 - Pulling images..."
- docker pull ${BACKEND_IMAGE}
- docker pull ${FRONTEND_IMAGE}
# Start inactive stack
- echo "Step 4/5 - Starting ${TARGET_STACK} stack..."
- |
export BACKEND_IMAGE=${BACKEND_IMAGE}
export FRONTEND_IMAGE=${FRONTEND_IMAGE}
docker compose -f $COMPOSE_FILE -f $COMPOSE_BLUE_GREEN up -d \
mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}
# Wait for stack to be ready
- echo "Step 5/5 - Waiting for stack health..."
- sleep 10
# Run health check
- echo "Running health check on ${TARGET_STACK} stack..."
- chmod +x scripts/ci/health-check.sh
- ./scripts/ci/health-check.sh ${TARGET_STACK} ${HEALTH_CHECK_TIMEOUT}
# Update state with deployment info
- |
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg stack "$TARGET_STACK" \
--arg commit "$CI_COMMIT_SHORT_SHA" \
--arg ts "$TIMESTAMP" \
'.[$stack].version = $commit | .[$stack].commit = $commit | .[$stack].deployed_at = $ts | .[$stack].healthy = true' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
- echo "=========================================="
- echo "Deploy preparation complete"
- echo "=========================================="
# ============================================
# Stage 4: DEPLOY-SWITCH
# Switch traffic to new stack
# ============================================
deploy-switch:
stage: deploy-switch
tags:
- production
- shell
only:
- main
needs:
- job: validate
artifacts: true
- job: deploy-prepare
script:
- echo "=========================================="
- echo "Switching traffic to ${TARGET_STACK} stack..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Switch traffic
- chmod +x scripts/ci/switch-traffic.sh
- ./scripts/ci/switch-traffic.sh ${TARGET_STACK} instant
# Update state
- |
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
jq --arg commit "$CI_COMMIT_SHORT_SHA" \
--arg ts "$TIMESTAMP" \
'.last_deployment = $ts | .last_deployment_commit = $commit | .last_deployment_status = "success" | .rollback_available = true' \
"$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
- echo "=========================================="
- echo "Traffic switch complete"
- echo "=========================================="
# ============================================
# Stage 5: VERIFY
# Production health verification after switch
# ============================================
verify:
stage: verify
tags:
- production
- shell
only:
- main
needs:
- job: validate
artifacts: true
- job: deploy-switch
script:
- echo "=========================================="
- echo "Verifying production deployment..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Wait for Traefik to propagate routing
- echo "Waiting for traffic routing to stabilize..."
- sleep 5
# Verify via external endpoint
- echo "Checking external endpoint..."
- |
for i in 1 2 3 4 5 6; do
if curl -sf https://motovaultpro.com/api/health > /dev/null 2>&1; then
echo "OK - External health check passed"
break
fi
if [ $i -eq 6 ]; then
echo "ERROR - External health check failed after 6 attempts"
exit 1
fi
echo "Attempt $i/6 - Waiting 10s..."
sleep 10
done
# Verify container status
- echo "Checking container status..."
- |
for service in mvp-frontend-${TARGET_STACK} mvp-backend-${TARGET_STACK}; do
status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
health=$(docker inspect --format='{{.State.Health.Status}}' $service 2>/dev/null || echo "unknown")
if [ "$status" != "running" ] || [ "$health" != "healthy" ]; then
echo "ERROR - $service is not healthy (status: $status, health: $health)"
docker logs $service --tail 50 2>/dev/null || true
exit 1
fi
echo "OK - $service is running and healthy"
done
- echo "=========================================="
- echo "Deployment verified successfully!"
- echo "Version ${CI_COMMIT_SHORT_SHA} is now live"
- echo "=========================================="
# ============================================
# Stage 6: ROLLBACK (on failure)
# Automatic rollback if verify stage fails
# ============================================
rollback:
stage: rollback
tags:
- production
- shell
only:
- main
when: on_failure
needs:
- job: validate
artifacts: true
- job: deploy-switch
- job: verify
script:
- echo "=========================================="
- echo "INITIATING AUTO-ROLLBACK"
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Run rollback script
- chmod +x scripts/ci/auto-rollback.sh
- ./scripts/ci/auto-rollback.sh "Verify stage failed - automatic rollback"
# Update state
- |
STATE_FILE="config/deployment/state.json"
if [ -f "$STATE_FILE" ] && command -v jq &> /dev/null; then
jq '.last_deployment_status = "rolled_back"' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
fi
- echo "=========================================="
- echo "Rollback complete"
- echo "=========================================="
# ============================================
# Stage 7: NOTIFY
# Send deployment notifications
# ============================================
notify-success:
stage: notify
tags:
- production
- shell
only:
- main
needs:
- job: verify
script:
- echo "Sending success notification..."
- cd "$DEPLOY_PATH"
- chmod +x scripts/ci/notify.sh
- ./scripts/ci/notify.sh success "Version ${CI_COMMIT_SHORT_SHA} deployed successfully" ${CI_COMMIT_SHORT_SHA}
notify-failure:
stage: notify
tags:
- production
- shell
only:
- main
when: on_failure
needs:
- job: build
optional: true
- job: deploy-prepare
optional: true
- job: deploy-switch
optional: true
- job: verify
optional: true
script:
- echo "Sending failure notification..."
- cd "$DEPLOY_PATH"
- chmod +x scripts/ci/notify.sh
- ./scripts/ci/notify.sh failure "Deployment of ${CI_COMMIT_SHORT_SHA} failed" ${CI_COMMIT_SHORT_SHA}
# ============================================
# Manual Jobs
# ============================================
# Manual maintenance migration job
maintenance-migration:
stage: deploy-prepare
tags:
- production
- shell
only:
- main
when: manual
script:
- echo "=========================================="
- echo "MAINTENANCE MODE MIGRATION"
- echo "=========================================="
- cd "$DEPLOY_PATH"
- chmod +x scripts/ci/maintenance-migrate.sh
- ./scripts/ci/maintenance-migrate.sh backup
# Mirror base images (scheduled or manual)
mirror-images:
stage: build
tags:
- build
only:
- schedules
- web
when: manual
script:
- echo "Mirroring base images to GitLab Container Registry..."
- echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$REGISTRY"
- chmod +x scripts/ci/mirror-base-images.sh
- REGISTRY=${REGISTRY}/mirrors ./scripts/ci/mirror-base-images.sh

View File

@@ -1,8 +1,8 @@
# Production Dockerfile for MotoVaultPro Backend # Production Dockerfile for MotoVaultPro Backend
# Uses mirrored base images from GitLab Container Registry # Uses mirrored base images from Gitea Package Registry
# Build argument for registry (defaults to GitLab mirrors, falls back to Docker Hub) # Build argument for registry (defaults to Gitea mirrors, falls back to Docker Hub)
ARG REGISTRY_MIRRORS=registry.motovaultpro.com/mirrors ARG REGISTRY_MIRRORS=git.motovaultpro.com/egullickson/mirrors
# Stage 1: Build stage # Stage 1: Build stage
FROM ${REGISTRY_MIRRORS}/node:20-alpine AS builder FROM ${REGISTRY_MIRRORS}/node:20-alpine AS builder

View File

@@ -16,7 +16,7 @@ services:
# BLUE Stack - Frontend # BLUE Stack - Frontend
# ======================================== # ========================================
mvp-frontend-blue: mvp-frontend-blue:
image: ${FRONTEND_IMAGE:-registry.motovaultpro.com/motovaultpro/frontend:latest} image: ${FRONTEND_IMAGE:-git.motovaultpro.com/egullickson/frontend:latest}
container_name: mvp-frontend-blue container_name: mvp-frontend-blue
restart: unless-stopped restart: unless-stopped
environment: environment:
@@ -52,7 +52,7 @@ services:
# BLUE Stack - Backend # BLUE Stack - Backend
# ======================================== # ========================================
mvp-backend-blue: mvp-backend-blue:
image: ${BACKEND_IMAGE:-registry.motovaultpro.com/motovaultpro/backend:latest} image: ${BACKEND_IMAGE:-git.motovaultpro.com/egullickson/backend:latest}
container_name: mvp-backend-blue container_name: mvp-backend-blue
restart: unless-stopped restart: unless-stopped
environment: environment:
@@ -101,7 +101,7 @@ services:
# GREEN Stack - Frontend # GREEN Stack - Frontend
# ======================================== # ========================================
mvp-frontend-green: mvp-frontend-green:
image: ${FRONTEND_IMAGE:-registry.motovaultpro.com/motovaultpro/frontend:latest} image: ${FRONTEND_IMAGE:-git.motovaultpro.com/egullickson/frontend:latest}
container_name: mvp-frontend-green container_name: mvp-frontend-green
restart: unless-stopped restart: unless-stopped
environment: environment:
@@ -137,7 +137,7 @@ services:
# GREEN Stack - Backend # GREEN Stack - Backend
# ======================================== # ========================================
mvp-backend-green: mvp-backend-green:
image: ${BACKEND_IMAGE:-registry.motovaultpro.com/motovaultpro/backend:latest} image: ${BACKEND_IMAGE:-git.motovaultpro.com/egullickson/backend:latest}
container_name: mvp-backend-green container_name: mvp-backend-green
restart: unless-stopped restart: unless-stopped
environment: environment:

View File

@@ -0,0 +1,80 @@
# Staging Environment Docker Compose
# Runs full application stack on staging server (staging.motovaultpro.com)
# Usage: docker compose -f docker-compose.yml -f docker-compose.staging.yml up -d
#
# Differences from production:
# - Single stack (no blue-green)
# - Staging domain (staging.motovaultpro.com)
# - Separate database (isolated from production)
# - Uses same images as production for accurate testing
services:
# ========================================
# Traefik - Reverse Proxy (Staging)
# ========================================
mvp-traefik:
image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/traefik:v3.6
container_name: mvp-traefik-staging
labels:
- "traefik.http.routers.traefik-dashboard.rule=Host(`traefik.staging.motovaultpro.com`)"
# ========================================
# Frontend (Staging)
# ========================================
mvp-frontend:
image: ${FRONTEND_IMAGE:-git.motovaultpro.com/egullickson/frontend:latest}
container_name: mvp-frontend-staging
labels:
- "traefik.enable=true"
- "traefik.docker.network=motovaultpro_frontend"
- "traefik.http.routers.mvp-frontend.rule=Host(`staging.motovaultpro.com`) && !PathPrefix(`/api`)"
- "traefik.http.routers.mvp-frontend.entrypoints=websecure"
- "traefik.http.routers.mvp-frontend.tls=true"
- "traefik.http.routers.mvp-frontend.tls.certresolver=letsencrypt"
- "traefik.http.routers.mvp-frontend.priority=10"
- "traefik.http.services.mvp-frontend.loadbalancer.server.port=3000"
# ========================================
# Backend (Staging)
# ========================================
mvp-backend:
image: ${BACKEND_IMAGE:-git.motovaultpro.com/egullickson/backend:latest}
container_name: mvp-backend-staging
labels:
- "traefik.enable=true"
- "traefik.docker.network=motovaultpro_backend"
- "traefik.http.routers.mvp-backend.rule=Host(`staging.motovaultpro.com`) && PathPrefix(`/api`)"
- "traefik.http.routers.mvp-backend.entrypoints=websecure"
- "traefik.http.routers.mvp-backend.tls=true"
- "traefik.http.routers.mvp-backend.tls.certresolver=letsencrypt"
- "traefik.http.routers.mvp-backend.priority=20"
- "traefik.http.routers.mvp-backend-health.rule=Host(`staging.motovaultpro.com`) && Path(`/api/health`)"
- "traefik.http.routers.mvp-backend-health.entrypoints=websecure"
- "traefik.http.routers.mvp-backend-health.tls=true"
- "traefik.http.routers.mvp-backend-health.tls.certresolver=letsencrypt"
- "traefik.http.routers.mvp-backend-health.priority=30"
- "traefik.http.services.mvp-backend.loadbalancer.server.port=3001"
# ========================================
# PostgreSQL (Staging - Separate Database)
# ========================================
mvp-postgres:
container_name: mvp-postgres-staging
volumes:
- mvp_postgres_staging_data:/var/lib/postgresql/data
- ./secrets/staging/postgres-password.txt:/run/secrets/postgres-password:ro
# ========================================
# Redis (Staging)
# ========================================
mvp-redis:
container_name: mvp-redis-staging
volumes:
- mvp_redis_staging_data:/data
# Staging-specific volumes (separate from production)
volumes:
mvp_postgres_staging_data:
name: mvp_postgres_staging_data
mvp_redis_staging_data:
name: mvp_redis_staging_data

View File

@@ -1,11 +1,11 @@
# Base registry for mirrored images (override with environment variable) # Base registry for mirrored images (override with environment variable)
x-registry: &registry x-registry: &registry
REGISTRY_MIRRORS: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors} REGISTRY_MIRRORS: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}
services: services:
# Traefik - Service Discovery and Load Balancing # Traefik - Service Discovery and Load Balancing
mvp-traefik: mvp-traefik:
image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/traefik:v3.6 image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/traefik:v3.6
container_name: mvp-traefik container_name: mvp-traefik
restart: unless-stopped restart: unless-stopped
command: command:
@@ -158,7 +158,7 @@ services:
# Database Services - Application PostgreSQL # Database Services - Application PostgreSQL
mvp-postgres: mvp-postgres:
image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/postgres:18-alpine image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/postgres:18-alpine
container_name: mvp-postgres container_name: mvp-postgres
restart: unless-stopped restart: unless-stopped
environment: environment:
@@ -183,7 +183,7 @@ services:
# Database Services - Application Redis # Database Services - Application Redis
mvp-redis: mvp-redis:
image: ${REGISTRY_MIRRORS:-registry.motovaultpro.com/mirrors}/redis:8.4-alpine image: ${REGISTRY_MIRRORS:-git.motovaultpro.com/egullickson/mirrors}/redis:8.4-alpine
container_name: mvp-redis container_name: mvp-redis
restart: unless-stopped restart: unless-stopped
command: redis-server --appendonly yes command: redis-server --appendonly yes

View File

@@ -1,22 +1,28 @@
# Build Server Setup Guide # Build/Staging Server Setup Guide
Complete guide for setting up a dedicated build VPS for MotoVaultPro CI/CD pipeline. Complete guide for setting up the build and staging server for MotoVaultPro CI/CD with Gitea Actions.
## Overview ## Overview
The build server isolates resource-intensive Docker builds from the production server, ensuring deployments don't impact application performance. The build server serves dual purposes:
1. **Build Server**: Builds Docker images and pushes to Gitea Package Registry
2. **Staging Server**: Runs full application stack at staging.motovaultpro.com
``` ```
+-------------------+ +--------------------+ +-------------------+ +--------------------+
| GitLab Server | | Production Server | | Gitea Server | | Production Server |
| (CI/CD + Registry)| | (Shell Runner) | | git.motovaultpro | | (mvp-prod runner) |
+--------+----------+ +----------+---------+ | + Package Registry| +----------+---------+
| | +--------+----------+ |
v v | v
+--------+----------+ +----------+---------+ v motovaultpro.com
| Build VPS | | Blue-Green Stacks | +--------+----------+
| (Docker Runner) |---->| + Shared Data | | Build/Staging VPS |
+-------------------+ +--------------------+ | (mvp-build runner)|
+-------------------+
|
v
staging.motovaultpro.com
``` ```
## Server Requirements ## Server Requirements
@@ -25,16 +31,16 @@ The build server isolates resource-intensive Docker builds from the production s
| Resource | Requirement | | Resource | Requirement |
|----------|-------------| |----------|-------------|
| CPU | 2 cores | | CPU | 4 cores |
| RAM | 4GB | | RAM | 8GB |
| Storage | 50GB SSD | | Storage | 100GB SSD |
| Network | 100Mbps+ | | Network | 100Mbps+ |
| OS | Ubuntu 22.04 LTS / Debian 12 | | OS | Ubuntu 22.04 LTS / Debian 12 |
### Network Requirements ### Network Requirements
- Outbound HTTPS to GitLab instance - Port 80/443 open (for staging.motovaultpro.com)
- Outbound HTTPS to Docker registries (for fallback) - Outbound HTTPS to git.motovaultpro.com
- SSH access for administration - SSH access for administration
--- ---
@@ -45,7 +51,7 @@ The build server isolates resource-intensive Docker builds from the production s
```bash ```bash
sudo apt update && sudo apt upgrade -y sudo apt update && sudo apt upgrade -y
sudo apt install -y curl git ca-certificates gnupg sudo apt install -y curl git ca-certificates gnupg jq
``` ```
### 2. Install Docker Engine ### 2. Install Docker Engine
@@ -56,7 +62,7 @@ sudo install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg sudo chmod a+r /etc/apt/keyrings/docker.gpg
# Add the repository to Apt sources # Add the repository
echo \ echo \
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
@@ -71,95 +77,162 @@ docker --version
docker compose version docker compose version
``` ```
### 3. Install GitLab Runner ### 3. Install act_runner
```bash ```bash
# Add GitLab Runner repository # Download act_runner binary
curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash curl -L https://gitea.com/gitea/act_runner/releases/download/v0.2.12/act_runner-0.2.12-linux-amd64 -o /tmp/act_runner
sudo mv /tmp/act_runner /usr/local/bin/act_runner
# Install GitLab Runner sudo chmod +x /usr/local/bin/act_runner
sudo apt install gitlab-runner
# Verify installation # Verify installation
gitlab-runner --version act_runner --version
``` ```
### 4. Register Runner with Shell Executor ### 4. Create act_runner User
```bash ```bash
sudo gitlab-runner register \ # Create user for running act_runner
--non-interactive \ sudo useradd -r -m -s /bin/bash act_runner
--url "https://git.motovaultpro.com" \ sudo usermod -aG docker act_runner
--registration-token "YOUR_REGISTRATION_TOKEN" \
--executor "shell" \ # Create config directory
--description "Build Server - Shell Executor" \ sudo mkdir -p /etc/act_runner
--tag-list "build" \ sudo chown act_runner:act_runner /etc/act_runner
--run-untagged="false" \
--locked="true"
``` ```
**Notes:** ### 5. Register Runner with Gitea
- Replace `YOUR_REGISTRATION_TOKEN` with the token from GitLab Admin > CI/CD > Runners
- Shell executor runs jobs directly on the host with access to Docker
- Tag `build` is used in `.gitlab-ci.yml` to route build jobs to this server
### 5. Add gitlab-runner to Docker Group Get a registration token from: `git.motovaultpro.com/egullickson/motovaultpro/settings/actions/runners`
The gitlab-runner user needs access to Docker:
```bash ```bash
sudo usermod -aG docker gitlab-runner # Generate config
sudo -u act_runner act_runner generate-config > /etc/act_runner/config.yaml
# Verify access # Register runner with staging/build label
sudo -u gitlab-runner docker info sudo -u act_runner act_runner register --no-interactive \
sudo -u gitlab-runner docker compose version --instance https://git.motovaultpro.com \
--token <REGISTRATION_TOKEN> \
--name "Build/Staging Server" \
--labels "mvp-build:host"
``` ```
### 6. Configure Docker Registry Authentication ### 6. Create Systemd Service
Create credentials file for GitLab Container Registry:
```bash ```bash
# Login to GitLab Container Registry (creates ~/.docker/config.json) cat << 'EOF' | sudo tee /etc/systemd/system/act_runner.service
docker login registry.motovaultpro.com -u <deploy-token-username> -p <deploy-token> [Unit]
Description=Gitea Actions Runner
After=docker.service network.target
[Service]
ExecStart=/usr/local/bin/act_runner daemon --config /etc/act_runner/config.yaml
WorkingDirectory=/home/act_runner
User=act_runner
Group=act_runner
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF
# Enable and start
sudo systemctl daemon-reload
sudo systemctl enable act_runner --now
sudo systemctl status act_runner
``` ```
**Creating Deploy Token:** ---
1. Go to GitLab Project > Settings > Repository > Deploy Tokens
2. Create token with `read_registry` and `write_registry` scopes ## Staging Environment Setup
3. Use the token username/password for Docker login
### 1. Clone Repository
```bash
sudo mkdir -p /opt/motovaultpro
sudo chown act_runner:act_runner /opt/motovaultpro
sudo -u act_runner git clone https://git.motovaultpro.com/egullickson/motovaultpro.git /opt/motovaultpro
```
### 2. Create Staging Secrets Directory
```bash
sudo mkdir -p /opt/motovaultpro/secrets/staging
sudo chown -R act_runner:act_runner /opt/motovaultpro/secrets
sudo chmod 700 /opt/motovaultpro/secrets/staging
```
### 3. Configure DNS
Add DNS A record:
```
staging.motovaultpro.com -> <build-server-ip>
```
### 4. Configure Cloudflare (if using)
Ensure `staging.motovaultpro.com` is proxied through Cloudflare or has a valid SSL certificate configured.
### 5. Initialize Data Directories
```bash
cd /opt/motovaultpro
sudo mkdir -p data/backups data/documents
sudo chown -R 1001:1001 data/backups data/documents
```
---
## Docker Registry Authentication
### Login to Gitea Package Registry
```bash
# Login as act_runner user
sudo -u act_runner docker login git.motovaultpro.com -u egullickson
# Enter your Gitea access token when prompted
```
### Create Access Token
1. Go to `git.motovaultpro.com/user/settings/applications`
2. Create new token with scopes:
- `read:packages`
- `write:packages`
3. Save token securely
--- ---
## Verification ## Verification
### Test Runner Registration ### Check Runner Status
```bash ```bash
sudo gitlab-runner verify sudo systemctl status act_runner
``` ```
Expected output: ### Check Runner Registration
```
Verifying runner... is alive runner=XXXXXX Go to `git.motovaultpro.com/egullickson/motovaultpro/settings/actions/runners` and verify the runner appears as "Online".
```
### Test Docker Access ### Test Docker Access
```bash ```bash
sudo gitlab-runner exec docker --docker-privileged test-job sudo -u act_runner docker info
sudo -u act_runner docker compose version
``` ```
### Test Registry Push ### Test Registry Push
```bash ```bash
# Build and push a test image # Build and push a test image
docker build -t registry.motovaultpro.com/motovaultpro/test:latest -f- . <<EOF sudo -u act_runner docker build -t git.motovaultpro.com/egullickson/test:latest -f- . <<EOF
FROM alpine:latest FROM alpine:latest
RUN echo "test" RUN echo "test"
EOF EOF
docker push registry.motovaultpro.com/motovaultpro/test:latest sudo -u act_runner docker push git.motovaultpro.com/egullickson/test:latest
``` ```
--- ---
@@ -168,8 +241,6 @@ docker push registry.motovaultpro.com/motovaultpro/test:latest
### Disk Cleanup ### Disk Cleanup
Docker builds accumulate disk space. Set up automated cleanup:
```bash ```bash
# Create cleanup script # Create cleanup script
sudo tee /usr/local/bin/docker-cleanup.sh > /dev/null <<'EOF' sudo tee /usr/local/bin/docker-cleanup.sh > /dev/null <<'EOF'
@@ -185,102 +256,64 @@ sudo chmod +x /usr/local/bin/docker-cleanup.sh
echo "0 3 * * * /usr/local/bin/docker-cleanup.sh >> /var/log/docker-cleanup.log 2>&1" | sudo crontab - echo "0 3 * * * /usr/local/bin/docker-cleanup.sh >> /var/log/docker-cleanup.log 2>&1" | sudo crontab -
``` ```
### Log Rotation
Configure log rotation for GitLab Runner:
```bash
sudo tee /etc/logrotate.d/gitlab-runner > /dev/null <<EOF
/var/log/gitlab-runner/*.log {
daily
rotate 7
compress
missingok
notifempty
}
EOF
```
### Update Runner ### Update Runner
```bash ```bash
# Update GitLab Runner # Download new version
sudo apt update curl -L https://gitea.com/gitea/act_runner/releases/download/v0.2.12/act_runner-0.2.12-linux-amd64 -o /tmp/act_runner
sudo apt upgrade gitlab-runner sudo mv /tmp/act_runner /usr/local/bin/act_runner
sudo chmod +x /usr/local/bin/act_runner
# Restart runner # Restart service
sudo gitlab-runner restart sudo systemctl restart act_runner
``` ```
--- ---
## Security Considerations
### Firewall Configuration
```bash
# Allow only necessary outbound traffic
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow ssh
sudo ufw enable
```
### Runner Security
- **Locked runner**: Only accepts jobs from the specific project
- **Protected tags**: Only runs on protected branches (main)
- **Docker socket**: Mounted read-only where possible
### Secrets Management
The build server does NOT store application secrets. All secrets are:
- Stored in GitLab CI/CD Variables
- Injected at runtime on the production server
- Never cached in Docker layers
---
## Troubleshooting ## Troubleshooting
### Runner Not Picking Up Jobs ### Runner Not Picking Up Jobs
```bash ```bash
# Check runner status # Check service status
sudo gitlab-runner status sudo systemctl status act_runner
# View runner logs # View logs
sudo journalctl -u gitlab-runner -f sudo journalctl -u act_runner -f
# Re-register runner if needed # Check registration
sudo gitlab-runner unregister --all-runners sudo -u act_runner act_runner list
sudo gitlab-runner register
``` ```
### Docker Build Failures ### Docker Permission Issues
```bash ```bash
# Check Docker daemon # Ensure act_runner is in docker group
sudo systemctl status docker sudo usermod -aG docker act_runner
# Check available disk space # Restart service
df -h sudo systemctl restart act_runner
# Clear Docker cache
docker system prune -af
``` ```
### Registry Push Failures ### Registry Authentication Failures
```bash ```bash
# Verify registry login # Re-login to registry
docker login registry.motovaultpro.com sudo -u act_runner docker logout git.motovaultpro.com
sudo -u act_runner docker login git.motovaultpro.com -u egullickson
```
# Check network connectivity ### Staging Not Accessible
curl -v https://registry.motovaultpro.com/v2/
# Verify image exists ```bash
docker images | grep motovaultpro # Check containers
docker ps
# Check Traefik logs
docker logs mvp-traefik-staging
# Check SSL certificate
curl -vI https://staging.motovaultpro.com
``` ```
--- ---
@@ -291,25 +324,27 @@ docker images | grep motovaultpro
| Path | Description | | Path | Description |
|------|-------------| |------|-------------|
| `/etc/gitlab-runner/config.toml` | Runner configuration | | `/opt/motovaultpro` | Application root |
| `/var/log/gitlab-runner/` | Runner logs | | `/opt/motovaultpro/secrets/staging` | Staging secrets |
| `~/.docker/config.json` | Docker registry credentials | | `/etc/act_runner/config.yaml` | Runner configuration |
| `/var/lib/docker/` | Docker data | | `/home/act_runner/.docker/config.json` | Registry credentials |
### Common Commands ### Common Commands
```bash ```bash
# Runner management # Runner management
sudo gitlab-runner status sudo systemctl status act_runner
sudo gitlab-runner restart sudo systemctl restart act_runner
sudo gitlab-runner verify sudo journalctl -u act_runner -f
# Docker management # Docker management
docker system df # Check disk usage docker system df
docker system prune -af # Clean all unused resources docker system prune -af
docker images # List images docker ps
docker ps -a # List containers docker logs -f mvp-backend-staging
# View build logs # Staging stack
sudo journalctl -u gitlab-runner --since "1 hour ago" cd /opt/motovaultpro
docker compose -f docker-compose.yml -f docker-compose.staging.yml ps
docker compose -f docker-compose.yml -f docker-compose.staging.yml logs -f
``` ```

View File

@@ -1,72 +1,52 @@
# MotoVaultPro GitLab CI/CD Deployment Guide # MotoVaultPro CI/CD Deployment Guide
Complete guide for deploying MotoVaultPro using GitLab CI/CD with blue-green deployment and auto-rollback. Complete guide for deploying MotoVaultPro using Gitea Actions with staging-first deployment and manual production approval.
## Table of Contents ## Table of Contents
1. [Architecture Overview](#architecture-overview) 1. [Architecture Overview](#architecture-overview)
2. [Prerequisites](#prerequisites) 2. [Prerequisites](#prerequisites)
3. [Pipeline Stages](#pipeline-stages) 3. [Workflow Structure](#workflow-structure)
4. [Blue-Green Deployment](#blue-green-deployment) 4. [Deployment Process](#deployment-process)
5. [CI/CD Variables Configuration](#cicd-variables-configuration) 5. [Secrets and Variables](#secrets-and-variables)
6. [Container Registry](#container-registry) 6. [Container Registry](#container-registry)
7. [Deployment Process](#deployment-process) 7. [Rollback Procedures](#rollback-procedures)
8. [Rollback Procedures](#rollback-procedures) 8. [Maintenance Migrations](#maintenance-migrations)
9. [Maintenance Migrations](#maintenance-migrations) 9. [Troubleshooting](#troubleshooting)
10. [Notifications](#notifications)
11. [Troubleshooting](#troubleshooting)
--- ---
## Architecture Overview ## Architecture Overview
MotoVaultPro uses a blue-green deployment strategy with automatic rollback: MotoVaultPro uses a **staging-first deployment** strategy with manual approval:
``` ```
+---------------------------------------------------+ +---------------------------------------------------+
| GitLab (CI/CD + Registry) | | Gitea (git.motovaultpro.com) |
| +--------------------+ +--------------------+ |
| | Gitea Actions | | Package Registry | |
| +--------------------+ +--------------------+ |
+---------------------------------------------------+ +---------------------------------------------------+
| | | |
v v v v
+------------------+ +-----------------------+ +-------------------+ +--------------------+
| Build VPS | | Production Server | | Build/Staging VPS | | Production Server |
| (Docker Runner) | | (Shell Runner) | | (mvp-build) | | (mvp-prod) |
| Tags: build | | Tags: production | | act_runner | | act_runner |
+------------------+ +-----------+-----------+ +--------+----------+ +----------+---------+
| | | |
| Push images | Pull + Deploy
v v v v
+---------------------------------------------------+ staging.motovaultpro.com motovaultpro.com
| GitLab Container Registry | (Full Stack) (Blue-Green)
| registry.motovaultpro.com/motovaultpro/ |
+---------------------------------------------------+
|
+---------------+---------------+
| |
+--------v--------+ +--------v--------+
| BLUE Stack | | GREEN Stack |
| mvp-frontend | | mvp-frontend |
| mvp-backend | | mvp-backend |
+-----------------+ +-----------------+
| |
+----------- Traefik -----------+
(weighted LB)
|
+---------------+---------------+
| |
+--------v--------+ +--------v--------+
| PostgreSQL | | Redis |
| (shared) | | (shared) |
+-----------------+ +-----------------+
``` ```
### Key Features ### Key Features
- **Zero-downtime deployments**: Traffic switches in under 5 seconds - **Staging-first**: All changes verified on staging before production
- **Instant rollback**: Previous version remains running - **Manual approval**: Production deploy requires manual trigger
- **Automatic rollback**: On health check failure - **Blue-green production**: Zero-downtime deployments
- **Auto-rollback**: Automatic rollback on health check failure
- **Email notifications**: Via Resend API - **Email notifications**: Via Resend API
- **Container registry**: Self-hosted on GitLab (no Docker Hub)
--- ---
@@ -74,180 +54,172 @@ MotoVaultPro uses a blue-green deployment strategy with automatic rollback:
### Server Requirements ### Server Requirements
| Server | Purpose | Specs | Runner Tags | | Server | Purpose | Specs | Runner Label |
|--------|---------|-------|-------------| |--------|---------|-------|--------------|
| Build VPS | Docker image builds | 2 CPU, 4GB RAM | `build` | | Build/Staging VPS | Build + Staging | 4 CPU, 8GB RAM | `mvp-build` |
| Prod Server | Application hosting | 8GB+ RAM | `production` | | Prod Server | Production | 8GB+ RAM | `mvp-prod` |
See [BUILD-SERVER-SETUP.md](BUILD-SERVER-SETUP.md) for build server setup. See [BUILD-SERVER-SETUP.md](BUILD-SERVER-SETUP.md) for setup instructions.
### Software Requirements ### Software Requirements
- GitLab 18.6+ - Gitea 1.21+ with Actions enabled
- Docker Engine 24.0+ - Docker Engine 24.0+
- Docker Compose v2 - Docker Compose v2
- GitLab Runner (shell executor on both servers) - act_runner (Gitea Actions runner)
- `jq` for JSON processing - `jq` for JSON processing
--- ---
## Pipeline Stages ## Workflow Structure
The CI/CD pipeline consists of 7 stages: ### Two-Workflow Strategy
``` | Workflow | Trigger | Purpose |
validate -> build -> deploy-prepare -> deploy-switch -> verify -> [rollback] -> notify |----------|---------|---------|
``` | `staging.yaml` | Push to main | Build, deploy to staging, verify |
| `production.yaml` | Manual (workflow_dispatch) | Deploy to production |
| Stage | Runner | Purpose | ### Staging Workflow (Automatic)
|-------|--------|---------|
| `validate` | prod | Check prerequisites, determine target stack |
| `build` | build | Build and push images to GitLab registry |
| `deploy-prepare` | prod | Pull images, start inactive stack, health check |
| `deploy-switch` | prod | Switch Traefik traffic weights |
| `verify` | prod | Production health verification |
| `rollback` | prod | Auto-triggered on verify failure |
| `notify` | prod | Email success/failure notifications |
### Pipeline Flow
``` ```
[Push to main] [Push to main]
| |
v v
[validate] - Checks Docker, paths, registry [build] -------- Build images, push to registry
| |
v v
[build] - Builds backend + frontend images [deploy-staging] - Deploy full stack to staging
| Pushes to registry.motovaultpro.com
v
[deploy-prepare] - Pulls new images
| Starts inactive stack (blue or green)
| Runs health checks
v
[deploy-switch] - Updates Traefik weights
| Switches traffic instantly
v
[verify] - External health check
| Container status verification
|
+--[SUCCESS]--> [notify-success] - Sends success email
|
+--[FAILURE]--> [rollback] - Switches back to previous stack
| |
v v
[notify-failure] - Sends failure email [verify-staging] - Health checks
|
+--[FAIL]--> [notify-staging-failure]
|
v
[notify-staging-ready] - Email with production deploy link
```
### Production Workflow (Manual)
```
[Manual Trigger] - User clicks "Run workflow"
|
v
[validate] - Check prerequisites, determine stack
|
v
[deploy-prod] - Blue-green deployment
|
v
[verify-prod] - External health checks
|
+--[FAIL]--> [rollback] --> [notify-failure]
|
v
[notify-success]
``` ```
--- ---
## Blue-Green Deployment ## Deployment Process
### Stack Configuration ### 1. Push to Main Branch
Both stacks share the same database layer: When you push to `main`:
1. Staging workflow triggers automatically
2. Images are built and pushed to Gitea Package Registry
3. Full stack deploys to staging.motovaultpro.com
4. Health checks verify staging works
5. Email notification sent with production deploy link
| Component | Blue Stack | Green Stack | Shared | ### 2. Review Staging
|-----------|------------|-------------|--------|
| Frontend | `mvp-frontend-blue` | `mvp-frontend-green` | - |
| Backend | `mvp-backend-blue` | `mvp-backend-green` | - |
| PostgreSQL | - | - | `mvp-postgres` |
| Redis | - | - | `mvp-redis` |
| Traefik | - | - | `mvp-traefik` |
### Traffic Routing After receiving the "Staging Ready" email:
1. Visit https://staging.motovaultpro.com
2. Test functionality
3. Review logs if needed: `docker logs mvp-backend-staging`
Traefik uses weighted services for traffic distribution: ### 3. Deploy to Production
```yaml When ready to deploy:
# config/traefik/dynamic/blue-green.yml 1. Go to `git.motovaultpro.com/egullickson/motovaultpro/actions`
services: 2. Select "Deploy to Production" workflow
mvp-frontend-weighted: 3. Click "Run workflow"
weighted: 4. Optionally specify image tag (defaults to `latest`)
services: 5. Click "Run workflow" to confirm
- name: mvp-frontend-blue-svc
weight: 100 # Active
- name: mvp-frontend-green-svc
weight: 0 # Standby
```
### Deployment State ### 4. Monitor Production Deploy
State is tracked in `config/deployment/state.json`: The production workflow will:
1. Determine target stack (blue or green)
```json 2. Pull and start the new stack
{ 3. Run health checks
"active_stack": "blue", 4. Switch traffic
"inactive_stack": "green", 5. Verify external health
"last_deployment": "2024-01-15T10:30:00Z", 6. Auto-rollback if verification fails
"last_deployment_commit": "abc123", 7. Send email notification
"rollback_available": true
}
```
--- ---
## CI/CD Variables Configuration ## Secrets and Variables
Navigate to **Settings > CI/CD > Variables** in your GitLab project. ### Secrets Configuration
### Required Variables Navigate to: `git.motovaultpro.com/egullickson/motovaultpro/settings/actions/secrets`
| Variable | Type | Protected | Purpose | | Secret | Description |
|----------|------|-----------|---------| |--------|-------------|
| `DEPLOY_NOTIFY_EMAIL` | Variable | Yes | Notification recipient | | `REGISTRY_USER` | Gitea username (egullickson) |
| `VITE_AUTH0_DOMAIN` | Variable | No | Auth0 domain | | `REGISTRY_PASSWORD` | Gitea access token |
| `VITE_AUTH0_CLIENT_ID` | Variable | No | Auth0 client ID | | `POSTGRES_PASSWORD` | Production PostgreSQL password |
| `VITE_AUTH0_AUDIENCE` | Variable | No | Auth0 audience | | `STAGING_POSTGRES_PASSWORD` | Staging PostgreSQL password |
| `AUTH0_CLIENT_SECRET` | Auth0 secret |
| `AUTH0_MANAGEMENT_CLIENT_ID` | Auth0 Management API ID |
| `AUTH0_MANAGEMENT_CLIENT_SECRET` | Auth0 Management API secret |
| `GOOGLE_MAPS_API_KEY` | Google Maps key |
| `GOOGLE_MAPS_MAP_ID` | Google Maps Map ID |
| `CF_DNS_API_TOKEN` | Cloudflare DNS token |
| `RESEND_API_KEY` | Resend email API key |
### Secret Files ### Variables Configuration
These use GitLab's **File** type and are injected via `scripts/inject-secrets.sh`: Navigate to: `git.motovaultpro.com/egullickson/motovaultpro/settings/actions/variables`
| Variable | Type | Protected | Masked | | Variable | Value |
|----------|------|-----------|--------| |----------|-------|
| `POSTGRES_PASSWORD` | File | Yes | Yes | | `DEPLOY_NOTIFY_EMAIL` | Notification recipient |
| `AUTH0_CLIENT_SECRET` | File | Yes | Yes | | `VITE_AUTH0_DOMAIN` | motovaultpro.us.auth0.com |
| `GOOGLE_MAPS_API_KEY` | File | Yes | Yes | | `VITE_AUTH0_CLIENT_ID` | Auth0 client ID |
| `GOOGLE_MAPS_MAP_ID` | File | Yes | No | | `VITE_AUTH0_AUDIENCE` | https://api.motovaultpro.com |
| `CF_DNS_API_TOKEN` | File | Yes | Yes |
| `RESEND_API_KEY` | File | Yes | Yes |
### Registry Authentication
GitLab provides these automatically:
- `CI_REGISTRY_USER` - Registry username
- `CI_REGISTRY_PASSWORD` - Registry token
- `CI_REGISTRY` - Registry URL
--- ---
## Container Registry ## Container Registry
All images are hosted on the GitLab Container Registry to avoid Docker Hub rate limits. All images are hosted on Gitea Package Registry.
### Registry URL ### Registry URL
``` ```
registry.motovaultpro.com git.motovaultpro.com
``` ```
### Image Paths ### Image Paths
| Image | Path | | Image | Path |
|-------|------| |-------|------|
| Backend | `registry.motovaultpro.com/motovaultpro/backend:$TAG` | | Backend | `git.motovaultpro.com/egullickson/backend:$TAG` |
| Frontend | `registry.motovaultpro.com/motovaultpro/frontend:$TAG` | | Frontend | `git.motovaultpro.com/egullickson/frontend:$TAG` |
| Mirrors | `registry.motovaultpro.com/mirrors/` | | Mirrors | `git.motovaultpro.com/egullickson/mirrors/` |
### Base Image Mirrors ### Base Image Mirrors
Mirror upstream images to avoid rate limits: Run the mirror workflow to avoid Docker Hub rate limits:
```bash 1. Go to Actions tab
# Run manually or via scheduled pipeline 2. Select "Mirror Base Images"
./scripts/ci/mirror-base-images.sh 3. Click "Run workflow"
```
Mirrored images: Mirrored images:
- `node:20-alpine` - `node:20-alpine`
@@ -255,41 +227,6 @@ Mirrored images:
- `postgres:18-alpine` - `postgres:18-alpine`
- `redis:8.4-alpine` - `redis:8.4-alpine`
- `traefik:v3.6` - `traefik:v3.6`
- `docker:24.0`
- `docker:24.0-dind`
---
## Deployment Process
### Automatic Deployment
Deployments trigger automatically on push to `main`:
1. **Validate**: Check prerequisites, determine target stack
2. **Build**: Build images on dedicated build server
3. **Prepare**: Start inactive stack, run health checks
4. **Switch**: Update Traefik weights (instant)
5. **Verify**: External health check
6. **Notify**: Send email notification
### Manual Deployment
1. Go to **CI/CD > Pipelines**
2. Click **Run pipeline**
3. Select `main` branch
4. Click **Run pipeline**
### Deployment Timeline
| Phase | Duration |
|-------|----------|
| Validate | ~5s |
| Build | ~2 min |
| Deploy-prepare | ~30s |
| Deploy-switch | ~3s |
| Verify | ~30s |
| **Total** | ~3 min |
--- ---
@@ -297,15 +234,9 @@ Deployments trigger automatically on push to `main`:
### Automatic Rollback ### Automatic Rollback
Triggers automatically when: Production workflow auto-rolls back when:
- Health check fails in `deploy-prepare` - Health check fails after traffic switch
- `verify` stage fails after switch - Container becomes unhealthy during verification
- Container becomes unhealthy within verification period
The pipeline runs `scripts/ci/auto-rollback.sh` which:
1. Verifies previous stack is healthy
2. Switches traffic back
3. Sends notification
### Manual Rollback ### Manual Rollback
@@ -326,6 +257,8 @@ cat config/deployment/state.json | jq .
If both stacks are unhealthy: If both stacks are unhealthy:
```bash ```bash
cd /opt/motovaultpro
# Stop everything # Stop everything
docker compose -f docker-compose.yml -f docker-compose.blue-green.yml down docker compose -f docker-compose.yml -f docker-compose.blue-green.yml down
@@ -336,8 +269,8 @@ docker compose up -d mvp-postgres mvp-redis mvp-traefik
sleep 15 sleep 15
# Start one stack # Start one stack
export BACKEND_IMAGE=registry.motovaultpro.com/motovaultpro/backend:latest export BACKEND_IMAGE=git.motovaultpro.com/egullickson/backend:latest
export FRONTEND_IMAGE=registry.motovaultpro.com/motovaultpro/frontend:latest export FRONTEND_IMAGE=git.motovaultpro.com/egullickson/frontend:latest
docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d \ docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d \
mvp-frontend-blue mvp-backend-blue mvp-frontend-blue mvp-backend-blue
@@ -351,11 +284,13 @@ docker compose -f docker-compose.yml -f docker-compose.blue-green.yml up -d \
For breaking database changes requiring downtime: For breaking database changes requiring downtime:
### Via Pipeline (Recommended) ### Via Gitea Actions
1. Go to **CI/CD > Pipelines** 1. Go to Actions tab
2. Find the `maintenance-migration` job 2. Select "Maintenance Migration"
3. Click **Play** to trigger manually 3. Click "Run workflow"
4. Choose whether to create backup
5. Click "Run workflow"
### Via Script ### Via Script
@@ -369,98 +304,68 @@ cd /opt/motovaultpro
./scripts/ci/maintenance-migrate.sh ./scripts/ci/maintenance-migrate.sh
``` ```
### What Happens
1. Sends maintenance notification
2. Enables maintenance mode (stops traffic)
3. Creates database backup (if requested)
4. Runs migrations
5. Restarts backends
6. Restores traffic
7. Sends completion notification
---
## Notifications
Email notifications via Resend API for:
| Event | Subject |
|-------|---------|
| `success` | Deployment Successful |
| `failure` | Deployment Failed |
| `rollback` | Auto-Rollback Executed |
| `rollback_failed` | CRITICAL: Rollback Failed |
| `maintenance_start` | Maintenance Mode Started |
| `maintenance_end` | Maintenance Complete |
Configure recipient in GitLab CI/CD variables:
```
DEPLOY_NOTIFY_EMAIL = admin@example.com
```
--- ---
## Troubleshooting ## Troubleshooting
### Pipeline Fails at Build Stage ### Staging Workflow Failures
**Check build server connectivity:** **Build fails:**
```bash ```bash
# On build server # On build server
sudo gitlab-runner verify docker system df
docker login registry.motovaultpro.com
```
**Check disk space:**
```bash
df -h
docker system prune -af docker system prune -af
``` ```
### Pipeline Fails at Deploy-Prepare **Deploy fails:**
**Container won't start:**
```bash ```bash
docker logs mvp-backend-blue --tail 100 docker logs mvp-backend-staging
docker logs mvp-frontend-blue --tail 100 docker logs mvp-frontend-staging
``` ```
### Production Workflow Failures
**Health check timeout:** **Health check timeout:**
```bash ```bash
# Increase timeout in .gitlab-ci.yml # Check containers
HEALTH_CHECK_TIMEOUT: "90" docker ps
docker logs mvp-backend-blue # or green
``` ```
### Traffic Not Switching **Traffic not switching:**
**Check Traefik config:**
```bash ```bash
# Check Traefik config
cat config/traefik/dynamic/blue-green.yml cat config/traefik/dynamic/blue-green.yml
docker exec mvp-traefik traefik healthcheck docker exec mvp-traefik traefik healthcheck
``` ```
**Check routing:** ### Runner Issues
**Runner offline:**
```bash ```bash
curl -I https://motovaultpro.com/api/health sudo systemctl status act_runner
sudo journalctl -u act_runner -f
``` ```
### Verify Stage Fails **Permission denied:**
**Check external connectivity:**
```bash ```bash
curl -sf https://motovaultpro.com/api/health sudo usermod -aG docker act_runner
``` sudo systemctl restart act_runner
**Check container health:**
```bash
docker inspect --format='{{.State.Health.Status}}' mvp-backend-blue
``` ```
--- ---
## Quick Reference ## Quick Reference
### Workflow Locations
| Workflow | File |
|----------|------|
| Staging | `.gitea/workflows/staging.yaml` |
| Production | `.gitea/workflows/production.yaml` |
| Maintenance | `.gitea/workflows/maintenance.yaml` |
| Mirror Images | `.gitea/workflows/mirror-images.yaml` |
### Important Paths ### Important Paths
| Path | Description | | Path | Description |
@@ -472,11 +377,11 @@ docker inspect --format='{{.State.Health.Status}}' mvp-backend-blue
### Common Commands ### Common Commands
```bash ```bash
# View current state # View deployment state
cat config/deployment/state.json | jq . cat config/deployment/state.json | jq .
# Check container status # Check containers
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Health}}" docker ps --format "table {{.Names}}\t{{.Status}}"
# View logs # View logs
docker logs mvp-backend-blue -f docker logs mvp-backend-blue -f
@@ -486,21 +391,14 @@ docker logs mvp-backend-blue -f
# Run health check # Run health check
./scripts/ci/health-check.sh blue ./scripts/ci/health-check.sh blue
# Send test notification
./scripts/ci/notify.sh success "Test message"
``` ```
### Memory Budget (8GB Server) ### Email Notifications
| Component | RAM | | Event | Trigger |
|-----------|-----| |-------|---------|
| Blue frontend | 512MB | | Staging Ready | Staging verified successfully |
| Blue backend | 1GB | | Success | Production deployed successfully |
| Green frontend | 512MB | | Failure | Deployment or verification failed |
| Green backend | 1GB | | Rollback | Auto-rollback executed |
| PostgreSQL | 2GB | | Maintenance | Migration started/completed |
| Redis | 512MB |
| Traefik | 128MB |
| System | 1.3GB |
| **Total** | ~7GB |

View File

@@ -1,8 +1,8 @@
# Production Dockerfile for MotoVaultPro Frontend # Production Dockerfile for MotoVaultPro Frontend
# Uses mirrored base images from GitLab Container Registry # Uses mirrored base images from Gitea Package Registry
# Build argument for registry (defaults to GitLab mirrors, falls back to Docker Hub) # Build argument for registry (defaults to Gitea mirrors, falls back to Docker Hub)
ARG REGISTRY_MIRRORS=registry.motovaultpro.com/mirrors ARG REGISTRY_MIRRORS=git.motovaultpro.com/egullickson/mirrors
# Stage 1: Base with dependencies # Stage 1: Base with dependencies
FROM ${REGISTRY_MIRRORS}/node:20-alpine AS base FROM ${REGISTRY_MIRRORS}/node:20-alpine AS base

View File

@@ -5,7 +5,7 @@
set -euo pipefail set -euo pipefail
REGISTRY="${REGISTRY:-registry.motovaultpro.com/mirrors}" REGISTRY="${REGISTRY:-git.motovaultpro.com/egullickson/mirrors}"
# Base images required by MotoVaultPro # Base images required by MotoVaultPro
IMAGES=( IMAGES=(

View File

@@ -3,7 +3,7 @@
# Sends email notifications for deployment events # Sends email notifications for deployment events
# #
# Usage: ./notify.sh <event_type> [message] [commit_sha] # Usage: ./notify.sh <event_type> [message] [commit_sha]
# event_type: success, failure, rollback, rollback_failed, maintenance_start, maintenance_end # event_type: success, failure, rollback, rollback_failed, maintenance_start, maintenance_end, staging_ready
# message: Optional custom message # message: Optional custom message
# commit_sha: Optional commit SHA for context # commit_sha: Optional commit SHA for context
# #
@@ -39,15 +39,16 @@ if [[ -z "$NOTIFY_EMAIL" ]]; then
exit 0 exit 0
fi fi
# Get Resend API key # Get Resend API key (check env first for Gitea, then files for containers)
RESEND_API_KEY="" if [[ -z "${RESEND_API_KEY:-}" ]]; then
if [[ -f "/run/secrets/resend-api-key" ]]; then if [[ -f "/run/secrets/resend-api-key" ]]; then
RESEND_API_KEY=$(cat /run/secrets/resend-api-key) RESEND_API_KEY=$(cat /run/secrets/resend-api-key)
elif [[ -f "$PROJECT_ROOT/secrets/app/resend-api-key.txt" ]]; then elif [[ -f "$PROJECT_ROOT/secrets/app/resend-api-key.txt" ]]; then
RESEND_API_KEY=$(cat "$PROJECT_ROOT/secrets/app/resend-api-key.txt") RESEND_API_KEY=$(cat "$PROJECT_ROOT/secrets/app/resend-api-key.txt")
fi
fi fi
if [[ -z "$RESEND_API_KEY" ]]; then if [[ -z "${RESEND_API_KEY:-}" ]]; then
echo "WARNING: Resend API key not found, skipping notification" echo "WARNING: Resend API key not found, skipping notification"
exit 0 exit 0
fi fi
@@ -96,6 +97,13 @@ case "$EVENT_TYPE" in
STATUS_TEXT="Maintenance Complete" STATUS_TEXT="Maintenance Complete"
DEFAULT_MESSAGE="Maintenance window complete. Application is online." DEFAULT_MESSAGE="Maintenance window complete. Application is online."
;; ;;
"staging_ready")
SUBJECT="Staging Ready for Production - MotoVaultPro"
STATUS_COLOR="#3b82f6"
STATUS_EMOJI="[STAGING]"
STATUS_TEXT="Staging Verified"
DEFAULT_MESSAGE="Staging deployment verified. Ready for production deployment."
;;
*) *)
echo "Unknown event type: $EVENT_TYPE" echo "Unknown event type: $EVENT_TYPE"
exit 1 exit 1

View File

@@ -1,33 +1,26 @@
#!/bin/bash #!/bin/bash
# inject-secrets.sh # inject-secrets.sh
# Writes GitLab CI File type variables to the secrets directory # Writes secrets to the secrets directory for K8s-style secret mounting
# for K8s-style secret mounting in Docker Compose
# #
# GitLab File variables provide the PATH to a temporary file containing the secret. # Supports two modes:
# This script copies those files to the expected secrets/app/ location. # 1. GitLab CI: File variables provide PATH to temp file
# 2. Gitea Actions: Environment variables contain the secret value directly
# #
# IMPORTANT: In GitLab, variables MUST be set as "File" type, not "Variable" type. # Required environment variables:
# File type variables provide a PATH to a temp file containing the secret.
# Variable type provides the raw value, which will NOT work with this script.
#
# Required GitLab CI/CD Variables (File type):
# - POSTGRES_PASSWORD # - POSTGRES_PASSWORD
# - AUTH0_CLIENT_SECRET # - AUTH0_CLIENT_SECRET
# - AUTH0_MANAGEMENT_CLIENT_ID (Auth0 Management API client ID for user signup) # - AUTH0_MANAGEMENT_CLIENT_ID
# - AUTH0_MANAGEMENT_CLIENT_SECRET (Auth0 Management API client secret) # - AUTH0_MANAGEMENT_CLIENT_SECRET
# - GOOGLE_MAPS_API_KEY # - GOOGLE_MAPS_API_KEY
# - GOOGLE_MAPS_MAP_ID # - GOOGLE_MAPS_MAP_ID
# - CF_DNS_API_TOKEN (Cloudflare DNS API token for Let's Encrypt certificates) # - CF_DNS_API_TOKEN
# - RESEND_API_KEY (Resend API key for email notifications) # - RESEND_API_KEY
#
# Required GitLab CI/CD Variables (Variable type):
# - DEPLOY_PATH
set -euo pipefail set -euo pipefail
# Configuration # Configuration
DEPLOY_PATH="${DEPLOY_PATH:-/opt/motovaultpro}" DEPLOY_PATH="${DEPLOY_PATH:-/opt/motovaultpro}"
SECRETS_DIR="${DEPLOY_PATH}/secrets/app" SECRETS_DIR="${SECRETS_DIR:-${DEPLOY_PATH}/secrets/app}"
# List of all secret files (must match docker-compose volume mounts) # List of all secret files (must match docker-compose volume mounts)
SECRET_FILES=( SECRET_FILES=(
@@ -66,37 +59,33 @@ mkdir -p "$SECRETS_DIR"
chmod 700 "$SECRETS_DIR" chmod 700 "$SECRETS_DIR"
# Function to inject a secret # Function to inject a secret
# Supports both:
# - Direct value (Gitea Actions): VAR contains the secret
# - File path (GitLab CI): VAR contains path to file with secret
inject_secret() { inject_secret() {
local var_name="$1" local var_name="$1"
local file_name="$2" local file_name="$2"
local target_path="${SECRETS_DIR}/${file_name}" local target_path="${SECRETS_DIR}/${file_name}"
# GitLab File variables contain the PATH to a temp file local source_value="${!var_name:-}"
local source_path="${!var_name:-}"
if [ -z "$source_path" ]; then if [ -z "$source_value" ]; then
echo " ERROR: Variable $var_name is not set" echo " ERROR: Variable $var_name is not set"
echo " Ensure it exists in GitLab CI/CD Variables"
return 1 return 1
fi fi
# Check if it looks like a raw value instead of a file path # Check if it's a file path (GitLab CI File variable)
if [[ ! "$source_path" =~ ^/ ]]; then if [[ "$source_value" =~ ^/ ]] && [ -f "$source_value" ]; then
echo " ERROR: $var_name appears to be a raw value, not a file path" # GitLab mode: copy from file
echo " In GitLab, change the variable Type from 'Variable' to 'File'" cp "$source_value" "$target_path"
return 1 echo " OK: $file_name (from file)"
else
# Gitea mode: write value directly
echo -n "$source_value" > "$target_path"
echo " OK: $file_name (from env)"
fi fi
if [ ! -f "$source_path" ]; then
echo " ERROR: File not found for $var_name at $source_path"
echo " Ensure the variable is set as 'File' type in GitLab"
return 1
fi
# Copy the secret file (644 so container users can read)
cp "$source_path" "$target_path"
chmod 644 "$target_path" chmod 644 "$target_path"
echo " OK: $file_name"
} }
# Inject all secrets # Inject all secrets
@@ -114,7 +103,6 @@ inject_secret "RESEND_API_KEY" "resend-api-key.txt" || FAILED=1
if [ $FAILED -eq 1 ]; then if [ $FAILED -eq 1 ]; then
echo "" echo ""
echo "ERROR: One or more secrets failed to inject" echo "ERROR: One or more secrets failed to inject"
echo "Ensure all required CI/CD variables are configured as File type in GitLab"
exit 1 exit 1
fi fi