CI/CD Improvements

This commit is contained in:
Eric Gullickson
2025-12-18 12:03:04 -06:00
parent 192f2edc04
commit c6e187e29e
6 changed files with 858 additions and 18 deletions

View File

@@ -1,19 +1,184 @@
stages: # MotoVaultPro GitLab CI/CD Pipeline
- deploy # GitLab 18.6+ with shell executor
# See docs/CICD-DEPLOY.md for complete documentation
deploy_prod: stages:
stage: deploy - validate
- build
- deploy
- verify
variables:
DOCKER_COMPOSE_FILE: docker-compose.yml
DOCKER_COMPOSE_PROD_FILE: docker-compose.prod.yml
# ------------------------------------------------------------------------------
# Validate Stage - Check prerequisites
# ------------------------------------------------------------------------------
validate:
stage: validate
only: only:
- main - main
script: script:
- echo ">>> Pulling latest code" - echo "=========================================="
- git pull origin main - echo "Validating deployment prerequisites..."
- echo "=========================================="
- echo ">>> Pulling updated images" - echo "Checking Docker..."
- sudo docker compose pull - docker info > /dev/null 2>&1 || (echo "ERROR: Docker not accessible" && exit 1)
- echo "OK: Docker is accessible"
- echo ">>> Rebuilding local images if needed" - echo "Checking Docker Compose..."
- sudo docker compose build - docker compose version > /dev/null 2>&1 || (echo "ERROR: Docker Compose not available" && exit 1)
- echo "OK: Docker Compose is available"
- echo ">>> Starting/Updating services" - echo "Checking deployment path..."
- sudo docker compose up -d - test -d "$DEPLOY_PATH" || (echo "ERROR: DEPLOY_PATH ($DEPLOY_PATH) not found" && exit 1)
- echo "OK: Deployment path exists"
- echo "=========================================="
- echo "Validation complete"
- echo "=========================================="
# ------------------------------------------------------------------------------
# Build Stage - Build Docker images
# ------------------------------------------------------------------------------
build:
stage: build
only:
- main
script:
- echo "=========================================="
- echo "Building Docker images..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
- echo "Pulling latest code..."
- git fetch origin main
- git reset --hard origin/main
- echo "Building images..."
- docker compose -f $DOCKER_COMPOSE_FILE build --no-cache
- echo "=========================================="
- echo "Build complete"
- echo "=========================================="
# ------------------------------------------------------------------------------
# Deploy Stage - Inject secrets and deploy services
# ------------------------------------------------------------------------------
deploy:
stage: deploy
only:
- main
environment:
name: production
url: https://motovaultpro.com
script:
- echo "=========================================="
- echo "Deploying MotoVaultPro..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Inject secrets from GitLab File variables
- echo "Step 1/6: Injecting secrets..."
- chmod +x scripts/inject-secrets.sh
- ./scripts/inject-secrets.sh
# Stop existing services gracefully
- echo "Step 2/6: Stopping existing services..."
- docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE down --timeout 30 || true
# Pull latest base images
- echo "Step 3/6: Pulling base images..."
- docker compose -f $DOCKER_COMPOSE_FILE pull
# Start database services first for migrations
- echo "Step 4/6: Starting database services..."
- docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d mvp-postgres mvp-redis
- echo "Waiting for database to be ready..."
- sleep 15
# Run database migrations
- echo "Step 5/6: Running database migrations..."
- docker compose -f $DOCKER_COMPOSE_FILE run --rm mvp-backend npm run migrate || echo "Migration command not found or no migrations to run"
# Start all services
- echo "Step 6/6: Starting all services..."
- docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d
# Wait for services to start
- echo "Waiting for services to initialize..."
- sleep 30
- echo "=========================================="
- echo "Deployment complete"
- echo "=========================================="
# ------------------------------------------------------------------------------
# Verify Stage - Health checks
# ------------------------------------------------------------------------------
verify:
stage: verify
only:
- main
script:
- echo "=========================================="
- echo "Verifying deployment..."
- echo "=========================================="
- cd "$DEPLOY_PATH"
# Check all containers are running
- echo "Checking container status..."
- |
FAILED=0
for service in mvp-traefik mvp-frontend mvp-backend mvp-postgres mvp-redis; do
status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
if [ "$status" != "running" ]; then
echo "ERROR: $service is not running (status: $status)"
docker logs $service --tail 50 2>/dev/null || true
FAILED=1
else
echo "OK: $service is running"
fi
done
if [ $FAILED -eq 1 ]; then
echo "One or more services failed to start"
exit 1
fi
# Check backend health endpoint
- echo "Checking backend health..."
- |
HEALTH_OK=0
for i in 1 2 3 4 5 6; do
if curl -sf http://localhost:3001/health > /dev/null 2>&1; then
echo "OK: Backend health check passed"
HEALTH_OK=1
break
fi
echo "Attempt $i/6: Backend not ready, waiting 10s..."
sleep 10
done
if [ $HEALTH_OK -eq 0 ]; then
echo "ERROR: Backend health check failed after 6 attempts"
docker logs mvp-backend --tail 100
exit 1
fi
# Check frontend is accessible
- echo "Checking frontend..."
- |
if curl -sf http://localhost:3000 > /dev/null 2>&1; then
echo "OK: Frontend is accessible"
else
echo "WARNING: Frontend check failed (might need Traefik routing)"
fi
- echo "=========================================="
- echo "Deployment verified successfully!"
- echo "=========================================="
- echo "Application URL: https://motovaultpro.com"

28
docker-compose.prod.yml Normal file
View File

@@ -0,0 +1,28 @@
# docker-compose.prod.yml
# Production overrides for MotoVaultPro
#
# Usage:
# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
#
# This file removes development-only configurations:
# - Database port exposure (PostgreSQL, Redis)
# - Development-specific settings
services:
# PostgreSQL - Remove development port exposure
mvp-postgres:
ports: []
# Redis - Remove development port exposure
mvp-redis:
ports: []
# Traefik - Ensure dashboard authentication is enforced
mvp-traefik:
labels:
- "traefik.enable=true"
- "traefik.http.routers.traefik-dashboard.rule=Host(`traefik.motovaultpro.local`)"
- "traefik.http.routers.traefik-dashboard.tls=true"
- "traefik.http.routers.traefik-dashboard.middlewares=dashboard-auth"
- "traefik.http.services.traefik-dashboard.loadbalancer.server.port=8080"
- "traefik.http.middlewares.dashboard-auth.basicauth.users=admin:$$2y$$10$$foobar"

464
docs/CICD-DEPLOY.md Normal file
View File

@@ -0,0 +1,464 @@
# MotoVaultPro GitLab CI/CD Deployment Guide
Complete guide for deploying MotoVaultPro using GitLab CI/CD with shell executor runners.
## Table of Contents
1. [Prerequisites](#prerequisites)
2. [GitLab Runner Setup](#gitlab-runner-setup)
3. [CI/CD Variables Configuration](#cicd-variables-configuration)
4. [Secrets Architecture](#secrets-architecture)
5. [Pipeline Overview](#pipeline-overview)
6. [Deployment Process](#deployment-process)
7. [Rollback Procedure](#rollback-procedure)
8. [Troubleshooting](#troubleshooting)
---
## Prerequisites
### Server Requirements
- Linux server with Docker Engine installed
- Docker Compose v2 (plugin version)
- GitLab Runner installed and registered
- Git installed
- curl installed (for health checks)
### GitLab Requirements
- GitLab 18.6+ (tested with 18.6.2)
- Project with CI/CD enabled
- Protected `main` branch
- Maintainer access for CI/CD variable configuration
---
## GitLab Runner Setup
### 1. Verify Runner Registration
```bash
sudo gitlab-runner verify
```
Expected output should show your runner as active with shell executor.
### 2. Verify Docker Permissions
The `gitlab-runner` user must have Docker access:
```bash
# Add gitlab-runner to docker group (if not already done)
sudo usermod -aG docker gitlab-runner
# Verify access
sudo -u gitlab-runner docker info
sudo -u gitlab-runner docker compose version
```
### 3. Verify Deployment Directory
Ensure the deployment directory exists and is accessible:
```bash
# Create deployment directory
sudo mkdir -p /opt/motovaultpro
sudo chown gitlab-runner:gitlab-runner /opt/motovaultpro
# Clone repository (first time only)
sudo -u gitlab-runner git clone <repository-url> /opt/motovaultpro
```
---
## CI/CD Variables Configuration
Navigate to **Settings > CI/CD > Variables** in your GitLab project.
### Secrets (File Type Variables)
These variables use GitLab's **File** type, which writes the value to a temporary file and provides the path as the environment variable. This replicates the Kubernetes secrets pattern used by the application.
| Variable Name | Type | Protected | Masked | Description |
|--------------|------|-----------|--------|-------------|
| `POSTGRES_PASSWORD` | File | Yes | Yes | PostgreSQL database password |
| `AUTH0_CLIENT_SECRET` | File | Yes | Yes | Auth0 client secret for backend |
| `GOOGLE_MAPS_API_KEY` | File | Yes | Yes | Google Maps API key |
| `GOOGLE_MAPS_MAP_ID` | File | Yes | No | Google Maps Map ID |
### Configuration Variables
| Variable Name | Type | Protected | Masked | Value |
|--------------|------|-----------|--------|-------|
| `DEPLOY_PATH` | Variable | No | No | `/opt/motovaultpro` |
| `VITE_AUTH0_DOMAIN` | Variable | No | No | `motovaultpro.us.auth0.com` |
| `VITE_AUTH0_CLIENT_ID` | Variable | No | No | Your Auth0 client ID |
| `VITE_AUTH0_AUDIENCE` | Variable | No | No | `https://api.motovaultpro.com` |
### Setting Up a File Type Variable
1. Go to **Settings > CI/CD > Variables**
2. Click **Add variable**
3. Enter the variable key (e.g., `POSTGRES_PASSWORD`)
4. Enter the secret value in the **Value** field
5. Set **Type** to **File**
6. Enable **Protect variable** (recommended)
7. Enable **Mask variable** (for sensitive data)
8. Click **Add variable**
---
## Secrets Architecture
MotoVaultPro uses a Kubernetes-style secrets pattern where secrets are mounted as files at `/run/secrets/` inside containers.
### How It Works
1. **GitLab stores secrets** as File type CI/CD variables
2. **During pipeline execution**, GitLab writes each secret to a temporary file
3. **The `inject-secrets.sh` script** copies these files to `secrets/app/` directory
4. **Docker Compose** mounts these files to `/run/secrets/` in containers
5. **Application code** reads secrets from the filesystem (not environment variables)
### Secret Files
```
secrets/app/
postgres-password.txt -> /run/secrets/postgres-password
auth0-client-secret.txt -> /run/secrets/auth0-client-secret
google-maps-api-key.txt -> /run/secrets/google-maps-api-key
google-maps-map-id.txt -> /run/secrets/google-maps-map-id
```
### Security Benefits
- Secrets never appear as environment variables (not visible in `env` or `printenv`)
- File permissions (600) restrict access
- Masked variables prevent accidental log exposure
- Protected variables only available on protected branches
---
## Pipeline Overview
The CI/CD pipeline consists of four stages:
### Stage 1: Validate
Verifies deployment prerequisites:
- Docker is accessible
- Docker Compose is available
- Deployment directory exists
### Stage 2: Build
Builds Docker images:
- Pulls latest code from repository
- Builds all service images with `--no-cache`
### Stage 3: Deploy
Deploys the application:
1. Injects secrets from GitLab variables
2. Stops existing services gracefully
3. Pulls base images
4. Starts database services (PostgreSQL, Redis)
5. Runs database migrations
6. Starts all services
### Stage 4: Verify
Validates deployment health:
- Checks all containers are running
- Tests backend health endpoint
- Reports deployment status
### Pipeline Diagram
```
[Validate] -> [Build] -> [Deploy] -> [Verify]
| | | |
Check Build Inject Health
prereqs images secrets checks
|
Migrate
|
Start
services
```
---
## Deployment Process
### Automatic Deployment
Deployments are triggered automatically when:
- Code is pushed to the `main` branch
- A merge request is merged into `main`
### Manual Deployment
To trigger a manual deployment:
1. Go to **CI/CD > Pipelines**
2. Click **Run pipeline**
3. Select the `main` branch
4. Click **Run pipeline**
### Deployment Steps (What Happens)
1. **Secrets Injection**
- `inject-secrets.sh` copies GitLab File variables to `secrets/app/`
- Permissions are set to 600 for security
2. **Service Shutdown**
- Existing containers are stopped gracefully (30s timeout)
- Volumes are preserved
3. **Database Startup**
- PostgreSQL and Redis start first
- 15-second wait for database readiness
4. **Migrations**
- Backend container runs database migrations
- Ensures schema is up-to-date
5. **Full Service Startup**
- All services start via `docker compose up -d`
- Traefik routes traffic automatically
6. **Health Verification**
- Container status checks
- Backend health endpoint validation
---
## Rollback Procedure
### Automatic Rollback
If the verify stage fails, the pipeline will report failure but services remain running. Manual intervention is required.
### Manual Rollback
Use the rollback script:
```bash
# SSH to server
ssh user@server
# Run rollback to previous commit
cd /opt/motovaultpro
./scripts/rollback.sh HEAD~1
# Or rollback to specific tag/commit
./scripts/rollback.sh v1.0.0
```
### Rollback Script Details
The script performs:
1. Stops all current services
2. Checks out the specified version
3. Rebuilds images
4. Starts services
### Emergency Recovery
If rollback fails:
```bash
cd /opt/motovaultpro
# Stop everything
docker compose down
# Check git history
git log --oneline -10
# Checkout known working version
git checkout <commit-hash>
# Rebuild and start
docker compose build
docker compose up -d
# Verify
docker compose ps
```
---
## Troubleshooting
### Pipeline Fails at Validate Stage
**Symptom**: `DEPLOY_PATH not found`
**Solution**:
```bash
# Create directory on runner server
sudo mkdir -p /opt/motovaultpro
sudo chown gitlab-runner:gitlab-runner /opt/motovaultpro
```
### Pipeline Fails at Build Stage
**Symptom**: Docker build errors
**Solutions**:
1. Check Dockerfile syntax
2. Verify network connectivity for npm/package downloads
3. Check disk space: `df -h`
4. Clear Docker cache: `docker system prune -a`
### Pipeline Fails at Deploy Stage
**Symptom**: Secrets injection fails
**Solutions**:
1. Verify CI/CD variables are configured correctly
2. Check variable types are set to **File** for secrets
3. Ensure variables are not restricted to specific environments
**Symptom**: Migration fails
**Solutions**:
1. Check database connectivity
2. Verify PostgreSQL is healthy: `docker logs mvp-postgres`
3. Run migrations manually:
```bash
docker compose exec mvp-backend npm run migrate
```
### Pipeline Fails at Verify Stage
**Symptom**: Container not running
**Solutions**:
1. Check container logs: `docker logs <container-name>`
2. Verify secrets are correctly mounted
3. Check for port conflicts
**Symptom**: Health check fails
**Solutions**:
1. Wait longer (service might be starting)
2. Check backend logs: `docker logs mvp-backend`
3. Verify database connection
### Services Start But Application Doesn't Work
**Check secrets are mounted**:
```bash
docker compose exec mvp-backend ls -la /run/secrets/
```
**Check configuration**:
```bash
docker compose exec mvp-backend cat /app/config/production.yml
```
**Check network connectivity**:
```bash
docker network ls
docker network inspect motovaultpro_backend
```
### Viewing Logs
```bash
# All services
docker compose logs -f
# Specific service
docker compose logs -f mvp-backend
# Last 100 lines
docker compose logs --tail 100 mvp-backend
```
---
## Maintenance
### Updating Secrets
1. Update the CI/CD variable in GitLab
2. Trigger a new pipeline (push or manual)
3. The new secrets will be injected during deployment
### Database Backups
Backups should be configured separately. Recommended approach:
```bash
# Manual backup
docker compose exec mvp-postgres pg_dump -U postgres motovaultpro > backup.sql
# Automated backup (add to cron)
0 2 * * * cd /opt/motovaultpro && docker compose exec -T mvp-postgres pg_dump -U postgres motovaultpro > /backups/mvp-$(date +\%Y\%m\%d).sql
```
### Monitoring
Consider adding:
- Prometheus metrics (Traefik already configured)
- Health check alerts
- Log aggregation
---
## Quick Reference
### Common Commands
```bash
# View pipeline status
# GitLab UI: CI/CD > Pipelines
# SSH to server
ssh user@your-server
# Navigate to project
cd /opt/motovaultpro
# View running containers
docker compose ps
# View logs
docker compose logs -f
# Restart a service
docker compose restart mvp-backend
# Run migrations manually
docker compose exec mvp-backend npm run migrate
# Access database
docker compose exec mvp-postgres psql -U postgres motovaultpro
# Health check
curl http://localhost:3001/health
```
### Important Paths
| Path | Description |
|------|-------------|
| `/opt/motovaultpro` | Application root |
| `/opt/motovaultpro/secrets/app/` | Secrets directory |
| `/opt/motovaultpro/data/documents/` | Document storage |
| `/opt/motovaultpro/config/` | Configuration files |
### Container Names
| Container | Purpose |
|-----------|---------|
| `mvp-traefik` | Reverse proxy, TLS termination |
| `mvp-frontend` | React SPA |
| `mvp-backend` | Node.js API |
| `mvp-postgres` | PostgreSQL database |
| `mvp-redis` | Redis cache |

View File

@@ -19,12 +19,15 @@ Your task is to create a plan that can be dispatched to a seprate set of AI agen
*** PERSONALITY *** *** ROLE ***
You are a senior application architect specializing in modern web applications. You are a senior devops systems reliablity engineer specializing modern web applications. Expert in linux, docker compose and gitlab.
Read README.md CLAUDE.md and AI-INDEX.md to understand this code repository in the context of this change. Read README.md CLAUDE.md and AI-INDEX.md to understand this code repository in the context of this change.
*** FEATURE *** *** ACTION ***
- The mobile site UX. - You need to create a plan for the end user to implement to make this application deployable with GitLab runners.
*** CHANGES TO IMPLEMENT *** *** CONTEXT ***
The logo for "Log Fuel" will wrap depending on what screen is highlighted. It appears that the font size changes based on if the button is selected or not. Update that so that the font doesn't change. - This is a docker compose app that is functioning in the local dev environment. It was developed with the plan to move to Kubernetes eventually but right now it's staying in docker compose. There is a secrets architecture that mirrors k8s that needs to be replicated in gitlab deployment into the docker compose environment. The gitlab version is 18.6.2 and is using the shell runtime on the gitlab runners.
*** EXECUTE ***
Create a plan the user can execute to make this app deployable with gitlab. Use brave, context7 and firecrawl if needed. Make no assumptions if your data does not have version 18.6 of gitlab. Save the plan to @docs/CICD-DEPLOY.md. Ultrathink.

75
scripts/inject-secrets.sh Executable file
View File

@@ -0,0 +1,75 @@
#!/bin/bash
# inject-secrets.sh
# Writes GitLab CI File type variables to the secrets directory
# for K8s-style secret mounting in Docker Compose
#
# GitLab File variables provide the PATH to a temporary file containing the secret.
# This script copies those files to the expected secrets/app/ location.
#
# Required GitLab CI/CD Variables (File type):
# - POSTGRES_PASSWORD
# - AUTH0_CLIENT_SECRET
# - GOOGLE_MAPS_API_KEY
# - GOOGLE_MAPS_MAP_ID
#
# Required GitLab CI/CD Variables (Variable type):
# - DEPLOY_PATH
set -euo pipefail
# Configuration
DEPLOY_PATH="${DEPLOY_PATH:-/opt/motovaultpro}"
SECRETS_DIR="${DEPLOY_PATH}/secrets/app"
echo "Injecting secrets..."
echo " Deploy path: $DEPLOY_PATH"
echo " Secrets dir: $SECRETS_DIR"
# Create secrets directory if it doesn't exist
mkdir -p "$SECRETS_DIR"
chmod 700 "$SECRETS_DIR"
# Function to inject a secret
inject_secret() {
local var_name="$1"
local file_name="$2"
local target_path="${SECRETS_DIR}/${file_name}"
# GitLab File variables contain the PATH to a temp file
local source_path="${!var_name:-}"
if [ -z "$source_path" ]; then
echo " ERROR: Variable $var_name is not set"
return 1
fi
if [ ! -f "$source_path" ]; then
echo " ERROR: File not found for $var_name at $source_path"
return 1
fi
# Copy the secret file
cp "$source_path" "$target_path"
chmod 600 "$target_path"
echo " OK: $file_name"
}
# Inject all secrets
FAILED=0
inject_secret "POSTGRES_PASSWORD" "postgres-password.txt" || FAILED=1
inject_secret "AUTH0_CLIENT_SECRET" "auth0-client-secret.txt" || FAILED=1
inject_secret "GOOGLE_MAPS_API_KEY" "google-maps-api-key.txt" || FAILED=1
inject_secret "GOOGLE_MAPS_MAP_ID" "google-maps-map-id.txt" || FAILED=1
if [ $FAILED -eq 1 ]; then
echo ""
echo "ERROR: One or more secrets failed to inject"
echo "Ensure all required CI/CD variables are configured as File type in GitLab"
exit 1
fi
echo ""
echo "Secrets injected successfully"
echo "Files created in $SECRETS_DIR:"
ls -la "$SECRETS_DIR"

105
scripts/rollback.sh Executable file
View File

@@ -0,0 +1,105 @@
#!/bin/bash
# rollback.sh
# Emergency rollback script for MotoVaultPro
#
# Usage:
# ./scripts/rollback.sh # Rollback to previous commit (HEAD~1)
# ./scripts/rollback.sh HEAD~2 # Rollback 2 commits back
# ./scripts/rollback.sh v1.0.0 # Rollback to specific tag
# ./scripts/rollback.sh abc123 # Rollback to specific commit
#
# This script:
# 1. Stops all running services
# 2. Checks out the specified version
# 3. Rebuilds Docker images
# 4. Starts all services
# 5. Runs basic health checks
set -euo pipefail
# Configuration
DEPLOY_PATH="${DEPLOY_PATH:-/opt/motovaultpro}"
ROLLBACK_TARGET="${1:-HEAD~1}"
DOCKER_COMPOSE_FILE="docker-compose.yml"
DOCKER_COMPOSE_PROD_FILE="docker-compose.prod.yml"
echo "=========================================="
echo "MotoVaultPro Rollback"
echo "=========================================="
echo "Deploy path: $DEPLOY_PATH"
echo "Target: $ROLLBACK_TARGET"
echo ""
cd "$DEPLOY_PATH"
# Confirm rollback
echo "WARNING: This will stop all services and rollback to: $ROLLBACK_TARGET"
echo ""
read -p "Continue? (y/N): " confirm
if [ "${confirm,,}" != "y" ]; then
echo "Rollback cancelled"
exit 0
fi
echo ""
echo "Step 1/5: Stopping current services..."
docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE down --timeout 30 || true
echo ""
echo "Step 2/5: Recording current version for reference..."
CURRENT_COMMIT=$(git rev-parse HEAD)
echo "Current commit: $CURRENT_COMMIT"
echo "$CURRENT_COMMIT" > .rollback-from
echo ""
echo "Step 3/5: Checking out $ROLLBACK_TARGET..."
git fetch origin
git checkout "$ROLLBACK_TARGET"
NEW_COMMIT=$(git rev-parse HEAD)
echo "Now at commit: $NEW_COMMIT"
echo ""
echo "Step 4/5: Rebuilding Docker images..."
docker compose -f $DOCKER_COMPOSE_FILE build
echo ""
echo "Step 5/5: Starting services..."
if [ -f "$DOCKER_COMPOSE_PROD_FILE" ]; then
docker compose -f $DOCKER_COMPOSE_FILE -f $DOCKER_COMPOSE_PROD_FILE up -d
else
docker compose -f $DOCKER_COMPOSE_FILE up -d
fi
echo ""
echo "Waiting for services to start..."
sleep 30
echo ""
echo "Checking service status..."
FAILED=0
for service in mvp-traefik mvp-frontend mvp-backend mvp-postgres mvp-redis; do
status=$(docker inspect --format='{{.State.Status}}' $service 2>/dev/null || echo "not found")
if [ "$status" = "running" ]; then
echo " OK: $service"
else
echo " ERROR: $service ($status)"
FAILED=1
fi
done
echo ""
echo "=========================================="
if [ $FAILED -eq 0 ]; then
echo "Rollback completed successfully!"
echo ""
echo "Rolled back from: $CURRENT_COMMIT"
echo "Now running: $NEW_COMMIT"
echo ""
echo "To undo this rollback, run:"
echo " ./scripts/rollback.sh $CURRENT_COMMIT"
else
echo "Rollback completed with errors!"
echo "Some services may not be running correctly."
echo "Check logs: docker compose logs"
fi
echo "=========================================="