Initial Commit

scripts/run-monthly-etl.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+cd "$ROOT_DIR"
+
+SERVICE_MSSQL="mvp-platform-vehicles-mssql"
+SERVICE_ETL="mvp-platform-vehicles-etl"
+MSSQL_VOLUME="platform_vehicles_mssql_data"
+
+echo "[monthly-etl] Starting MSSQL (profile: mssql-monthly)"
+docker compose --profile mssql-monthly up -d "$SERVICE_MSSQL"
+
+echo "[monthly-etl] Waiting for MSSQL to accept connections..."
+ATTEMPTS=120  # up to ~10 minutes (120 * 5s)
+SLEEP=5
+for i in $(seq 1 $ATTEMPTS); do
+  if docker compose exec -T "$SERVICE_MSSQL" /opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P 'Platform123!' -Q "SELECT 1" >/dev/null 2>&1; then
+    echo "[monthly-etl] MSSQL is ready"
+    break
+  fi
+  echo "[monthly-etl] MSSQL not ready yet... ($i/$ATTEMPTS)"
+  sleep "$SLEEP"
+done
+
+if ! docker compose exec -T "$SERVICE_MSSQL" /opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P 'Platform123!' -Q "SELECT 1" >/dev/null 2>&1; then
+  echo "[monthly-etl] ERROR: MSSQL did not become ready in time" >&2
+  exit 1
+fi
+
+cleanup() {
+  echo "[monthly-etl] Stopping and removing MSSQL container..."
+  docker compose stop "$SERVICE_MSSQL" >/dev/null 2>&1 || true
+  docker compose rm -f "$SERVICE_MSSQL" >/dev/null 2>&1 || true
+  echo "[monthly-etl] Removing MSSQL data volume ($MSSQL_VOLUME)..."
+  docker volume rm "$MSSQL_VOLUME" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "[monthly-etl] Running ETL pipeline (build-catalog)..."
+docker compose run --rm "$SERVICE_ETL" python -m etl.main build-catalog
+
+echo "[monthly-etl] Monthly ETL completed successfully."