fix: Fix imports and database bugs. Removed legacy ETL code.

This commit is contained in:
Eric Gullickson
2025-12-27 12:07:24 -06:00
parent 0d9edbe761
commit bfb0c23ae1
30 changed files with 239174 additions and 4441 deletions

View File

@@ -0,0 +1,117 @@
#!/bin/bash
#
# Vehicle Catalog CSV Bulk Import Wrapper
#
# Copies CSV file into mvp-backend container and executes bulk import script.
# Handles large CSV files (250k+ rows) that fail in web import.
#
# Usage:
# ./import_catalog.sh <path_to_csv_file>
#
# Example:
# ./import_catalog.sh data/vehicle-etl/import/vehicle-catalog-master.csv
#
# Requirements:
# - mvp-backend container must be running
# - CSV file must have headers: year, make, model, trim
# - Optional headers: engine_name, transmission_type
#
set -euo pipefail
CONTAINER="mvp-backend"
TEMP_CSV_PATH="/tmp/catalog-import.csv"
SCRIPT_PATH="dist/features/admin/scripts/bulk-import-catalog.js"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Print error and exit
error() {
echo -e "${RED}Error: $1${NC}" >&2
exit 1
}
# Print success message
success() {
echo -e "${GREEN}$1${NC}"
}
# Print warning message
warn() {
echo -e "${YELLOW}$1${NC}"
}
# Check if CSV file argument provided
if [ $# -eq 0 ]; then
error "No CSV file specified.
Usage: $0 <path_to_csv_file>
Example:
$0 data/vehicle-etl/import/vehicle-catalog-master.csv"
fi
CSV_FILE="$1"
# Validate CSV file exists
if [ ! -f "$CSV_FILE" ]; then
error "CSV file not found: $CSV_FILE"
fi
# Get absolute path to CSV file
CSV_FILE_ABS=$(cd "$(dirname "$CSV_FILE")" && pwd)/$(basename "$CSV_FILE")
# Check if container is running
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER}$"; then
error "Container '${CONTAINER}' is not running. Start it with: make start"
fi
echo "=========================================="
echo "Vehicle Catalog Bulk Import"
echo "=========================================="
echo "CSV File: $CSV_FILE_ABS"
echo "Container: $CONTAINER"
echo ""
# Copy CSV file into container
echo "Step 1: Copying CSV file into container..."
if ! docker cp "$CSV_FILE_ABS" "${CONTAINER}:${TEMP_CSV_PATH}"; then
error "Failed to copy CSV file into container"
fi
success "CSV file copied successfully"
echo ""
# Execute import script inside container
echo "Step 2: Running import script..."
echo ""
if docker exec -it "$CONTAINER" node "$SCRIPT_PATH"; then
success "Import completed successfully!"
IMPORT_SUCCESS=true
else
error "Import failed. Check the logs above for details."
IMPORT_SUCCESS=false
fi
# Cleanup: Remove temp CSV file from container
echo ""
echo "Step 3: Cleaning up..."
if docker exec "$CONTAINER" rm -f "$TEMP_CSV_PATH" 2>/dev/null; then
success "Temporary files cleaned up"
else
warn "Warning: Failed to cleanup temp CSV file in container"
fi
echo ""
if [ "$IMPORT_SUCCESS" = true ]; then
echo "=========================================="
success "Import process completed successfully!"
echo "=========================================="
exit 0
else
exit 1
fi