fix: Fix imports and database bugs. Removed legacy ETL code.
This commit is contained in:
117
data/vehicle-etl/import_catalog.sh
Executable file
117
data/vehicle-etl/import_catalog.sh
Executable file
@@ -0,0 +1,117 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Vehicle Catalog CSV Bulk Import Wrapper
|
||||
#
|
||||
# Copies CSV file into mvp-backend container and executes bulk import script.
|
||||
# Handles large CSV files (250k+ rows) that fail in web import.
|
||||
#
|
||||
# Usage:
|
||||
# ./import_catalog.sh <path_to_csv_file>
|
||||
#
|
||||
# Example:
|
||||
# ./import_catalog.sh data/vehicle-etl/import/vehicle-catalog-master.csv
|
||||
#
|
||||
# Requirements:
|
||||
# - mvp-backend container must be running
|
||||
# - CSV file must have headers: year, make, model, trim
|
||||
# - Optional headers: engine_name, transmission_type
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER="mvp-backend"
|
||||
TEMP_CSV_PATH="/tmp/catalog-import.csv"
|
||||
SCRIPT_PATH="dist/features/admin/scripts/bulk-import-catalog.js"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print error and exit
|
||||
error() {
|
||||
echo -e "${RED}Error: $1${NC}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Print success message
|
||||
success() {
|
||||
echo -e "${GREEN}$1${NC}"
|
||||
}
|
||||
|
||||
# Print warning message
|
||||
warn() {
|
||||
echo -e "${YELLOW}$1${NC}"
|
||||
}
|
||||
|
||||
# Check if CSV file argument provided
|
||||
if [ $# -eq 0 ]; then
|
||||
error "No CSV file specified.
|
||||
|
||||
Usage: $0 <path_to_csv_file>
|
||||
|
||||
Example:
|
||||
$0 data/vehicle-etl/import/vehicle-catalog-master.csv"
|
||||
fi
|
||||
|
||||
CSV_FILE="$1"
|
||||
|
||||
# Validate CSV file exists
|
||||
if [ ! -f "$CSV_FILE" ]; then
|
||||
error "CSV file not found: $CSV_FILE"
|
||||
fi
|
||||
|
||||
# Get absolute path to CSV file
|
||||
CSV_FILE_ABS=$(cd "$(dirname "$CSV_FILE")" && pwd)/$(basename "$CSV_FILE")
|
||||
|
||||
# Check if container is running
|
||||
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER}$"; then
|
||||
error "Container '${CONTAINER}' is not running. Start it with: make start"
|
||||
fi
|
||||
|
||||
echo "=========================================="
|
||||
echo "Vehicle Catalog Bulk Import"
|
||||
echo "=========================================="
|
||||
echo "CSV File: $CSV_FILE_ABS"
|
||||
echo "Container: $CONTAINER"
|
||||
echo ""
|
||||
|
||||
# Copy CSV file into container
|
||||
echo "Step 1: Copying CSV file into container..."
|
||||
if ! docker cp "$CSV_FILE_ABS" "${CONTAINER}:${TEMP_CSV_PATH}"; then
|
||||
error "Failed to copy CSV file into container"
|
||||
fi
|
||||
success "CSV file copied successfully"
|
||||
echo ""
|
||||
|
||||
# Execute import script inside container
|
||||
echo "Step 2: Running import script..."
|
||||
echo ""
|
||||
|
||||
if docker exec -it "$CONTAINER" node "$SCRIPT_PATH"; then
|
||||
success "Import completed successfully!"
|
||||
IMPORT_SUCCESS=true
|
||||
else
|
||||
error "Import failed. Check the logs above for details."
|
||||
IMPORT_SUCCESS=false
|
||||
fi
|
||||
|
||||
# Cleanup: Remove temp CSV file from container
|
||||
echo ""
|
||||
echo "Step 3: Cleaning up..."
|
||||
if docker exec "$CONTAINER" rm -f "$TEMP_CSV_PATH" 2>/dev/null; then
|
||||
success "Temporary files cleaned up"
|
||||
else
|
||||
warn "Warning: Failed to cleanup temp CSV file in container"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [ "$IMPORT_SUCCESS" = true ]; then
|
||||
echo "=========================================="
|
||||
success "Import process completed successfully!"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user