Possible working ETL
This commit is contained in:
Eric Gullickson
331
apply-etl-plan.sh
Executable file
331
apply-etl-plan.sh
Executable file
@@ -0,0 +1,331 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${YELLOW}=== MotoVaultPro ETL Plan V1 - Automated Application ===${NC}"
|
||||
echo ""
|
||||
|
||||
# Get script directory
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Backup directory
|
||||
BACKUP_DIR="$SCRIPT_DIR/.etl-plan-backup-$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo -e "${GREEN}[1/4] Backing up files...${NC}"
|
||||
# Backup ETL files
|
||||
mkdir -p "$BACKUP_DIR/data/vehicle-etl"
|
||||
cp data/vehicle-etl/vehapi_fetch_snapshot.py "$BACKUP_DIR/data/vehicle-etl/" 2>/dev/null || true
|
||||
cp data/vehicle-etl/qa_validate.py "$BACKUP_DIR/data/vehicle-etl/" 2>/dev/null || true
|
||||
|
||||
# Backup backend files
|
||||
mkdir -p "$BACKUP_DIR/backend/src/features/vehicles/api"
|
||||
mkdir -p "$BACKUP_DIR/backend/src/features/vehicles/domain"
|
||||
mkdir -p "$BACKUP_DIR/backend/src/features/platform/api"
|
||||
mkdir -p "$BACKUP_DIR/backend/src/features/platform/domain"
|
||||
mkdir -p "$BACKUP_DIR/backend/src/features/platform/data"
|
||||
cp backend/src/features/vehicles/api/vehicles.controller.ts "$BACKUP_DIR/backend/src/features/vehicles/api/" 2>/dev/null || true
|
||||
cp backend/src/features/vehicles/api/vehicles.routes.ts "$BACKUP_DIR/backend/src/features/vehicles/api/" 2>/dev/null || true
|
||||
cp backend/src/features/vehicles/domain/vehicles.service.ts "$BACKUP_DIR/backend/src/features/vehicles/domain/" 2>/dev/null || true
|
||||
cp backend/src/features/platform/api/platform.controller.ts "$BACKUP_DIR/backend/src/features/platform/api/" 2>/dev/null || true
|
||||
cp backend/src/features/platform/index.ts "$BACKUP_DIR/backend/src/features/platform/" 2>/dev/null || true
|
||||
cp backend/src/features/platform/domain/vin-decode.service.ts "$BACKUP_DIR/backend/src/features/platform/domain/" 2>/dev/null || true
|
||||
cp backend/src/features/platform/data/vpic-client.ts "$BACKUP_DIR/backend/src/features/platform/data/" 2>/dev/null || true
|
||||
|
||||
# Backup frontend files
|
||||
mkdir -p "$BACKUP_DIR/frontend/src/features/vehicles/components"
|
||||
mkdir -p "$BACKUP_DIR/frontend/src/features/vehicles/api"
|
||||
mkdir -p "$BACKUP_DIR/frontend/src/features/vehicles/types"
|
||||
cp frontend/src/features/vehicles/components/VehicleForm.tsx "$BACKUP_DIR/frontend/src/features/vehicles/components/" 2>/dev/null || true
|
||||
cp frontend/src/features/vehicles/api/vehicles.api.ts "$BACKUP_DIR/frontend/src/features/vehicles/api/" 2>/dev/null || true
|
||||
cp frontend/src/features/vehicles/types/vehicles.types.ts "$BACKUP_DIR/frontend/src/features/vehicles/types/" 2>/dev/null || true
|
||||
|
||||
echo -e "${GREEN} Backup created at: $BACKUP_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}[2/4] Applying ETL Configuration Changes...${NC}"
|
||||
|
||||
# Update vehapi_fetch_snapshot.py
|
||||
if [ -f "data/vehicle-etl/vehapi_fetch_snapshot.py" ]; then
|
||||
echo " - Updating DEFAULT_MIN_YEAR from 1980 to 2017..."
|
||||
sed -i.bak 's/DEFAULT_MIN_YEAR = 1980/DEFAULT_MIN_YEAR = 2017/g' data/vehicle-etl/vehapi_fetch_snapshot.py
|
||||
sed -i.bak 's/default env MIN_YEAR or 1980/default env MIN_YEAR or 2017/g' data/vehicle-etl/vehapi_fetch_snapshot.py
|
||||
rm data/vehicle-etl/vehapi_fetch_snapshot.py.bak
|
||||
echo -e " ${GREEN}✓${NC} vehapi_fetch_snapshot.py updated"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} vehapi_fetch_snapshot.py not found"
|
||||
fi
|
||||
|
||||
# Update qa_validate.py
|
||||
if [ -f "data/vehicle-etl/qa_validate.py" ]; then
|
||||
echo " - Updating year range validation from 1980-2022 to 2017-2022..."
|
||||
sed -i.bak 's/year < 1980 OR year > 2022/year < 2017 OR year > 2022/g' data/vehicle-etl/qa_validate.py
|
||||
rm data/vehicle-etl/qa_validate.py.bak
|
||||
echo -e " ${GREEN}✓${NC} qa_validate.py updated"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} qa_validate.py not found"
|
||||
fi
|
||||
|
||||
# Delete old ETL documentation
|
||||
echo " - Removing old ETL documentation..."
|
||||
rm -f data/vehicle-etl/ETL-FIX-V2.md
|
||||
rm -f data/vehicle-etl/ETL-FIXES.md
|
||||
rm -f data/vehicle-etl/ETL-VEHAPI-PLAN.md
|
||||
rm -f data/vehicle-etl/ETL-VEHAPI-REDESIGN.md
|
||||
echo -e " ${GREEN}✓${NC} Old ETL documentation removed"
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}[3/4] Applying Backend Changes...${NC}"
|
||||
|
||||
# Update vehicles.controller.ts MIN_YEAR
|
||||
if [ -f "backend/src/features/vehicles/api/vehicles.controller.ts" ]; then
|
||||
echo " - Updating MIN_YEAR from 1980 to 2017..."
|
||||
sed -i.bak 's/private static readonly MIN_YEAR = 1980;/private static readonly MIN_YEAR = 2017;/g' backend/src/features/vehicles/api/vehicles.controller.ts
|
||||
rm backend/src/features/vehicles/api/vehicles.controller.ts.bak
|
||||
echo -e " ${GREEN}✓${NC} vehicles.controller.ts MIN_YEAR updated"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} vehicles.controller.ts not found"
|
||||
fi
|
||||
|
||||
# Remove VIN decode route from vehicles.routes.ts
|
||||
if [ -f "backend/src/features/vehicles/api/vehicles.routes.ts" ]; then
|
||||
echo " - Removing VIN decode route..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('backend/src/features/vehicles/api/vehicles.routes.ts', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove the decode-vin route (multi-line pattern)
|
||||
pattern = r"\n\s*fastify\.post<\{ Body: \{ vin: string \} \}>\('/vehicles/decode-vin',\s*\{[^}]*\}\);"
|
||||
content = re.sub(pattern, '', content, flags=re.DOTALL)
|
||||
|
||||
with open('backend/src/features/vehicles/api/vehicles.routes.ts', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} VIN decode route removed from vehicles.routes.ts"
|
||||
else
|
||||
echo -e " ${YELLOW}⚠${NC} vehicles.routes.ts not found (may not exist)"
|
||||
fi
|
||||
|
||||
# Remove VIN decode from vehicles.service.ts
|
||||
if [ -f "backend/src/features/vehicles/domain/vehicles.service.ts" ]; then
|
||||
echo " - Removing VIN decode logic from vehicles.service.ts..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('backend/src/features/vehicles/domain/vehicles.service.ts', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove import statement for vin-decode
|
||||
content = re.sub(r"import\s*\{[^}]*getVINDecodeService[^}]*\}[^;]*;?\n?", '', content)
|
||||
content = re.sub(r"import\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
|
||||
|
||||
# Remove decodeVIN method if it exists
|
||||
pattern = r'\n\s*async decodeVIN\([^)]*\)[^{]*\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}\n'
|
||||
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
|
||||
|
||||
# Remove VIN decode logic from createVehicle method (between specific comments or patterns)
|
||||
# This is more conservative - just remove obvious decode blocks
|
||||
content = re.sub(r'\n\s*// VIN decode logic.*?(?=\n\s*(?:const|return|if|//|$))', '', content, flags=re.DOTALL)
|
||||
|
||||
with open('backend/src/features/vehicles/domain/vehicles.service.ts', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} VIN decode logic removed from vehicles.service.ts"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} vehicles.service.ts not found"
|
||||
fi
|
||||
|
||||
# Remove VIN decode from platform.controller.ts
|
||||
if [ -f "backend/src/features/platform/api/platform.controller.ts" ]; then
|
||||
echo " - Removing VIN decode from platform.controller.ts..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('backend/src/features/platform/api/platform.controller.ts', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove VINDecodeService import
|
||||
content = re.sub(r"import\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
|
||||
|
||||
# Remove VINDecodeService from constructor
|
||||
content = re.sub(r',?\s*private\s+vinDecodeService:\s*VINDecodeService', '', content)
|
||||
content = re.sub(r'private\s+vinDecodeService:\s*VINDecodeService\s*,?', '', content)
|
||||
|
||||
# Remove decodeVIN method
|
||||
pattern = r'\n\s*async decodeVIN\([^)]*\)[^{]*\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}\n'
|
||||
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
|
||||
|
||||
with open('backend/src/features/platform/api/platform.controller.ts', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} VIN decode removed from platform.controller.ts"
|
||||
else
|
||||
echo -e " ${YELLOW}⚠${NC} platform.controller.ts not found"
|
||||
fi
|
||||
|
||||
# Remove VINDecodeService from platform/index.ts
|
||||
if [ -f "backend/src/features/platform/index.ts" ]; then
|
||||
echo " - Removing VINDecodeService from platform/index.ts..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('backend/src/features/platform/index.ts', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove VINDecodeService import
|
||||
content = re.sub(r"import\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
|
||||
|
||||
# Remove VINDecodeService export
|
||||
content = re.sub(r"export\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
|
||||
content = re.sub(r",\s*VINDecodeService\s*", '', content)
|
||||
content = re.sub(r"VINDecodeService\s*,\s*", '', content)
|
||||
|
||||
# Remove getVINDecodeService function
|
||||
pattern = r'\n\s*export\s+function\s+getVINDecodeService\([^)]*\)[^{]*\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}\n'
|
||||
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
|
||||
|
||||
with open('backend/src/features/platform/index.ts', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} VINDecodeService removed from platform/index.ts"
|
||||
else
|
||||
echo -e " ${YELLOW}⚠${NC} platform/index.ts not found"
|
||||
fi
|
||||
|
||||
# Delete VIN decode service files
|
||||
echo " - Deleting VIN decode service files..."
|
||||
rm -f backend/src/features/platform/domain/vin-decode.service.ts
|
||||
rm -f backend/src/features/platform/data/vpic-client.ts
|
||||
echo -e " ${GREEN}✓${NC} VIN decode service files deleted"
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}[4/4] Applying Frontend Changes...${NC}"
|
||||
|
||||
# Update VehicleForm.tsx
|
||||
if [ -f "frontend/src/features/vehicles/components/VehicleForm.tsx" ]; then
|
||||
echo " - Removing VIN decode UI from VehicleForm.tsx..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('frontend/src/features/vehicles/components/VehicleForm.tsx', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove decodingVIN and decodeSuccess state variables
|
||||
content = re.sub(r"\n\s*const \[decodingVIN, setDecodingVIN\] = useState\(false\);", '', content)
|
||||
content = re.sub(r"\n\s*const \[decodeSuccess, setDecodeSuccess\] = useState\(false\);", '', content)
|
||||
content = re.sub(r"\n\s*const \[decodeSuccess, setDecodeSuccess\] = useState<boolean \| null>\(null\);", '', content)
|
||||
|
||||
# Remove watchedVIN if it exists
|
||||
content = re.sub(r"\n\s*const watchedVIN = watch\('vin'\);", '', content)
|
||||
|
||||
# Remove handleDecodeVIN function
|
||||
pattern = r'\n\s*const handleDecodeVIN = async \(\) => \{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\};'
|
||||
content = re.sub(pattern, '', content, flags=re.DOTALL)
|
||||
|
||||
# Update helper text
|
||||
content = re.sub(
|
||||
r'Enter VIN to auto-fill vehicle details OR manually select from dropdowns below',
|
||||
'Enter vehicle VIN (optional)',
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'Enter VIN to auto-populate fields',
|
||||
'Enter vehicle VIN (optional)',
|
||||
content
|
||||
)
|
||||
|
||||
# Remove the Decode VIN button and surrounding div
|
||||
# Pattern 1: div with flex layout containing input and button
|
||||
pattern1 = r'<div className="flex flex-col sm:flex-row gap-2">\s*<input\s+\{\.\.\.register\(\'vin\'\)\}[^>]*>\s*</input>\s*<Button[^>]*onClick=\{handleDecodeVIN\}[^>]*>.*?</Button>\s*</div>'
|
||||
replacement1 = r'<input {...register(\'vin\')} className="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-primary-500 text-base" placeholder="Enter 17-character VIN (optional if License Plate provided)" style={{ fontSize: \'16px\' }} />'
|
||||
content = re.sub(pattern1, replacement1, content, flags=re.DOTALL)
|
||||
|
||||
# Pattern 2: self-closing input with button after
|
||||
pattern2 = r'<input\s+\{\.\.\.register\(\'vin\'\)\}[^/]*/>\s*<Button[^>]*onClick=\{handleDecodeVIN\}[^>]*>.*?</Button>'
|
||||
content = re.sub(pattern2, replacement1, content, flags=re.DOTALL)
|
||||
|
||||
# Remove decode success message
|
||||
content = re.sub(r'\n\s*\{decodeSuccess && \([^)]*\)\}', '', content, flags=re.DOTALL)
|
||||
content = re.sub(r'\n\s*<p className="mt-1 text-sm text-green-600">VIN decoded successfully.*?</p>', '', content, flags=re.DOTALL)
|
||||
|
||||
with open('frontend/src/features/vehicles/components/VehicleForm.tsx', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} VIN decode UI removed from VehicleForm.tsx"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} VehicleForm.tsx not found"
|
||||
fi
|
||||
|
||||
# Update vehicles.api.ts
|
||||
if [ -f "frontend/src/features/vehicles/api/vehicles.api.ts" ]; then
|
||||
echo " - Removing decodeVIN method from vehicles.api.ts..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('frontend/src/features/vehicles/api/vehicles.api.ts', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove VINDecodeResponse from imports
|
||||
content = re.sub(r',\s*VINDecodeResponse', '', content)
|
||||
content = re.sub(r'VINDecodeResponse\s*,', '', content)
|
||||
|
||||
# Remove decodeVIN method
|
||||
pattern = r'\n\s*decodeVIN:\s*async\s*\([^)]*\)[^{]*\{[^}]*\},?'
|
||||
content = re.sub(pattern, '', content, flags=re.DOTALL)
|
||||
|
||||
# Remove trailing comma before closing brace if present
|
||||
content = re.sub(r',(\s*\};?\s*$)', r'\1', content)
|
||||
|
||||
with open('frontend/src/features/vehicles/api/vehicles.api.ts', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} decodeVIN method removed from vehicles.api.ts"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} vehicles.api.ts not found"
|
||||
fi
|
||||
|
||||
# Update vehicles.types.ts
|
||||
if [ -f "frontend/src/features/vehicles/types/vehicles.types.ts" ]; then
|
||||
echo " - Removing VINDecodeResponse interface from vehicles.types.ts..."
|
||||
python3 << 'PYTHON_EOF'
|
||||
import re
|
||||
with open('frontend/src/features/vehicles/types/vehicles.types.ts', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove VINDecodeResponse interface
|
||||
pattern = r'\n\s*export interface VINDecodeResponse \{[^}]*\}\n?'
|
||||
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
|
||||
|
||||
with open('frontend/src/features/vehicles/types/vehicles.types.ts', 'w') as f:
|
||||
f.write(content)
|
||||
PYTHON_EOF
|
||||
echo -e " ${GREEN}✓${NC} VINDecodeResponse interface removed from vehicles.types.ts"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} vehicles.types.ts not found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Changes Applied Successfully ===${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Verification Commands:${NC}"
|
||||
echo ""
|
||||
echo "1. ETL Configuration:"
|
||||
echo " cd data/vehicle-etl && python3 -c \"import vehapi_fetch_snapshot; print(vehapi_fetch_snapshot.DEFAULT_MIN_YEAR)\""
|
||||
echo " Expected output: 2017"
|
||||
echo ""
|
||||
echo "2. Backend TypeScript:"
|
||||
echo " cd backend && npx tsc --noEmit"
|
||||
echo " Expected: No errors"
|
||||
echo ""
|
||||
echo "3. Frontend TypeScript:"
|
||||
echo " cd frontend && npx tsc --noEmit"
|
||||
echo " Expected: No errors"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Next Steps:${NC}"
|
||||
echo "1. Run the verification commands above"
|
||||
echo "2. If all checks pass, proceed with Agent 4 (ETL Execution)"
|
||||
echo "3. If there are issues, restore from backup: $BACKUP_DIR"
|
||||
echo ""
|
||||
echo -e "${GREEN}Backup location: $BACKUP_DIR${NC}"
|
||||
Reference in New Issue
Block a user