Possible working ETL

This commit is contained in:
Eric Gullickson
2025-12-15 18:19:55 -06:00
parent 1fc69b7779
commit 1e599e334f
110 changed files with 4843 additions and 2078706 deletions

7
.gitignore vendored
View File

@@ -10,8 +10,13 @@ coverage/
.idea/ .idea/
*.swp *.swp
*.swo *.swo
#SQL Import Files .venv
#Data Import Files
data/make-model-import/output/*.sql data/make-model-import/output/*.sql
data/make-model-import/vehapi.key
data/make-model-import/snapshots/*
!data/make-model-import/snapshots/.gitkeep
# K8s-aligned configuration and secret mounts (real files ignored; examples committed) # K8s-aligned configuration and secret mounts (real files ignored; examples committed)
config/** config/**

View File

@@ -1,518 +0,0 @@
# ETL Fix V2: Year-Accurate Vehicle Dropdown Data
## Executive Summary
This document provides a complete implementation plan for fixing the vehicle dropdown ETL to produce year-accurate data. The fix addresses impossible year/trim combinations (e.g., "1992 Corvette Z06") by using the NHTSA VPIC API for authoritative Year/Make/Model validation and automobiles.json for evidence-based trim data with year ranges.
---
## Problem Statement
### Current Issues
1. **Year-inaccurate trims**: The `makes-filter/*.json` files contain ALL trims ever made for a model, applied to EVERY year
2. **Impossible combinations**: Users can select "1992 Corvette Z06" (Z06 didn't exist until 2001)
3. **Data bloat**: 400 records for 2000 Corvette with 20 trims instead of ~3-4
### Root Cause
The `makes-filter/*.json` data structure does NOT have year-specific trims. Example from `chevrolet.json`:
```json
{
"year": "2025",
"models": [{
"name": "corvette",
"submodels": ["LT", "35th Anniversary Edition", "427", "Z06", "ZR1", ...]
}]
}
```
The same `submodels` array is repeated for every year, making ALL trims appear valid for ALL years.
---
## Solution Architecture
### Data Sources (Priority Order)
1. **NHTSA VPIC API** - Authoritative Year/Make/Model validation
2. **automobiles.json** - Primary trim source with year-range evidence
3. **makes-filter/*.json** - Engine data enrichment
4. **Defaults** - "Base" trim, "Gas" engine, "Manual"/"Automatic" transmission
### Year Range
- Minimum: 1990
- Maximum: 2026
---
## Implementation Steps
### Phase 1: Create NHTSA Data Fetcher
**Create file:** `data/make-model-import/nhtsa_fetch.py`
```python
#!/usr/bin/env python3
"""
NHTSA VPIC API Data Fetcher
Fetches authoritative Year/Make/Model data from the US government database.
"""
import json
import os
import time
from pathlib import Path
from typing import Dict, List, Set
import urllib.request
import urllib.error
class NHTSAFetcher:
BASE_URL = "https://vpic.nhtsa.dot.gov/api/vehicles"
CACHE_DIR = Path("nhtsa_cache")
OUTPUT_FILE = Path("nhtsa_vehicles.json")
def __init__(self):
self.min_year = int(os.getenv("MIN_YEAR", "1990"))
self.max_year = int(os.getenv("MAX_YEAR", "2026"))
self.request_delay = 0.1 # 100ms between requests
# Makes we care about (from makes-filter)
self.target_makes = self._load_target_makes()
def _load_target_makes(self) -> Set[str]:
"""Load makes from makes-filter directory."""
makes_dir = Path("makes-filter")
makes = set()
for f in makes_dir.glob("*.json"):
make_name = f.stem.replace("_", " ").title()
makes.add(make_name)
return makes
def fetch_url(self, url: str) -> dict:
"""Fetch JSON from URL with error handling."""
try:
with urllib.request.urlopen(url, timeout=30) as response:
return json.loads(response.read().decode())
except urllib.error.URLError as e:
print(f" Error fetching {url}: {e}")
return {"Results": []}
def get_all_makes(self) -> List[Dict]:
"""Fetch all makes for passenger cars and trucks."""
makes = []
for vehicle_type in ["car", "truck"]:
url = f"{self.BASE_URL}/GetMakesForVehicleType/{vehicle_type}?format=json"
data = self.fetch_url(url)
makes.extend(data.get("Results", []))
return makes
def get_models_for_make_year(self, make: str, year: int) -> List[str]:
"""Fetch models for a specific make and year."""
cache_file = self.CACHE_DIR / f"{make.lower().replace(' ', '_')}_{year}.json"
# Check cache first
if cache_file.exists():
with open(cache_file) as f:
return json.load(f)
url = f"{self.BASE_URL}/GetModelsForMakeYear/make/{make}/modelyear/{year}?format=json"
time.sleep(self.request_delay)
data = self.fetch_url(url)
models = list(set(r.get("Model_Name", "") for r in data.get("Results", []) if r.get("Model_Name")))
# Cache result
self.CACHE_DIR.mkdir(exist_ok=True)
with open(cache_file, "w") as f:
json.dump(models, f)
return models
def fetch_all_data(self) -> Dict:
"""Fetch all Year/Make/Model data."""
print("Fetching NHTSA data...")
# Filter to target makes
all_makes = self.get_all_makes()
target_make_names = [m["MakeName"] for m in all_makes
if m["MakeName"].title() in self.target_makes
or m["MakeName"].upper() in ["BMW", "GMC", "RAM"]]
print(f"Found {len(target_make_names)} matching makes")
result = {}
for year in range(self.min_year, self.max_year + 1):
result[str(year)] = {}
for make in target_make_names:
models = self.get_models_for_make_year(make, year)
if models:
# Normalize make name
make_normalized = make.title()
if make.upper() in ["BMW", "GMC", "RAM"]:
make_normalized = make.upper()
result[str(year)][make_normalized] = sorted(models)
print(f" Year {year}: {sum(len(v) for v in result[str(year)].values())} models")
# Save output
with open(self.OUTPUT_FILE, "w") as f:
json.dump(result, f, indent=2)
print(f"Saved to {self.OUTPUT_FILE}")
return result
if __name__ == "__main__":
NHTSAFetcher().fetch_all_data()
```
### Phase 2: Refactor ETL Script
**Modify file:** `data/make-model-import/etl_generate_sql.py`
Key changes:
#### 2.1 Load NHTSA data as primary source
```python
def load_nhtsa_data(self):
"""Load NHTSA Year/Make/Model data."""
nhtsa_file = Path("nhtsa_vehicles.json")
if not nhtsa_file.exists():
raise FileNotFoundError("Run nhtsa_fetch.py first to generate nhtsa_vehicles.json")
with open(nhtsa_file) as f:
self.nhtsa_data = json.load(f)
print(f" Loaded NHTSA data for {len(self.nhtsa_data)} years")
```
#### 2.2 Build trim evidence from automobiles.json
```python
def build_trim_evidence(self):
"""
Parse automobiles.json to build year-range evidence for trims.
"""
self.trim_evidence: Dict[Tuple[str, str], List[Dict]] = defaultdict(list)
brand_lookup = {b.get("id"): self.get_canonical_make_name(b.get("name", ""))
for b in self.brands_data}
for auto in self.automobiles_data:
brand_id = auto.get("brand_id")
make = brand_lookup.get(brand_id)
if not make:
continue
name = auto.get("name", "")
year_range = self.parse_year_range_from_name(name)
if not year_range:
continue
year_start, year_end = year_range
# Extract model and trim from name
model, trim = self.extract_model_trim_from_name(name, make)
if not model:
continue
self.trim_evidence[(make, model)].append({
"trim": trim or "Base",
"year_start": year_start,
"year_end": year_end,
"source_name": name
})
print(f" Built trim evidence for {len(self.trim_evidence)} make/model combinations")
def extract_model_trim_from_name(self, name: str, make: str) -> Tuple[Optional[str], Optional[str]]:
"""
Extract model and trim from automobile name.
Examples:
"CHEVROLET Corvette Z06 2021-Present" -> ("Corvette", "Z06")
"2020 Chevrolet Corvette C8 Stingray" -> ("Corvette", "Stingray")
"FORD F-150 Raptor 2021-Present" -> ("F-150", "Raptor")
"""
# Remove make prefix
clean = re.sub(rf"^{re.escape(make)}\s+", "", name, flags=re.IGNORECASE)
# Remove year/range patterns
clean = re.sub(r"\d{4}(-\d{4}|-Present)?", "", clean)
# Remove common suffixes
clean = re.sub(r"\s*(Photos|engines|full specs|&).*$", "", clean, flags=re.IGNORECASE)
# Clean up extra spaces
clean = " ".join(clean.split())
# Try to match against known models
known_models = self.known_models_by_make.get(make, set())
for model in sorted(known_models, key=len, reverse=True):
pattern = re.compile(rf"^{re.escape(model)}\b\s*(.*)", re.IGNORECASE)
match = pattern.match(clean)
if match:
trim = match.group(1).strip()
# Remove generation codes like C5, C6, C7, C8
trim = re.sub(r"^C\d+\s*", "", trim)
return (model, trim if trim else None)
return (None, None)
```
#### 2.3 New trim resolution logic
```python
def get_trims_for_vehicle(self, year: int, make: str, model: str) -> List[str]:
"""
Get valid trims for a year/make/model combination.
Uses evidence from automobiles.json, falls back to "Base".
"""
evidence = self.trim_evidence.get((make, model), [])
valid_trims = set()
for entry in evidence:
if entry['year_start'] <= year <= entry['year_end']:
valid_trims.add(entry['trim'])
# Always include "Base" as an option
valid_trims.add("Base")
return sorted(valid_trims)
```
#### 2.4 Updated vehicle record building
```python
def build_vehicle_records(self):
"""Build vehicle records using NHTSA for Y/M/M, evidence for trims."""
print("\n Building vehicle option records...")
records = []
for year_str, makes in self.nhtsa_data.items():
year = int(year_str)
if year < self.min_year or year > self.max_year:
continue
for make, models in makes.items():
for model in models:
# Get valid trims from evidence
trims = self.get_trims_for_vehicle(year, make, model)
# Get engines from makes-filter (or default)
engines = self.get_engines_for_vehicle(year, make, model)
# Default transmissions
transmissions = ["Manual", "Automatic"]
for trim in trims:
for engine in engines:
for trans in transmissions:
records.append({
"year": year,
"make": make,
"model": model,
"trim": trim,
"engine_name": engine,
"trans_name": trans
})
# Deduplicate
unique_set = set()
deduped = []
for r in records:
key = (r["year"], r["make"].lower(), r["model"].lower(),
r["trim"].lower(), r["engine_name"].lower(), r["trans_name"].lower())
if key not in unique_set:
unique_set.add(key)
deduped.append(r)
self.vehicle_records = deduped
print(f" Vehicle records: {len(self.vehicle_records):,}")
def get_engines_for_vehicle(self, year: int, make: str, model: str) -> List[str]:
"""Get engines from makes-filter or use defaults."""
# Try to find in makes-filter data
for baseline in self.baseline_records:
if (baseline['year'] == year and
baseline['make'].lower() == make.lower() and
baseline['model'].lower() == model.lower()):
engines = []
for trim_data in baseline.get('trims', []):
engines.extend(trim_data.get('engines', []))
if engines:
return list(set(engines))
# Default based on make/model patterns
model_lower = model.lower()
if 'electric' in model_lower or 'ev' in model_lower or 'lightning' in model_lower:
return ["Electric"]
return ["Gas"]
```
### Phase 3: Update Import Script
**Modify file:** `data/make-model-import/import_data.sh`
Add NHTSA cache check:
```bash
#!/bin/bash
# Import generated SQL files into PostgreSQL database
set -e
echo "=========================================="
echo " Automotive Database Import"
echo "=========================================="
# Check NHTSA cache freshness
NHTSA_FILE="nhtsa_vehicles.json"
CACHE_AGE_DAYS=30
if [ ! -f "$NHTSA_FILE" ]; then
echo "NHTSA data not found. Fetching..."
python3 nhtsa_fetch.py
elif [ $(find "$NHTSA_FILE" -mtime +$CACHE_AGE_DAYS 2>/dev/null | wc -l) -gt 0 ]; then
echo "NHTSA cache is stale (>$CACHE_AGE_DAYS days). Refreshing..."
python3 nhtsa_fetch.py
else
echo "Using cached NHTSA data"
fi
# Continue with existing import logic...
```
### Phase 4: Update QA Validation
**Modify file:** `data/make-model-import/qa_validate.py`
Add invalid combination checks:
```python
def check_invalid_combinations():
"""Verify known invalid combinations do not exist."""
invalid_combos = [
# (year, make, model, trim) - known to be invalid
(1992, 'Chevrolet', 'Corvette', 'Z06'), # Z06 started 2001
(2000, 'Chevrolet', 'Corvette', '35th Anniversary Edition'), # Was 1988
(2000, 'Chevrolet', 'Corvette', 'Stingray'), # Stingray started 2014
(1995, 'Ford', 'Mustang', 'Mach-E'), # Mach-E is 2021+
]
issues = []
for year, make, model, trim in invalid_combos:
query = f"""
SELECT COUNT(*) FROM vehicle_options
WHERE year = {year}
AND make = '{make}'
AND model = '{model}'
AND trim = '{trim}'
"""
count = int(run_psql(query).strip())
if count > 0:
issues.append(f"Invalid combo found: {year} {make} {model} {trim}")
return issues
def check_trim_coverage():
"""Report on trim coverage statistics."""
query = """
SELECT
COUNT(DISTINCT (year, make, model)) as total_models,
COUNT(DISTINCT (year, make, model)) FILTER (WHERE trim = 'Base') as base_only,
COUNT(DISTINCT (year, make, model)) FILTER (WHERE trim != 'Base') as has_specific_trims
FROM vehicle_options
"""
result = run_psql(query).strip()
print(f"Trim coverage: {result}")
```
---
## Files Summary
| File | Action | Purpose |
|------|--------|---------|
| `data/make-model-import/nhtsa_fetch.py` | CREATE | Fetch Year/Make/Model from NHTSA API |
| `data/make-model-import/etl_generate_sql.py` | MODIFY | Use NHTSA data, evidence-based trims |
| `data/make-model-import/import_data.sh` | MODIFY | Add NHTSA cache refresh |
| `data/make-model-import/qa_validate.py` | MODIFY | Add invalid combo checks |
---
## Execution Order
```bash
# 1. Navigate to ETL directory
cd data/make-model-import
# 2. Fetch NHTSA data (creates nhtsa_vehicles.json)
python3 nhtsa_fetch.py
# 3. Generate SQL files
python3 etl_generate_sql.py
# 4. Import to database
./import_data.sh
# 5. Validate results
python3 qa_validate.py
```
---
## Expected Results
### Before
- 2000 Corvette: 400 records, 20 trims (most invalid)
- Total records: ~1,675,335
- Many impossible year/trim combinations
### After
- 2000 Corvette: ~8 records (Base, Coupe, Convertible)
- 2015 Corvette: ~20 records (Stingray, Z06, Grand Sport, Base)
- Total records: ~400,000-600,000
- No invalid year/trim combinations
### Validation Checks
1. No 1992 Corvette Z06
2. No 2000 Corvette Stingray
3. No 1995 Mustang Mach-E
4. Year range: 1990-2026
---
## Data Source Coverage
**automobiles.json trim coverage (samples):**
| Model | Entries | Trims Found |
|-------|---------|-------------|
| Civic | 67 | Si, Type R, eHEV, Sedan, Hatchback |
| Mustang | 38 | GT, Dark Horse, Mach-E GT, GTD |
| Accord | 35 | Sedan, Coupe (various years) |
| Corvette | 31 | Z06, ZR1, Stingray, Grand Sport |
| Camaro | 29 | ZL1, Convertible, Coupe |
| F-150 | 19 | Lightning, Raptor, Tremor |
---
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `MIN_YEAR` | 1990 | Minimum year to include |
| `MAX_YEAR` | 2026 | Maximum year to include |
---
## Troubleshooting
### NHTSA API Rate Limiting
The script includes 100ms delays between requests. If you encounter rate limiting:
- Increase `request_delay` in `nhtsa_fetch.py`
- Use cached data in `nhtsa_cache/` directory
### Missing Models
If NHTSA returns fewer models than expected:
- Check if the make name matches exactly
- Some brands (BMW, GMC) need uppercase handling
- Verify the year range is supported (NHTSA has data back to ~1995)
### Cache Refresh
To force refresh NHTSA data:
```bash
rm nhtsa_vehicles.json
rm -rf nhtsa_cache/
python3 nhtsa_fetch.py
```

View File

@@ -1,282 +0,0 @@
# ETL Fixes Plan (MultiAgent Dispatch) — Vehicle Dropdown Data
## Purpose
Fix the ETL that populates vehicle dropdown data so it is:
- Clean (no duplicate dimension rows, no duplicate fact rows).
- Year-accurate for trims (no “impossible” year/make/model/trim combinations).
- Rerunnable across environments.
- Limited to a configurable year window (default **20002026**) with **no API-level filtering changes**.
This plan is written to be dispatched to multiple AI agents working in parallel.
## Scope
Backend vehicle dropdowns (Year → Make → Model → Trim → Engine → Transmission).
In-scope:
- ETL logic and output SQL generation (`data/make-model-import/etl_generate_sql.py`).
- Import script behavior (`data/make-model-import/import_data.sh`).
- ETL schema migration used by the import (`data/make-model-import/migrations/001_create_vehicle_database.sql`).
- Data quality validation harness (new script(s)).
- Documentation updates for rerun workflow.
Out-of-scope:
- Any API filtering logic changes. The API must continue to reflect whatever data exists in the DB.
- Network calls or new scraping. **Use local scraped data only.**
## Current Data Contract (as used by backend)
Backend dropdowns currently query:
- `public.vehicle_options`
- `public.engines` (joined by `engine_id`)
- `public.transmissions` (joined by `transmission_id`)
Primary call sites:
- `backend/src/features/platform/data/vehicle-data.repository.ts`
- `backend/src/features/platform/domain/vehicle-data.service.ts`
- Dropdown routes: `backend/src/features/vehicles/api/vehicles.routes.ts`
## Requirements (Confirmed)
### Year range behavior
- Data outside the configured year window must **not be loaded**.
- Default year window: **20002026** (configurable).
- No API changes to filter years.
### Missing data defaults
- If **no trim exists**, map to trim `"Base"`.
- If **no detailed engine spec exists**, default to one of: `Gas` / `Diesel` / `Electric` / `Hybrid`.
- If local scraped data indicates EV → show `Electric`.
- If indicates Diesel → show `Diesel`.
- If indicates Hybrid (including mild / plug-in) → show `Hybrid`.
- Else default → `Gas`.
- If **no specific transmission data** exists for a `(year, make, model)`, show **both** `Manual` and `Automatic`.
- If a detailed engine spec is known, **always use the detailed engine spec** (do not replace it with the fuel-type default label).
### Transmission granularity
- Transmission dropdown should be correct at the `(year, make, model)` level (trim-specific not required).
## Observed Defects (Root Causes)
### 1) Massive duplicate dimension rows
Examples:
- `data/make-model-import/output/02_transmissions.sql` contains repeated values like:
- `(1,'1-Speed Automatic')`, `(2,'1-Speed Automatic')`, …
Reason:
- ETL dedupes transmissions on a raw tuple `(gearbox_string, speed, drive_type)` but *stores* only a simplified display string, so many distinct raw tuples collapse to the same output `type`.
Similarly for engines:
- `data/make-model-import/output/01_engines.sql` has many repeated engine display names.
Reason:
- ETL assigns IDs per raw scraped engine record (30,066), even though the UI-facing engine name collapses to far fewer distinct names.
### 2) Inaccurate year/make/model/trim mappings (dropdown integrity break)
Example:
- User can select `1992 Chevrolet Corvette Z06` which never existed.
Root cause:
- `data/make-model-import/makes-filter/*.json` includes trims/submodels that appear to be “all-time” variants, not year-accurate.
- Example evidence: `data/make-model-import/makes-filter/chevrolet.json` contains `Z06` for 1992 Corvette.
- Resulting DB evidence: `data/make-model-import/output/03_vehicle_options.sql` includes `(1992,'Chevrolet','Corvette','Z06',...)`.
### 3) Duplicate rows in `vehicle_options`
Example evidence:
- `data/make-model-import/output/03_vehicle_options.sql` shows repeated identical rows for the same year/make/model/trim/engine/transmission.
Root causes:
- No dedupe at the fact level prior to SQL generation.
- Dimension ID strategy makes it difficult to dedupe correctly.
## Data Sources (Local Only)
Inputs in `data/make-model-import/`:
- `makes-filter/*.json`: provides coverage for makes/models by year, but trims/engines are not reliable for year accuracy.
- `automobiles.json`: contains “model pages” with names that include year or year ranges (e.g., `2013-2019`, `2021-Present`).
- `engines.json`: engine records keyed to `automobile_id` with specs and “Transmission Specs”.
- `brands.json`: make name metadata (ALL CAPS) + `id` used by `automobiles.json.brand_id`.
## Target ETL Strategy (Baseline Grid + Evidence Overlay)
We cannot use network sources, so the best available path is:
1) **Baseline coverage** from `makes-filter` for `(year, make, model)` within `[MIN_YEAR, MAX_YEAR]`.
2) **Year-accuracy overlay** from `automobiles.json` + `engines.json`:
- Parse each automobile into:
- Canonical make (via `brand_id → brands.json` mapping).
- Model name and an inferred trim/variant string.
- Year range (start/end) from the automobile name.
- Use these to build **evidence sets**:
- Which trims are evidenced for a `(make, model)` and which year ranges they apply to.
- Which engines/transmissions are evidenced (via `engines.json`) for that automobile entry.
3) Generate `vehicle_options`:
- For each baseline `(year, make, model)`:
- If overlay evidence exists for that `(make, model, year)`:
- Use evidenced trims for that year (trim defaults to `Base` if missing).
- Engines: use detailed engine display names when available; else fuel-type fallback label.
- Transmissions: derive from engine specs when available; else fallback to `Manual`+`Automatic`.
- If no overlay evidence exists:
- Create a single row with trim `Base`.
- Engine default label `Gas` (or other fuel label if you can infer it locally without guessing; otherwise `Gas`).
- Transmission fallback `Manual`+`Automatic`.
This approach ensures:
- Completeness: you still have a working dropdown for all year/make/model combos in-range.
- Accuracy improvements where the scraped evidence supports it (especially trims by year).
- No invented trims like `Z06` in years where there is no overlay evidence for `Z06` in that year range.
## Engine & Transmission Normalization Rules
### Engine display name
Use existing ETL display logic as a base (from `etl_generate_sql.py`) but change the ID strategy:
- If you can create a detailed engine display string (e.g., `V8 5.7L`, `L4 2.0L Turbo`), use it.
- Only use default labels when detailed specs are not available:
- `Electric` if fuel indicates electric.
- `Diesel` if fuel indicates diesel.
- `Hybrid` if fuel indicates any hybrid variant.
- Else `Gas`.
Fuel mapping should be derived from `engines.json → specs → Engine Specs → Fuel:` which currently includes values like:
- `Electric`
- `Diesel`
- `Hybrid`, `Hybrid Gasoline`, `Mild Hybrid`, `Mild Hybrid Diesel`, `Plug-In Hybrid`, etc.
### Transmission display name
Normalize to a small set of UI-friendly strings:
- Prefer `"{N}-Speed Manual"` or `"{N}-Speed Automatic"` when speed and type are known.
- Preserve `CVT`.
- If unknown for a `(year, make, model)`, provide both `Manual` and `Automatic`.
Important: transmission table IDs must be keyed by the **final display name**, not the raw tuple.
## Schema + Import Requirements (Rerunnable + Clean)
### Migration changes
Update `data/make-model-import/migrations/001_create_vehicle_database.sql` to:
- Match actual stored columns (current migration defines extra columns not populated by ETL).
- Enforce uniqueness to prevent duplicates:
- `engines`: unique on normalized name (e.g., `UNIQUE (LOWER(name))`).
- `transmissions`: unique on normalized type (e.g., `UNIQUE (LOWER(type))`).
- `vehicle_options`: unique on `(year, make, model, trim, engine_id, transmission_id)`.
### Import script changes
Update `data/make-model-import/import_data.sh` so reruns are consistent:
- Either:
- `TRUNCATE vehicle_options, engines, transmissions RESTART IDENTITY CASCADE;` before import, then insert, OR
- Use `INSERT ... ON CONFLICT DO NOTHING` with deterministic IDs (more complex).
Given constraints and large volume, truncation + re-import is simplest and most deterministic for dev environments.
## Validation / QA Harness (New)
Add a new script (recommended location: `data/make-model-import/qa_validate.py`) plus a small SQL file or inline queries.
Must-check assertions:
1) **Year window enforced**
- `MIN(year) >= MIN_YEAR` and `MAX(year) <= MAX_YEAR`.
2) **No dimension duplicates**
- `SELECT LOWER(name), COUNT(*) FROM engines GROUP BY 1 HAVING COUNT(*) > 1;` returns 0 rows.
- `SELECT LOWER(type), COUNT(*) FROM transmissions GROUP BY 1 HAVING COUNT(*) > 1;` returns 0 rows.
3) **No fact duplicates**
- `SELECT year, make, model, trim, engine_id, transmission_id, COUNT(*) FROM vehicle_options GROUP BY 1,2,3,4,5,6 HAVING COUNT(*) > 1;` returns 0 rows.
4) **Dropdown integrity sanity**
- For sampled `(year, make, model)`, trims returned by `get_trims_for_year_make_model()` must match distinct trims in `vehicle_options` for that tuple.
- For sampled `(year, make, model, trim)`, engines query matches `vehicle_options` join to `engines`.
- For sampled `(year, make, model)`, transmissions query matches `vehicle_options` join to `transmissions` (plus fallbacks when missing).
Optional (recommended) golden assertions:
- Add a small list of “known invalid historically” checks (like `1992 Corvette Z06`) that must return empty / not present.
- These should be driven by overlay evidence (do not hardcode large historical facts without evidence in local data).
## Work Breakdown (Assign to Agents)
### Agent A — ETL Core Refactor
Owner: ETL generation logic.
Deliverables:
- Update `data/make-model-import/etl_generate_sql.py`:
- Add config: `MIN_YEAR`/`MAX_YEAR` (defaults `2000`/`2026`).
- Replace current engine/transmission ID assignment with dedup-by-display-name mapping.
- Remove coupling where an `engine_id` implies an index into `engines.json` for transmission lookup.
- Implement fuel-type fallback label logic (`Gas/Diesel/Electric/Hybrid`) only when detailed engine spec cannot be built.
- Dedupe `vehicle_options` rows before writing SQL.
Acceptance:
- Generated `output/01_engines.sql` and `output/02_transmissions.sql` contain only unique values.
- Generated `output/03_vehicle_options.sql` contains no duplicate tuples.
- Output respects `[MIN_YEAR, MAX_YEAR]`.
### Agent B — Overlay Evidence Builder (Year-Accurate Trims)
Owner: parse `automobiles.json` and build trim/year evidence.
Deliverables:
- Implement parsing in `etl_generate_sql.py` (or a helper module if splitting is allowed) to:
- Extract year or year range from `automobiles.json.name` (handle `YYYY`, `YYYY-YYYY`, `YYYY-Present`).
- Map `brand_id → canonical make`.
- Normalize automobile “model+variant” string.
- Match against known models for that make (derived from `makes-filter`) to split `model` vs `trim`.
- Produce an evidence structure: for `(make, model)`, a list of `(trim, year_start, year_end)`.
Acceptance:
- Evidence filtering prevents trims that have no evidenced year overlap from appearing in those years when generating `vehicle_options`.
Notes:
- Matching model vs trim is heuristic; implement conservative logic:
- Prefer the longest model name match.
- If ambiguity, do not guess trim; default to `Base` and log a counter for review.
### Agent C — DB Migration + Constraints
Owner: schema correctness and preventing duplicates.
Deliverables:
- Update `data/make-model-import/migrations/001_create_vehicle_database.sql`:
- Align columns to the ETL output (keep only whats used).
- Add uniqueness constraints (engines/transmissions dims + vehicle_options fact).
- Ensure functions `get_makes_for_year`, `get_models_for_year_make`, `get_trims_for_year_make_model` remain compatible.
Acceptance:
- Rerunning import does not create duplicates even if the ETL output accidentally contains repeats (constraints will reject).
### Agent D — Import Script Rerun Safety
Owner: repeatable import process.
Deliverables:
- Update `data/make-model-import/import_data.sh`:
- Clear tables deterministically (truncate + restart identity) before import.
- Import order: schema → engines → transmissions → vehicle_options.
- Print verification counts and min/max year.
Acceptance:
- Running `./import_data.sh` twice produces identical row counts and no errors.
### Agent E — QA Harness
Owner: automated validation.
Deliverables:
- Add `data/make-model-import/qa_validate.py` with:
- Connect-free checks using generated SQL files (fast pre-import) AND/OR
- Post-import checks executed via `docker exec mvp-postgres psql ...` (slower, authoritative).
- Add a short `data/make-model-import/QA_README.md` or extend existing docs with exact commands.
Acceptance:
- QA script fails on duplicates, out-of-range years, and basic dropdown integrity mismatches.
### Agent F (Optional) — Backend/Docs Consistency
Owner: documentation accuracy.
Deliverables:
- Update docs that reference the old normalized `vehicles.*` schema if they conflict with the current `vehicle_options` based system.
- Primary references: `docs/VEHICLES-API.md`, `backend/src/features/platform/README.md` (verify claims).
Acceptance:
- Docs correctly describe the actual dropdown data source and rerun steps.
## Rollout Plan
1) Implement ETL refactor + evidence overlay + constraints + rerunnable import.
2) Regenerate SQL (`python3 etl_generate_sql.py` in `data/make-model-import/`).
3) Re-import (`./import_data.sh`).
4) Flush Redis dropdown caches (if needed) and re-test dropdowns.
5) Run QA harness and capture summary output in a `stats.txt` (or similar).
## Status Update (completed)
- ETL rewritten to use makes-filter as baseline (year/make/model + trims/engines) and overlay evidence only to prune impossible year/trim combos and enrich engines/transmissions.
- Engines/transmissions now deduped by display name; vehicle_options deduped on full key.
- Uniqueness constraints added to prevent duplicates on import.
- Import script made rerunnable (truncate + restart identity) and prints year range.
- QA script added and validated (duplicates=0, year range 20002026).
- Example issue (GMC Sierra 1500 AT4X 6.2L V8) now present via baseline engines for that trim/year and Automatic/Manual fallback when transmissions are absent.
## Acceptance Criteria (End-to-End)
- Years available in dropdown are exactly those loaded (default 20002026).
- Makes for a year only include makes with models in that year.
- Models for year+make only include models available for that tuple.
- Trims for year+make+model do not include impossible trims (e.g., no `1992 Corvette Z06` unless local evidence supports it).
- Engines show detailed specs when available; otherwise show one of `Gas/Diesel/Electric/Hybrid`.
- Transmissions show derived options when available; otherwise show both `Manual` and `Automatic`.
- No duplicate dimension rows; no duplicate fact rows.

331
apply-etl-plan.sh Executable file
View File

@@ -0,0 +1,331 @@
#!/bin/bash
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${YELLOW}=== MotoVaultPro ETL Plan V1 - Automated Application ===${NC}"
echo ""
# Get script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
# Backup directory
BACKUP_DIR="$SCRIPT_DIR/.etl-plan-backup-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$BACKUP_DIR"
echo -e "${GREEN}[1/4] Backing up files...${NC}"
# Backup ETL files
mkdir -p "$BACKUP_DIR/data/vehicle-etl"
cp data/vehicle-etl/vehapi_fetch_snapshot.py "$BACKUP_DIR/data/vehicle-etl/" 2>/dev/null || true
cp data/vehicle-etl/qa_validate.py "$BACKUP_DIR/data/vehicle-etl/" 2>/dev/null || true
# Backup backend files
mkdir -p "$BACKUP_DIR/backend/src/features/vehicles/api"
mkdir -p "$BACKUP_DIR/backend/src/features/vehicles/domain"
mkdir -p "$BACKUP_DIR/backend/src/features/platform/api"
mkdir -p "$BACKUP_DIR/backend/src/features/platform/domain"
mkdir -p "$BACKUP_DIR/backend/src/features/platform/data"
cp backend/src/features/vehicles/api/vehicles.controller.ts "$BACKUP_DIR/backend/src/features/vehicles/api/" 2>/dev/null || true
cp backend/src/features/vehicles/api/vehicles.routes.ts "$BACKUP_DIR/backend/src/features/vehicles/api/" 2>/dev/null || true
cp backend/src/features/vehicles/domain/vehicles.service.ts "$BACKUP_DIR/backend/src/features/vehicles/domain/" 2>/dev/null || true
cp backend/src/features/platform/api/platform.controller.ts "$BACKUP_DIR/backend/src/features/platform/api/" 2>/dev/null || true
cp backend/src/features/platform/index.ts "$BACKUP_DIR/backend/src/features/platform/" 2>/dev/null || true
cp backend/src/features/platform/domain/vin-decode.service.ts "$BACKUP_DIR/backend/src/features/platform/domain/" 2>/dev/null || true
cp backend/src/features/platform/data/vpic-client.ts "$BACKUP_DIR/backend/src/features/platform/data/" 2>/dev/null || true
# Backup frontend files
mkdir -p "$BACKUP_DIR/frontend/src/features/vehicles/components"
mkdir -p "$BACKUP_DIR/frontend/src/features/vehicles/api"
mkdir -p "$BACKUP_DIR/frontend/src/features/vehicles/types"
cp frontend/src/features/vehicles/components/VehicleForm.tsx "$BACKUP_DIR/frontend/src/features/vehicles/components/" 2>/dev/null || true
cp frontend/src/features/vehicles/api/vehicles.api.ts "$BACKUP_DIR/frontend/src/features/vehicles/api/" 2>/dev/null || true
cp frontend/src/features/vehicles/types/vehicles.types.ts "$BACKUP_DIR/frontend/src/features/vehicles/types/" 2>/dev/null || true
echo -e "${GREEN} Backup created at: $BACKUP_DIR${NC}"
echo ""
echo -e "${GREEN}[2/4] Applying ETL Configuration Changes...${NC}"
# Update vehapi_fetch_snapshot.py
if [ -f "data/vehicle-etl/vehapi_fetch_snapshot.py" ]; then
echo " - Updating DEFAULT_MIN_YEAR from 1980 to 2017..."
sed -i.bak 's/DEFAULT_MIN_YEAR = 1980/DEFAULT_MIN_YEAR = 2017/g' data/vehicle-etl/vehapi_fetch_snapshot.py
sed -i.bak 's/default env MIN_YEAR or 1980/default env MIN_YEAR or 2017/g' data/vehicle-etl/vehapi_fetch_snapshot.py
rm data/vehicle-etl/vehapi_fetch_snapshot.py.bak
echo -e " ${GREEN}${NC} vehapi_fetch_snapshot.py updated"
else
echo -e " ${RED}${NC} vehapi_fetch_snapshot.py not found"
fi
# Update qa_validate.py
if [ -f "data/vehicle-etl/qa_validate.py" ]; then
echo " - Updating year range validation from 1980-2022 to 2017-2022..."
sed -i.bak 's/year < 1980 OR year > 2022/year < 2017 OR year > 2022/g' data/vehicle-etl/qa_validate.py
rm data/vehicle-etl/qa_validate.py.bak
echo -e " ${GREEN}${NC} qa_validate.py updated"
else
echo -e " ${RED}${NC} qa_validate.py not found"
fi
# Delete old ETL documentation
echo " - Removing old ETL documentation..."
rm -f data/vehicle-etl/ETL-FIX-V2.md
rm -f data/vehicle-etl/ETL-FIXES.md
rm -f data/vehicle-etl/ETL-VEHAPI-PLAN.md
rm -f data/vehicle-etl/ETL-VEHAPI-REDESIGN.md
echo -e " ${GREEN}${NC} Old ETL documentation removed"
echo ""
echo -e "${GREEN}[3/4] Applying Backend Changes...${NC}"
# Update vehicles.controller.ts MIN_YEAR
if [ -f "backend/src/features/vehicles/api/vehicles.controller.ts" ]; then
echo " - Updating MIN_YEAR from 1980 to 2017..."
sed -i.bak 's/private static readonly MIN_YEAR = 1980;/private static readonly MIN_YEAR = 2017;/g' backend/src/features/vehicles/api/vehicles.controller.ts
rm backend/src/features/vehicles/api/vehicles.controller.ts.bak
echo -e " ${GREEN}${NC} vehicles.controller.ts MIN_YEAR updated"
else
echo -e " ${RED}${NC} vehicles.controller.ts not found"
fi
# Remove VIN decode route from vehicles.routes.ts
if [ -f "backend/src/features/vehicles/api/vehicles.routes.ts" ]; then
echo " - Removing VIN decode route..."
python3 << 'PYTHON_EOF'
import re
with open('backend/src/features/vehicles/api/vehicles.routes.ts', 'r') as f:
content = f.read()
# Remove the decode-vin route (multi-line pattern)
pattern = r"\n\s*fastify\.post<\{ Body: \{ vin: string \} \}>\('/vehicles/decode-vin',\s*\{[^}]*\}\);"
content = re.sub(pattern, '', content, flags=re.DOTALL)
with open('backend/src/features/vehicles/api/vehicles.routes.ts', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} VIN decode route removed from vehicles.routes.ts"
else
echo -e " ${YELLOW}${NC} vehicles.routes.ts not found (may not exist)"
fi
# Remove VIN decode from vehicles.service.ts
if [ -f "backend/src/features/vehicles/domain/vehicles.service.ts" ]; then
echo " - Removing VIN decode logic from vehicles.service.ts..."
python3 << 'PYTHON_EOF'
import re
with open('backend/src/features/vehicles/domain/vehicles.service.ts', 'r') as f:
content = f.read()
# Remove import statement for vin-decode
content = re.sub(r"import\s*\{[^}]*getVINDecodeService[^}]*\}[^;]*;?\n?", '', content)
content = re.sub(r"import\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
# Remove decodeVIN method if it exists
pattern = r'\n\s*async decodeVIN\([^)]*\)[^{]*\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}\n'
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
# Remove VIN decode logic from createVehicle method (between specific comments or patterns)
# This is more conservative - just remove obvious decode blocks
content = re.sub(r'\n\s*// VIN decode logic.*?(?=\n\s*(?:const|return|if|//|$))', '', content, flags=re.DOTALL)
with open('backend/src/features/vehicles/domain/vehicles.service.ts', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} VIN decode logic removed from vehicles.service.ts"
else
echo -e " ${RED}${NC} vehicles.service.ts not found"
fi
# Remove VIN decode from platform.controller.ts
if [ -f "backend/src/features/platform/api/platform.controller.ts" ]; then
echo " - Removing VIN decode from platform.controller.ts..."
python3 << 'PYTHON_EOF'
import re
with open('backend/src/features/platform/api/platform.controller.ts', 'r') as f:
content = f.read()
# Remove VINDecodeService import
content = re.sub(r"import\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
# Remove VINDecodeService from constructor
content = re.sub(r',?\s*private\s+vinDecodeService:\s*VINDecodeService', '', content)
content = re.sub(r'private\s+vinDecodeService:\s*VINDecodeService\s*,?', '', content)
# Remove decodeVIN method
pattern = r'\n\s*async decodeVIN\([^)]*\)[^{]*\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}\n'
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
with open('backend/src/features/platform/api/platform.controller.ts', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} VIN decode removed from platform.controller.ts"
else
echo -e " ${YELLOW}${NC} platform.controller.ts not found"
fi
# Remove VINDecodeService from platform/index.ts
if [ -f "backend/src/features/platform/index.ts" ]; then
echo " - Removing VINDecodeService from platform/index.ts..."
python3 << 'PYTHON_EOF'
import re
with open('backend/src/features/platform/index.ts', 'r') as f:
content = f.read()
# Remove VINDecodeService import
content = re.sub(r"import\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
# Remove VINDecodeService export
content = re.sub(r"export\s*\{[^}]*VINDecodeService[^}]*\}[^;]*;?\n?", '', content)
content = re.sub(r",\s*VINDecodeService\s*", '', content)
content = re.sub(r"VINDecodeService\s*,\s*", '', content)
# Remove getVINDecodeService function
pattern = r'\n\s*export\s+function\s+getVINDecodeService\([^)]*\)[^{]*\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}\n'
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
with open('backend/src/features/platform/index.ts', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} VINDecodeService removed from platform/index.ts"
else
echo -e " ${YELLOW}${NC} platform/index.ts not found"
fi
# Delete VIN decode service files
echo " - Deleting VIN decode service files..."
rm -f backend/src/features/platform/domain/vin-decode.service.ts
rm -f backend/src/features/platform/data/vpic-client.ts
echo -e " ${GREEN}${NC} VIN decode service files deleted"
echo ""
echo -e "${GREEN}[4/4] Applying Frontend Changes...${NC}"
# Update VehicleForm.tsx
if [ -f "frontend/src/features/vehicles/components/VehicleForm.tsx" ]; then
echo " - Removing VIN decode UI from VehicleForm.tsx..."
python3 << 'PYTHON_EOF'
import re
with open('frontend/src/features/vehicles/components/VehicleForm.tsx', 'r') as f:
content = f.read()
# Remove decodingVIN and decodeSuccess state variables
content = re.sub(r"\n\s*const \[decodingVIN, setDecodingVIN\] = useState\(false\);", '', content)
content = re.sub(r"\n\s*const \[decodeSuccess, setDecodeSuccess\] = useState\(false\);", '', content)
content = re.sub(r"\n\s*const \[decodeSuccess, setDecodeSuccess\] = useState<boolean \| null>\(null\);", '', content)
# Remove watchedVIN if it exists
content = re.sub(r"\n\s*const watchedVIN = watch\('vin'\);", '', content)
# Remove handleDecodeVIN function
pattern = r'\n\s*const handleDecodeVIN = async \(\) => \{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\};'
content = re.sub(pattern, '', content, flags=re.DOTALL)
# Update helper text
content = re.sub(
r'Enter VIN to auto-fill vehicle details OR manually select from dropdowns below',
'Enter vehicle VIN (optional)',
content
)
content = re.sub(
r'Enter VIN to auto-populate fields',
'Enter vehicle VIN (optional)',
content
)
# Remove the Decode VIN button and surrounding div
# Pattern 1: div with flex layout containing input and button
pattern1 = r'<div className="flex flex-col sm:flex-row gap-2">\s*<input\s+\{\.\.\.register\(\'vin\'\)\}[^>]*>\s*</input>\s*<Button[^>]*onClick=\{handleDecodeVIN\}[^>]*>.*?</Button>\s*</div>'
replacement1 = r'<input {...register(\'vin\')} className="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-primary-500 text-base" placeholder="Enter 17-character VIN (optional if License Plate provided)" style={{ fontSize: \'16px\' }} />'
content = re.sub(pattern1, replacement1, content, flags=re.DOTALL)
# Pattern 2: self-closing input with button after
pattern2 = r'<input\s+\{\.\.\.register\(\'vin\'\)\}[^/]*/>\s*<Button[^>]*onClick=\{handleDecodeVIN\}[^>]*>.*?</Button>'
content = re.sub(pattern2, replacement1, content, flags=re.DOTALL)
# Remove decode success message
content = re.sub(r'\n\s*\{decodeSuccess && \([^)]*\)\}', '', content, flags=re.DOTALL)
content = re.sub(r'\n\s*<p className="mt-1 text-sm text-green-600">VIN decoded successfully.*?</p>', '', content, flags=re.DOTALL)
with open('frontend/src/features/vehicles/components/VehicleForm.tsx', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} VIN decode UI removed from VehicleForm.tsx"
else
echo -e " ${RED}${NC} VehicleForm.tsx not found"
fi
# Update vehicles.api.ts
if [ -f "frontend/src/features/vehicles/api/vehicles.api.ts" ]; then
echo " - Removing decodeVIN method from vehicles.api.ts..."
python3 << 'PYTHON_EOF'
import re
with open('frontend/src/features/vehicles/api/vehicles.api.ts', 'r') as f:
content = f.read()
# Remove VINDecodeResponse from imports
content = re.sub(r',\s*VINDecodeResponse', '', content)
content = re.sub(r'VINDecodeResponse\s*,', '', content)
# Remove decodeVIN method
pattern = r'\n\s*decodeVIN:\s*async\s*\([^)]*\)[^{]*\{[^}]*\},?'
content = re.sub(pattern, '', content, flags=re.DOTALL)
# Remove trailing comma before closing brace if present
content = re.sub(r',(\s*\};?\s*$)', r'\1', content)
with open('frontend/src/features/vehicles/api/vehicles.api.ts', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} decodeVIN method removed from vehicles.api.ts"
else
echo -e " ${RED}${NC} vehicles.api.ts not found"
fi
# Update vehicles.types.ts
if [ -f "frontend/src/features/vehicles/types/vehicles.types.ts" ]; then
echo " - Removing VINDecodeResponse interface from vehicles.types.ts..."
python3 << 'PYTHON_EOF'
import re
with open('frontend/src/features/vehicles/types/vehicles.types.ts', 'r') as f:
content = f.read()
# Remove VINDecodeResponse interface
pattern = r'\n\s*export interface VINDecodeResponse \{[^}]*\}\n?'
content = re.sub(pattern, '\n', content, flags=re.DOTALL)
with open('frontend/src/features/vehicles/types/vehicles.types.ts', 'w') as f:
f.write(content)
PYTHON_EOF
echo -e " ${GREEN}${NC} VINDecodeResponse interface removed from vehicles.types.ts"
else
echo -e " ${RED}${NC} vehicles.types.ts not found"
fi
echo ""
echo -e "${GREEN}=== Changes Applied Successfully ===${NC}"
echo ""
echo -e "${YELLOW}Verification Commands:${NC}"
echo ""
echo "1. ETL Configuration:"
echo " cd data/vehicle-etl && python3 -c \"import vehapi_fetch_snapshot; print(vehapi_fetch_snapshot.DEFAULT_MIN_YEAR)\""
echo " Expected output: 2017"
echo ""
echo "2. Backend TypeScript:"
echo " cd backend && npx tsc --noEmit"
echo " Expected: No errors"
echo ""
echo "3. Frontend TypeScript:"
echo " cd frontend && npx tsc --noEmit"
echo " Expected: No errors"
echo ""
echo -e "${YELLOW}Next Steps:${NC}"
echo "1. Run the verification commands above"
echo "2. If all checks pass, proceed with Agent 4 (ETL Execution)"
echo "3. If there are issues, restore from backup: $BACKUP_DIR"
echo ""
echo -e "${GREEN}Backup location: $BACKUP_DIR${NC}"

View File

@@ -5,7 +5,6 @@
import { FastifyRequest, FastifyReply } from 'fastify'; import { FastifyRequest, FastifyReply } from 'fastify';
import { Pool } from 'pg'; import { Pool } from 'pg';
import { VehicleDataService } from '../domain/vehicle-data.service'; import { VehicleDataService } from '../domain/vehicle-data.service';
import { VINDecodeService } from '../domain/vin-decode.service';
import { PlatformCacheService } from '../domain/platform-cache.service'; import { PlatformCacheService } from '../domain/platform-cache.service';
import { cacheService } from '../../../core/config/redis'; import { cacheService } from '../../../core/config/redis';
import { import {
@@ -13,21 +12,18 @@ import {
ModelsQuery, ModelsQuery,
TrimsQuery, TrimsQuery,
EnginesQuery, EnginesQuery,
TransmissionsQuery, TransmissionsQuery
VINDecodeRequest
} from '../models/requests'; } from '../models/requests';
import { logger } from '../../../core/logging/logger'; import { logger } from '../../../core/logging/logger';
export class PlatformController { export class PlatformController {
private vehicleDataService: VehicleDataService; private vehicleDataService: VehicleDataService;
private vinDecodeService: VINDecodeService;
private pool: Pool; private pool: Pool;
constructor(pool: Pool) { constructor(pool: Pool) {
this.pool = pool; this.pool = pool;
const platformCache = new PlatformCacheService(cacheService); const platformCache = new PlatformCacheService(cacheService);
this.vehicleDataService = new VehicleDataService(platformCache); this.vehicleDataService = new VehicleDataService(platformCache);
this.vinDecodeService = new VINDecodeService(platformCache);
} }
/** /**
@@ -100,12 +96,14 @@ export class PlatformController {
} }
/** /**
* GET /api/platform/transmissions?year={year}&make={make}&model={model} * GET /api/platform/transmissions?year={year}&make={make}&model={model}&trim={trim}[&engine={engine}]
*/ */
async getTransmissions(request: FastifyRequest<{ Querystring: TransmissionsQuery }>, reply: FastifyReply): Promise<void> { async getTransmissions(request: FastifyRequest<{ Querystring: TransmissionsQuery }>, reply: FastifyReply): Promise<void> {
try { try {
const { year, make, model } = request.query as any; const { year, make, model, trim, engine } = request.query as any;
const transmissions = await this.vehicleDataService.getTransmissions(this.pool, year, make, model); const transmissions = engine
? await this.vehicleDataService.getTransmissionsForTrimAndEngine(this.pool, year, make, model, trim, engine)
: await this.vehicleDataService.getTransmissionsForTrim(this.pool, year, make, model, trim);
reply.code(200).send({ transmissions }); reply.code(200).send({ transmissions });
} catch (error) { } catch (error) {
logger.error('Controller error: getTransmissions', { error, query: request.query }); logger.error('Controller error: getTransmissions', { error, query: request.query });
@@ -113,34 +111,4 @@ export class PlatformController {
} }
} }
/**
* GET /api/platform/vehicle?vin={vin}
*/
async decodeVIN(request: FastifyRequest<{ Querystring: VINDecodeRequest }>, reply: FastifyReply): Promise<void> {
try {
const { vin } = request.query;
const result = await this.vinDecodeService.decodeVIN(this.pool, vin);
if (!result.success) {
if (result.error && result.error.includes('Invalid VIN')) {
reply.code(400).send(result);
} else if (result.error && result.error.includes('unavailable')) {
reply.code(503).send(result);
} else {
reply.code(404).send(result);
}
return;
}
reply.code(200).send(result);
} catch (error) {
logger.error('Controller error: decodeVIN', { error, query: request.query });
reply.code(500).send({
vin: request.query.vin,
result: null,
success: false,
error: 'Internal server error during VIN decoding'
});
}
}
} }

View File

@@ -10,8 +10,7 @@ import {
ModelsQuery, ModelsQuery,
TrimsQuery, TrimsQuery,
EnginesQuery, EnginesQuery,
TransmissionsQuery, TransmissionsQuery
VINDecodeRequest
} from '../models/requests'; } from '../models/requests';
import pool from '../../../core/config/database'; import pool from '../../../core/config/database';
@@ -41,10 +40,6 @@ async function platformRoutes(fastify: FastifyInstance) {
fastify.get<{ Querystring: TransmissionsQuery }>('/platform/transmissions', { fastify.get<{ Querystring: TransmissionsQuery }>('/platform/transmissions', {
preHandler: [fastify.authenticate] preHandler: [fastify.authenticate]
}, controller.getTransmissions.bind(controller)); }, controller.getTransmissions.bind(controller));
fastify.get<{ Querystring: VINDecodeRequest }>('/platform/vehicle', {
preHandler: [fastify.authenticate]
}, controller.decodeVIN.bind(controller));
} }
export default fastifyPlugin(platformRoutes); export default fastifyPlugin(platformRoutes);

View File

@@ -84,18 +84,7 @@ export class VehicleDataRepository {
*/ */
async getEngines(pool: Pool, year: number, make: string, model: string, trim: string): Promise<string[]> { async getEngines(pool: Pool, year: number, make: string, model: string, trim: string): Promise<string[]> {
const query = ` const query = `
SELECT DISTINCT SELECT engine_name FROM get_engines_for_vehicle($1, $2, $3, $4)
CASE
WHEN vo.engine_id IS NULL THEN 'N/A (Electric)'
ELSE e.name
END as engine_name
FROM vehicle_options vo
LEFT JOIN engines e ON e.id = vo.engine_id
WHERE vo.year = $1
AND vo.make = $2
AND vo.model = $3
AND vo.trim = $4
ORDER BY engine_name
`; `;
try { try {
@@ -108,30 +97,46 @@ export class VehicleDataRepository {
} }
/** /**
* Get transmissions for a specific year, make, and model * Get transmissions for a specific year, make, model, and trim
* Returns real transmission types from the database (not hardcoded) * Returns real transmission types from the database (not hardcoded)
*/ */
async getTransmissions(pool: Pool, year: number, make: string, model: string): Promise<string[]> { async getTransmissionsForTrim(pool: Pool, year: number, make: string, model: string, trim: string): Promise<string[]> {
const query = ` const query = `
SELECT DISTINCT SELECT transmission_type FROM get_transmissions_for_vehicle($1, $2, $3, $4)
CASE
WHEN vo.transmission_id IS NULL THEN 'N/A'
ELSE t.type
END as transmission_type
FROM vehicle_options vo
LEFT JOIN transmissions t ON t.id = vo.transmission_id
WHERE vo.year = $1
AND vo.make = $2
AND vo.model = $3
ORDER BY transmission_type
`; `;
try { try {
const result = await pool.query(query, [year, make, model]); const result = await pool.query(query, [year, make, model, trim]);
return result.rows.map(row => row.transmission_type); return result.rows.map(row => row.transmission_type);
} catch (error) { } catch (error) {
logger.error('Repository error: getTransmissions', { error, year, make, model }); logger.error('Repository error: getTransmissionsForTrim', { error, year, make, model, trim });
throw new Error(`Failed to retrieve transmissions for year ${year}, make ${make}, model ${model}`); throw new Error(`Failed to retrieve transmissions for year ${year}, make ${make}, model ${model}, trim ${trim}`);
}
}
async getTransmissionsForTrimAndEngine(pool: Pool, year: number, make: string, model: string, trim: string, engine: string): Promise<string[]> {
const query = `
SELECT transmission_type FROM get_transmissions_for_vehicle_engine($1, $2, $3, $4, $5)
`;
try {
const result = await pool.query(query, [year, make, model, trim, engine]);
return result.rows.map(row => row.transmission_type);
} catch (error) {
logger.error('Repository error: getTransmissionsForTrimAndEngine', { error, year, make, model, trim, engine });
throw new Error(`Failed to retrieve transmissions for year ${year}, make ${make}, model ${model}, trim ${trim}, engine ${engine}`);
}
}
async getEnginesForTrimAndTransmission(pool: Pool, year: number, make: string, model: string, trim: string, transmission: string): Promise<string[]> {
const query = `
SELECT engine_name FROM get_engines_for_vehicle_trans($1, $2, $3, $4, $5)
`;
try {
const result = await pool.query(query, [year, make, model, trim, transmission]);
return result.rows.map(row => row.engine_name);
} catch (error) {
logger.error('Repository error: getEnginesForTrimAndTransmission', { error, year, make, model, trim, transmission });
throw new Error(`Failed to retrieve engines for year ${year}, make ${make}, model ${model}, trim ${trim}, transmission ${transmission}`);
} }
} }

View File

@@ -1,125 +0,0 @@
/**
* @ai-summary NHTSA vPIC API client for VIN decoding fallback
* @ai-context External API client with timeout and error handling
*/
import axios, { AxiosInstance } from 'axios';
import { VPICResponse, VINDecodeResult } from '../models/responses';
import { logger } from '../../../core/logging/logger';
export class VPICClient {
private client: AxiosInstance;
private readonly baseURL = 'https://vpic.nhtsa.dot.gov/api';
private readonly timeout = 5000; // 5 seconds
constructor() {
this.client = axios.create({
baseURL: this.baseURL,
timeout: this.timeout,
headers: {
'Accept': 'application/json',
'User-Agent': 'MotoVaultPro/1.0'
}
});
}
/**
* Decode VIN using NHTSA vPIC API
*/
async decodeVIN(vin: string): Promise<VINDecodeResult | null> {
try {
const url = `/vehicles/DecodeVin/${vin}?format=json`;
logger.debug('Calling vPIC API', { url, vin });
const response = await this.client.get<VPICResponse>(url);
if (!response.data || !response.data.Results) {
logger.warn('vPIC API returned invalid response', { vin });
return null;
}
// Parse vPIC response into our format
const result = this.parseVPICResponse(response.data.Results);
if (!result.make || !result.model || !result.year) {
logger.warn('vPIC API returned incomplete data', { vin, result });
return null;
}
logger.info('Successfully decoded VIN via vPIC', { vin, make: result.make, model: result.model, year: result.year });
return result;
} catch (error) {
if (axios.isAxiosError(error)) {
if (error.code === 'ECONNABORTED') {
logger.error('vPIC API timeout', { vin, timeout: this.timeout });
} else if (error.response) {
logger.error('vPIC API error response', {
vin,
status: error.response.status,
statusText: error.response.statusText
});
} else if (error.request) {
logger.error('vPIC API no response', { vin });
} else {
logger.error('vPIC API request error', { vin, error: error.message });
}
} else {
logger.error('Unexpected error calling vPIC', { vin, error });
}
return null;
}
}
/**
* Parse vPIC API response variables into our format
*/
private parseVPICResponse(results: Array<{ Variable: string; Value: string | null }>): VINDecodeResult {
const getValue = (variableName: string): string | null => {
const variable = results.find(v => v.Variable === variableName);
return variable?.Value || null;
};
const getNumberValue = (variableName: string): number | null => {
const value = getValue(variableName);
if (!value) return null;
const parsed = parseFloat(value);
return isNaN(parsed) ? null : parsed;
};
return {
make: getValue('Make'),
model: getValue('Model'),
year: getNumberValue('Model Year'),
trim_name: getValue('Trim'),
engine_description: this.buildEngineDescription(results),
transmission_description: getValue('Transmission Style'),
horsepower: null, // vPIC doesn't provide horsepower
torque: null, // vPIC doesn't provide torque
top_speed: null, // vPIC doesn't provide top speed
fuel: getValue('Fuel Type - Primary'),
confidence_score: 0.5, // Lower confidence for vPIC fallback
vehicle_type: getValue('Vehicle Type')
};
}
/**
* Build engine description from multiple vPIC fields
*/
private buildEngineDescription(results: Array<{ Variable: string; Value: string | null }>): string | null {
const getValue = (variableName: string): string | null => {
const variable = results.find(v => v.Variable === variableName);
return variable?.Value || null;
};
const displacement = getValue('Displacement (L)');
const cylinders = getValue('Engine Number of Cylinders');
const configuration = getValue('Engine Configuration');
const parts: string[] = [];
if (displacement) parts.push(`${displacement}L`);
if (configuration) parts.push(configuration);
if (cylinders) parts.push(`${cylinders} cyl`);
return parts.length > 0 ? parts.join(' ') : null;
}
}

View File

@@ -94,21 +94,41 @@ export class PlatformCacheService {
} }
/** /**
* Get cached transmissions for year, make, and model * Get cached transmissions for year, make, model, and trim
*/ */
async getTransmissions(year: number, make: string, model: string): Promise<string[] | null> { async getTransmissionsForTrim(year: number, make: string, model: string, trim: string): Promise<string[] | null> {
const key = this.prefix + 'vehicle-data:transmissions:' + year + ':' + make + ':' + model; const key = this.prefix + 'vehicle-data:transmissions:' + year + ':' + make + ':' + model + ':' + trim;
return await this.cacheService.get<string[]>(key); return await this.cacheService.get<string[]>(key);
} }
/** /**
* Set cached transmissions for year, make, and model * Set cached transmissions for year, make, model, and trim
*/ */
async setTransmissions(year: number, make: string, model: string, transmissions: string[], ttl: number = 6 * 3600): Promise<void> { async setTransmissionsForTrim(year: number, make: string, model: string, trim: string, transmissions: string[], ttl: number = 6 * 3600): Promise<void> {
const key = this.prefix + 'vehicle-data:transmissions:' + year + ':' + make + ':' + model; const key = this.prefix + 'vehicle-data:transmissions:' + year + ':' + make + ':' + model + ':' + trim;
await this.cacheService.set(key, transmissions, ttl); await this.cacheService.set(key, transmissions, ttl);
} }
async getTransmissionsForTrimAndEngine(year: number, make: string, model: string, trim: string, engine: string): Promise<string[] | null> {
const key = this.prefix + 'vehicle-data:transmissions:' + year + ':' + make + ':' + model + ':' + trim + ':engine:' + engine;
return await this.cacheService.get<string[]>(key);
}
async setTransmissionsForTrimAndEngine(year: number, make: string, model: string, trim: string, engine: string, transmissions: string[], ttl: number = 6 * 3600): Promise<void> {
const key = this.prefix + 'vehicle-data:transmissions:' + year + ':' + make + ':' + model + ':' + trim + ':engine:' + engine;
await this.cacheService.set(key, transmissions, ttl);
}
async getEnginesForTrimAndTransmission(year: number, make: string, model: string, trim: string, transmission: string): Promise<string[] | null> {
const key = this.prefix + 'vehicle-data:engines:' + year + ':' + make + ':' + model + ':' + trim + ':transmission:' + transmission;
return await this.cacheService.get<string[]>(key);
}
async setEnginesForTrimAndTransmission(year: number, make: string, model: string, trim: string, transmission: string, engines: string[], ttl: number = 6 * 3600): Promise<void> {
const key = this.prefix + 'vehicle-data:engines:' + year + ':' + make + ':' + model + ':' + trim + ':transmission:' + transmission;
await this.cacheService.set(key, engines, ttl);
}
/** /**
* Get cached VIN decode result * Get cached VIN decode result
*/ */

View File

@@ -123,23 +123,94 @@ export class VehicleDataService {
} }
/** /**
* Get transmissions for a year, make, and model with caching * Get transmissions for a year, make, model, and trim with caching
*/ */
async getTransmissions(pool: Pool, year: number, make: string, model: string): Promise<string[]> { async getTransmissionsForTrim(pool: Pool, year: number, make: string, model: string, trim: string): Promise<string[]> {
try { try {
const cached = await this.cache.getTransmissions(year, make, model); const cached = await this.cache.getTransmissionsForTrim(year, make, model, trim);
if (cached) { if (cached) {
logger.debug('Transmissions retrieved from cache', { year, make, model }); logger.debug('Transmissions retrieved from cache', { year, make, model, trim });
return cached; return cached;
} }
const transmissions = await this.repository.getTransmissions(pool, year, make, model); const transmissions = await this.repository.getTransmissionsForTrim(pool, year, make, model, trim);
await this.cache.setTransmissions(year, make, model, transmissions); await this.cache.setTransmissionsForTrim(year, make, model, trim, transmissions);
logger.debug('Transmissions retrieved from database and cached', { year, make, model, count: transmissions.length }); logger.debug('Transmissions retrieved from database and cached', { year, make, model, trim, count: transmissions.length });
return transmissions; return transmissions;
} catch (error) { } catch (error) {
logger.error('Service error: getTransmissions', { error, year, make, model }); logger.error('Service error: getTransmissionsForTrim', { error, year, make, model, trim });
throw error; throw error;
} }
} }
async getTransmissionsForTrimAndEngine(
pool: Pool,
year: number,
make: string,
model: string,
trim: string,
engine: string
): Promise<string[]> {
try {
const cached = await this.cache.getTransmissionsForTrimAndEngine(year, make, model, trim, engine);
if (cached) {
logger.debug('Transmissions (engine-filtered) retrieved from cache', { year, make, model, trim, engine });
return cached;
}
const transmissions = await this.repository.getTransmissionsForTrimAndEngine(pool, year, make, model, trim, engine);
await this.cache.setTransmissionsForTrimAndEngine(year, make, model, trim, engine, transmissions);
logger.debug('Transmissions (engine-filtered) retrieved from database and cached', { year, make, model, trim, engine, count: transmissions.length });
return transmissions;
} catch (error) {
logger.error('Service error: getTransmissionsForTrimAndEngine', { error, year, make, model, trim, engine });
throw error;
}
}
async getEnginesForTrimAndTransmission(
pool: Pool,
year: number,
make: string,
model: string,
trim: string,
transmission: string
): Promise<string[]> {
try {
const cached = await this.cache.getEnginesForTrimAndTransmission(year, make, model, trim, transmission);
if (cached) {
logger.debug('Engines (transmission-filtered) retrieved from cache', { year, make, model, trim, transmission });
return cached;
}
const engines = await this.repository.getEnginesForTrimAndTransmission(pool, year, make, model, trim, transmission);
await this.cache.setEnginesForTrimAndTransmission(year, make, model, trim, transmission, engines);
logger.debug('Engines (transmission-filtered) retrieved from database and cached', { year, make, model, trim, transmission, count: engines.length });
return engines;
} catch (error) {
logger.error('Service error: getEnginesForTrimAndTransmission', { error, year, make, model, trim, transmission });
throw error;
}
}
async getOptions(
pool: Pool,
year: number,
make: string,
model: string,
trim: string,
engine?: string,
transmission?: string
): Promise<{ engines: string[]; transmissions: string[] }> {
const [engines, transmissions] = await Promise.all([
transmission
? this.getEnginesForTrimAndTransmission(pool, year, make, model, trim, transmission)
: this.getEngines(pool, year, make, model, trim),
engine
? this.getTransmissionsForTrimAndEngine(pool, year, make, model, trim, engine)
: this.getTransmissionsForTrim(pool, year, make, model, trim)
]);
return { engines, transmissions };
}
} }

View File

@@ -1,156 +0,0 @@
/**
* @ai-summary VIN decoding service with circuit breaker and fallback
* @ai-context PostgreSQL first, vPIC API fallback, Redis caching
*/
import { Pool } from 'pg';
import CircuitBreaker from 'opossum';
import { VehicleDataRepository } from '../data/vehicle-data.repository';
import { VPICClient } from '../data/vpic-client';
import { PlatformCacheService } from './platform-cache.service';
import { VINDecodeResponse, VINDecodeResult } from '../models/responses';
import { logger } from '../../../core/logging/logger';
export class VINDecodeService {
private repository: VehicleDataRepository;
private vpicClient: VPICClient;
private cache: PlatformCacheService;
private circuitBreaker: CircuitBreaker;
constructor(cache: PlatformCacheService) {
this.cache = cache;
this.repository = new VehicleDataRepository();
this.vpicClient = new VPICClient();
this.circuitBreaker = new CircuitBreaker(
async (vin: string) => this.vpicClient.decodeVIN(vin),
{
timeout: 6000,
errorThresholdPercentage: 50,
resetTimeout: 30000,
name: 'vpic-api'
}
);
this.circuitBreaker.on('open', () => {
logger.warn('Circuit breaker opened for vPIC API');
});
this.circuitBreaker.on('halfOpen', () => {
logger.info('Circuit breaker half-open for vPIC API');
});
this.circuitBreaker.on('close', () => {
logger.info('Circuit breaker closed for vPIC API');
});
}
/**
* Validate VIN format
*/
validateVIN(vin: string): { valid: boolean; error?: string } {
if (vin.length !== 17) {
return { valid: false, error: 'VIN must be exactly 17 characters' };
}
const invalidChars = /[IOQ]/i;
if (invalidChars.test(vin)) {
return { valid: false, error: 'VIN contains invalid characters (cannot contain I, O, Q)' };
}
const validFormat = /^[A-HJ-NPR-Z0-9]{17}$/i;
if (!validFormat.test(vin)) {
return { valid: false, error: 'VIN contains invalid characters' };
}
return { valid: true };
}
/**
* Decode VIN with multi-tier strategy:
* 1. Check cache
* 2. Try PostgreSQL function
* 3. Fallback to vPIC API (with circuit breaker)
*/
async decodeVIN(pool: Pool, vin: string): Promise<VINDecodeResponse> {
const normalizedVIN = vin.toUpperCase().trim();
const validation = this.validateVIN(normalizedVIN);
if (!validation.valid) {
return {
vin: normalizedVIN,
result: null,
success: false,
error: validation.error
};
}
try {
const cached = await this.cache.getVINDecode(normalizedVIN);
if (cached) {
logger.debug('VIN decode result retrieved from cache', { vin: normalizedVIN });
return cached;
}
let result = await this.repository.decodeVIN(pool, normalizedVIN);
if (result) {
const response: VINDecodeResponse = {
vin: normalizedVIN,
result,
success: true
};
await this.cache.setVINDecode(normalizedVIN, response, true);
logger.info('VIN decoded successfully via PostgreSQL', { vin: normalizedVIN, make: result.make, model: result.model, year: result.year });
return response;
}
logger.info('VIN not found in PostgreSQL, attempting vPIC fallback', { vin: normalizedVIN });
try {
result = await this.circuitBreaker.fire(normalizedVIN) as VINDecodeResult | null;
if (result) {
const response: VINDecodeResponse = {
vin: normalizedVIN,
result,
success: true
};
await this.cache.setVINDecode(normalizedVIN, response, true);
logger.info('VIN decoded successfully via vPIC fallback', { vin: normalizedVIN, make: result.make, model: result.model, year: result.year });
return response;
}
} catch (circuitError) {
logger.warn('vPIC API unavailable or circuit breaker open', { vin: normalizedVIN, error: circuitError });
}
const failureResponse: VINDecodeResponse = {
vin: normalizedVIN,
result: null,
success: false,
error: 'VIN not found in database and external API unavailable'
};
await this.cache.setVINDecode(normalizedVIN, failureResponse, false);
return failureResponse;
} catch (error) {
logger.error('VIN decode error', { vin: normalizedVIN, error });
return {
vin: normalizedVIN,
result: null,
success: false,
error: 'Internal server error during VIN decoding'
};
}
}
/**
* Get circuit breaker status
*/
getCircuitBreakerStatus(): { state: string; stats: any } {
return {
state: this.circuitBreaker.opened ? 'open' : this.circuitBreaker.halfOpen ? 'half-open' : 'closed',
stats: this.circuitBreaker.stats
};
}
}

View File

@@ -5,30 +5,19 @@
import { Pool } from 'pg'; import { Pool } from 'pg';
import pool from '../../core/config/database'; import pool from '../../core/config/database';
import { cacheService } from '../../core/config/redis'; import { cacheService } from '../../core/config/redis';
import { VINDecodeService } from './domain/vin-decode.service';
import { PlatformCacheService } from './domain/platform-cache.service'; import { PlatformCacheService } from './domain/platform-cache.service';
import { VehicleDataService } from './domain/vehicle-data.service'; import { VehicleDataService } from './domain/vehicle-data.service';
export { platformRoutes } from './api/platform.routes'; export { platformRoutes } from './api/platform.routes';
export { PlatformController } from './api/platform.controller'; export { PlatformController } from './api/platform.controller';
export { VehicleDataService } from './domain/vehicle-data.service'; export { VehicleDataService } from './domain/vehicle-data.service';
export { VINDecodeService } from './domain/vin-decode.service';
export { PlatformCacheService } from './domain/platform-cache.service'; export { PlatformCacheService } from './domain/platform-cache.service';
export * from './models/requests'; export * from './models/requests';
export * from './models/responses'; export * from './models/responses';
// Singleton VIN decode service for use by other features // Singleton vehicle data service for use by other features
let vinDecodeServiceInstance: VINDecodeService | null = null;
let vehicleDataServiceInstance: VehicleDataService | null = null; let vehicleDataServiceInstance: VehicleDataService | null = null;
export function getVINDecodeService(): VINDecodeService {
if (!vinDecodeServiceInstance) {
const platformCache = new PlatformCacheService(cacheService);
vinDecodeServiceInstance = new VINDecodeService(platformCache);
}
return vinDecodeServiceInstance;
}
export function getVehicleDataService(): VehicleDataService { export function getVehicleDataService(): VehicleDataService {
if (!vehicleDataServiceInstance) { if (!vehicleDataServiceInstance) {
const platformCache = new PlatformCacheService(cacheService); const platformCache = new PlatformCacheService(cacheService);

View File

@@ -102,7 +102,11 @@ export const transmissionsQuerySchema = z.object({
.max(100, 'Make must be less than 100 characters'), .max(100, 'Make must be less than 100 characters'),
model: z.string() model: z.string()
.min(1, 'Model is required') .min(1, 'Model is required')
.max(100, 'Model must be less than 100 characters') .max(100, 'Model must be less than 100 characters'),
trim: z.string()
.min(1, 'Trim is required')
.max(100, 'Trim must be less than 100 characters'),
engine: z.string().optional()
}); });
export type TransmissionsQuery = z.infer<typeof transmissionsQuerySchema>; export type TransmissionsQuery = z.infer<typeof transmissionsQuerySchema>;

View File

@@ -1,173 +0,0 @@
/**
* @ai-summary Unit tests for VIN decode service
* @ai-context Tests VIN validation, PostgreSQL decode, vPIC fallback, circuit breaker
*/
import { Pool } from 'pg';
import { VINDecodeService } from '../../domain/vin-decode.service';
import { PlatformCacheService } from '../../domain/platform-cache.service';
import { VehicleDataRepository } from '../../data/vehicle-data.repository';
import { VPICClient } from '../../data/vpic-client';
jest.mock('../../data/vehicle-data.repository');
jest.mock('../../data/vpic-client');
jest.mock('../../domain/platform-cache.service');
describe('VINDecodeService', () => {
let service: VINDecodeService;
let mockCache: jest.Mocked<PlatformCacheService>;
let mockPool: jest.Mocked<Pool>;
beforeEach(() => {
mockCache = {
getVINDecode: jest.fn(),
setVINDecode: jest.fn()
} as any;
mockPool = {} as any;
service = new VINDecodeService(mockCache);
});
describe('validateVIN', () => {
it('should validate correct VIN', () => {
const result = service.validateVIN('1HGCM82633A123456');
expect(result.valid).toBe(true);
expect(result.error).toBeUndefined();
});
it('should reject VIN with incorrect length', () => {
const result = service.validateVIN('SHORT');
expect(result.valid).toBe(false);
expect(result.error).toContain('17 characters');
});
it('should reject VIN with invalid characters I, O, Q', () => {
const resultI = service.validateVIN('1HGCM82633A12345I');
const resultO = service.validateVIN('1HGCM82633A12345O');
const resultQ = service.validateVIN('1HGCM82633A12345Q');
expect(resultI.valid).toBe(false);
expect(resultO.valid).toBe(false);
expect(resultQ.valid).toBe(false);
});
it('should reject VIN with non-alphanumeric characters', () => {
const result = service.validateVIN('1HGCM82633A12345@');
expect(result.valid).toBe(false);
});
});
describe('decodeVIN', () => {
const validVIN = '1HGCM82633A123456';
const mockResult = {
make: 'Honda',
model: 'Accord',
year: 2003,
trim_name: 'LX',
engine_description: '2.4L I4',
transmission_description: '5-Speed Automatic',
horsepower: 160,
torque: 161,
top_speed: null,
fuel: 'Gasoline',
confidence_score: 0.95,
vehicle_type: 'Passenger Car'
};
it('should return cached result if available', async () => {
const cachedResponse = {
vin: validVIN,
result: mockResult,
success: true
};
mockCache.getVINDecode.mockResolvedValue(cachedResponse);
const result = await service.decodeVIN(mockPool, validVIN);
expect(result).toEqual(cachedResponse);
expect(mockCache.getVINDecode).toHaveBeenCalledWith(validVIN);
});
it('should return error for invalid VIN format', async () => {
const invalidVIN = 'INVALID';
mockCache.getVINDecode.mockResolvedValue(null);
const result = await service.decodeVIN(mockPool, invalidVIN);
expect(result.success).toBe(false);
expect(result.error).toContain('17 characters');
});
it('should uppercase and trim VIN', async () => {
const lowerVIN = ' 1hgcm82633a123456 ';
mockCache.getVINDecode.mockResolvedValue(null);
(service as any).repository.decodeVIN = jest.fn().mockResolvedValue(mockResult);
await service.decodeVIN(mockPool, lowerVIN);
expect(mockCache.getVINDecode).toHaveBeenCalledWith('1HGCM82633A123456');
});
it('should decode VIN from PostgreSQL and cache result', async () => {
mockCache.getVINDecode.mockResolvedValue(null);
(service as any).repository.decodeVIN = jest.fn().mockResolvedValue(mockResult);
const result = await service.decodeVIN(mockPool, validVIN);
expect(result.success).toBe(true);
expect(result.result).toEqual(mockResult);
expect(mockCache.setVINDecode).toHaveBeenCalledWith(
validVIN,
expect.objectContaining({ vin: validVIN, success: true }),
true
);
});
it('should fallback to vPIC when PostgreSQL returns null', async () => {
mockCache.getVINDecode.mockResolvedValue(null);
(service as any).repository.decodeVIN = jest.fn().mockResolvedValue(null);
(service as any).vpicClient.decodeVIN = jest.fn().mockResolvedValue(mockResult);
const result = await service.decodeVIN(mockPool, validVIN);
expect(result.success).toBe(true);
expect(result.result).toEqual(mockResult);
});
it('should return failure when both PostgreSQL and vPIC fail', async () => {
mockCache.getVINDecode.mockResolvedValue(null);
(service as any).repository.decodeVIN = jest.fn().mockResolvedValue(null);
(service as any).vpicClient.decodeVIN = jest.fn().mockResolvedValue(null);
const result = await service.decodeVIN(mockPool, validVIN);
expect(result.success).toBe(false);
expect(result.error).toContain('VIN not found');
});
it('should cache failed decode with shorter TTL', async () => {
mockCache.getVINDecode.mockResolvedValue(null);
(service as any).repository.decodeVIN = jest.fn().mockResolvedValue(null);
(service as any).vpicClient.decodeVIN = jest.fn().mockResolvedValue(null);
await service.decodeVIN(mockPool, validVIN);
expect(mockCache.setVINDecode).toHaveBeenCalledWith(
validVIN,
expect.objectContaining({ success: false }),
false
);
});
});
describe('getCircuitBreakerStatus', () => {
it('should return circuit breaker status', () => {
const status = service.getCircuitBreakerStatus();
expect(status).toHaveProperty('state');
expect(status).toHaveProperty('stats');
expect(['open', 'half-open', 'closed']).toContain(status.state);
});
});
});

View File

@@ -12,6 +12,8 @@ import { CreateVehicleBody, UpdateVehicleBody, VehicleParams } from '../domain/v
export class VehiclesController { export class VehiclesController {
private vehiclesService: VehiclesService; private vehiclesService: VehiclesService;
private static readonly MIN_YEAR = 2017;
private static readonly MAX_YEAR = 2022;
constructor() { constructor() {
const repository = new VehiclesRepository(pool); const repository = new VehiclesRepository(pool);
@@ -153,10 +155,10 @@ export class VehiclesController {
async getDropdownMakes(request: FastifyRequest<{ Querystring: { year: number } }>, reply: FastifyReply) { async getDropdownMakes(request: FastifyRequest<{ Querystring: { year: number } }>, reply: FastifyReply) {
try { try {
const { year } = request.query; const { year } = request.query;
if (!year || year < 1980 || year > new Date().getFullYear() + 1) { if (!year || year < VehiclesController.MIN_YEAR || year > VehiclesController.MAX_YEAR) {
return reply.code(400).send({ return reply.code(400).send({
error: 'Bad Request', error: 'Bad Request',
message: 'Valid year parameter is required (1980-' + (new Date().getFullYear() + 1) + ')' message: `Valid year parameter is required (${VehiclesController.MIN_YEAR}-${VehiclesController.MAX_YEAR})`
}); });
} }
@@ -174,10 +176,10 @@ export class VehiclesController {
async getDropdownModels(request: FastifyRequest<{ Querystring: { year: number; make: string } }>, reply: FastifyReply) { async getDropdownModels(request: FastifyRequest<{ Querystring: { year: number; make: string } }>, reply: FastifyReply) {
try { try {
const { year, make } = request.query; const { year, make } = request.query;
if (!year || !make || year < 1980 || year > new Date().getFullYear() + 1 || make.trim().length === 0) { if (!year || !make || year < VehiclesController.MIN_YEAR || year > VehiclesController.MAX_YEAR || make.trim().length === 0) {
return reply.code(400).send({ return reply.code(400).send({
error: 'Bad Request', error: 'Bad Request',
message: 'Valid year and make parameters are required' message: `Valid year and make parameters are required (${VehiclesController.MIN_YEAR}-${VehiclesController.MAX_YEAR})`
}); });
} }
@@ -192,20 +194,20 @@ export class VehiclesController {
} }
} }
async getDropdownTransmissions(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string } }>, reply: FastifyReply) { async getDropdownTransmissions(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string; trim: string } }>, reply: FastifyReply) {
try { try {
const { year, make, model } = request.query; const { year, make, model, trim } = request.query;
if (!year || !make || !model || year < 1980 || year > new Date().getFullYear() + 1 || make.trim().length === 0 || model.trim().length === 0) { if (!year || !make || !model || !trim || year < VehiclesController.MIN_YEAR || year > VehiclesController.MAX_YEAR || make.trim().length === 0 || model.trim().length === 0 || trim.trim().length === 0) {
return reply.code(400).send({ return reply.code(400).send({
error: 'Bad Request', error: 'Bad Request',
message: 'Valid year, make, and model parameters are required' message: `Valid year, make, model, and trim parameters are required (${VehiclesController.MIN_YEAR}-${VehiclesController.MAX_YEAR})`
}); });
} }
const transmissions = await this.vehiclesService.getDropdownTransmissions(year, make, model); const transmissions = await this.vehiclesService.getDropdownTransmissions(year, make, model, trim);
return reply.code(200).send(transmissions); return reply.code(200).send(transmissions);
} catch (error) { } catch (error) {
logger.error('Error getting dropdown transmissions', { error, year: request.query?.year, make: request.query?.make, model: request.query?.model }); logger.error('Error getting dropdown transmissions', { error, year: request.query?.year, make: request.query?.make, model: request.query?.model, trim: request.query?.trim });
return reply.code(500).send({ return reply.code(500).send({
error: 'Internal server error', error: 'Internal server error',
message: 'Failed to get transmissions' message: 'Failed to get transmissions'
@@ -216,10 +218,10 @@ export class VehiclesController {
async getDropdownEngines(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string; trim: string } }>, reply: FastifyReply) { async getDropdownEngines(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string; trim: string } }>, reply: FastifyReply) {
try { try {
const { year, make, model, trim } = request.query; const { year, make, model, trim } = request.query;
if (!year || !make || !model || !trim || year < 1980 || year > new Date().getFullYear() + 1 || make.trim().length === 0 || model.trim().length === 0 || trim.trim().length === 0) { if (!year || !make || !model || !trim || year < VehiclesController.MIN_YEAR || year > VehiclesController.MAX_YEAR || make.trim().length === 0 || model.trim().length === 0 || trim.trim().length === 0) {
return reply.code(400).send({ return reply.code(400).send({
error: 'Bad Request', error: 'Bad Request',
message: 'Valid year, make, model, and trim parameters are required' message: `Valid year, make, model, and trim parameters are required (${VehiclesController.MIN_YEAR}-${VehiclesController.MAX_YEAR})`
}); });
} }
@@ -237,10 +239,10 @@ export class VehiclesController {
async getDropdownTrims(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string } }>, reply: FastifyReply) { async getDropdownTrims(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string } }>, reply: FastifyReply) {
try { try {
const { year, make, model } = request.query; const { year, make, model } = request.query;
if (!year || !make || !model || year < 1980 || year > new Date().getFullYear() + 1 || make.trim().length === 0 || model.trim().length === 0) { if (!year || !make || !model || year < VehiclesController.MIN_YEAR || year > VehiclesController.MAX_YEAR || make.trim().length === 0 || model.trim().length === 0) {
return reply.code(400).send({ return reply.code(400).send({
error: 'Bad Request', error: 'Bad Request',
message: 'Valid year, make, and model parameters are required' message: `Valid year, make, and model parameters are required (${VehiclesController.MIN_YEAR}-${VehiclesController.MAX_YEAR})`
}); });
} }
@@ -269,26 +271,23 @@ export class VehiclesController {
} }
} }
async decodeVIN(request: FastifyRequest<{ Body: { vin: string } }>, reply: FastifyReply) { async getDropdownOptions(request: FastifyRequest<{ Querystring: { year: number; make: string; model: string; trim: string; engine?: string; transmission?: string } }>, reply: FastifyReply) {
try { try {
const { vin } = request.body; const { year, make, model, trim, engine, transmission } = request.query;
if (!year || !make || !model || !trim || year < VehiclesController.MIN_YEAR || year > VehiclesController.MAX_YEAR || make.trim().length === 0 || model.trim().length === 0 || trim.trim().length === 0) {
if (!vin || vin.length !== 17) {
return reply.code(400).send({ return reply.code(400).send({
vin: vin || '', error: 'Bad Request',
success: false, message: `Valid year, make, model, and trim parameters are required (${VehiclesController.MIN_YEAR}-${VehiclesController.MAX_YEAR})`
error: 'VIN must be exactly 17 characters'
}); });
} }
const result = await this.vehiclesService.decodeVIN(vin); const options = await this.vehiclesService.getDropdownOptions(year, make, model, trim, engine, transmission);
return reply.code(200).send(result); return reply.code(200).send(options);
} catch (error: any) { } catch (error) {
logger.error('Error decoding VIN', { error, vin: request.body?.vin }); logger.error('Error getting dropdown options', { error, query: request.query });
return reply.code(500).send({ return reply.code(500).send({
vin: request.body?.vin || '', error: 'Internal server error',
success: false, message: 'Failed to get engine/transmission options'
error: 'VIN decode failed'
}); });
} }
} }

View File

@@ -63,16 +63,16 @@ export const vehiclesRoutes: FastifyPluginAsync = async (
handler: vehiclesController.getDropdownEngines.bind(vehiclesController) handler: vehiclesController.getDropdownEngines.bind(vehiclesController)
}); });
// GET /api/vehicles/dropdown/transmissions?year=2024&make=Ford&model=F-150 - Get transmissions (Level 3) // GET /api/vehicles/dropdown/transmissions?year=2024&make=Ford&model=F-150&trim=XLT - Get transmissions (Level 4, trim-filtered)
fastify.get<{ Querystring: { year: number; make: string; model: string } }>('/vehicles/dropdown/transmissions', { fastify.get<{ Querystring: { year: number; make: string; model: string; trim: string } }>('/vehicles/dropdown/transmissions', {
preHandler: [fastify.authenticate], preHandler: [fastify.authenticate],
handler: vehiclesController.getDropdownTransmissions.bind(vehiclesController) handler: vehiclesController.getDropdownTransmissions.bind(vehiclesController)
}); });
// POST /api/vehicles/decode-vin - Decode VIN and return vehicle information // GET /api/vehicles/dropdown/options?year&make&model&trim[&engine=...][&transmission=...] - Pair-safe options for engine/transmission
fastify.post<{ Body: { vin: string } }>('/vehicles/decode-vin', { fastify.get<{ Querystring: { year: number; make: string; model: string; trim: string; engine?: string; transmission?: string } }>('/vehicles/dropdown/options', {
preHandler: [fastify.authenticate], preHandler: [fastify.authenticate],
handler: vehiclesController.decodeVIN.bind(vehiclesController) handler: vehiclesController.getDropdownOptions.bind(vehiclesController)
}); });
// Dynamic :id routes MUST come last to avoid matching specific paths like "dropdown" // Dynamic :id routes MUST come last to avoid matching specific paths like "dropdown"

View File

@@ -4,7 +4,6 @@
*/ */
import { VehiclesRepository } from '../data/vehicles.repository'; import { VehiclesRepository } from '../data/vehicles.repository';
import { getVINDecodeService, getVehicleDataService, getPool } from '../../platform';
import { import {
Vehicle, Vehicle,
CreateVehicleRequest, CreateVehicleRequest,
@@ -15,6 +14,7 @@ import { logger } from '../../../core/logging/logger';
import { cacheService } from '../../../core/config/redis'; import { cacheService } from '../../../core/config/redis';
import { isValidVIN } from '../../../shared-minimal/utils/validators'; import { isValidVIN } from '../../../shared-minimal/utils/validators';
import { normalizeMakeName, normalizeModelName } from './name-normalizer'; import { normalizeMakeName, normalizeModelName } from './name-normalizer';
import { getVehicleDataService, getPool } from '../../platform';
export class VehiclesService { export class VehiclesService {
private readonly cachePrefix = 'vehicles'; private readonly cachePrefix = 'vehicles';
@@ -27,10 +27,6 @@ export class VehiclesService {
async createVehicle(data: CreateVehicleRequest, userId: string): Promise<VehicleResponse> { async createVehicle(data: CreateVehicleRequest, userId: string): Promise<VehicleResponse> {
logger.info('Creating vehicle', { userId, vin: data.vin, licensePlate: (data as any).licensePlate }); logger.info('Creating vehicle', { userId, vin: data.vin, licensePlate: (data as any).licensePlate });
let make: string | undefined;
let model: string | undefined;
let year: number | undefined;
if (data.vin) { if (data.vin) {
// Validate VIN if provided // Validate VIN if provided
if (!isValidVIN(data.vin)) { if (!isValidVIN(data.vin)) {
@@ -41,33 +37,19 @@ export class VehiclesService {
if (existing) { if (existing) {
throw new Error('Vehicle with this VIN already exists'); throw new Error('Vehicle with this VIN already exists');
} }
// Attempt VIN decode to enrich fields using platform service
const vinDecodeService = getVINDecodeService();
const pool = getPool();
const vinDecodeResult = await vinDecodeService.decodeVIN(pool, data.vin);
if (vinDecodeResult.success && vinDecodeResult.result) {
make = normalizeMakeName(vinDecodeResult.result.make);
model = normalizeModelName(vinDecodeResult.result.model);
year = vinDecodeResult.result.year ?? undefined;
// VIN caching is now handled by platform feature
}
} }
// Create vehicle (VIN optional). Client-sent make/model/year override decode if provided.
const inputMake = (data as any).make ?? make;
const inputModel = (data as any).model ?? model;
// Create vehicle with user-provided data
const vehicle = await this.repository.create({ const vehicle = await this.repository.create({
...data, ...data,
userId, userId,
make: normalizeMakeName(inputMake), make: data.make ? normalizeMakeName(data.make) : undefined,
model: normalizeModelName(inputModel), model: data.model ? normalizeModelName(data.model) : undefined,
year: (data as any).year ?? year,
}); });
// Invalidate user's vehicle list cache // Invalidate user's vehicle list cache
await this.invalidateUserCache(userId); await this.invalidateUserCache(userId);
return this.toResponse(vehicle); return this.toResponse(vehicle);
} }
@@ -178,12 +160,12 @@ export class VehiclesService {
return vehicleDataService.getModels(pool, year, make); return vehicleDataService.getModels(pool, year, make);
} }
async getDropdownTransmissions(year: number, make: string, model: string): Promise<string[]> { async getDropdownTransmissions(year: number, make: string, model: string, trim: string): Promise<string[]> {
const vehicleDataService = getVehicleDataService(); const vehicleDataService = getVehicleDataService();
const pool = getPool(); const pool = getPool();
logger.info('Fetching dropdown transmissions via platform module', { year, make, model }); logger.info('Fetching dropdown transmissions via platform module', { year, make, model, trim });
return vehicleDataService.getTransmissions(pool, year, make, model); return vehicleDataService.getTransmissionsForTrim(pool, year, make, model, trim);
} }
async getDropdownEngines(year: number, make: string, model: string, trim: string): Promise<string[]> { async getDropdownEngines(year: number, make: string, model: string, trim: string): Promise<string[]> {
@@ -202,6 +184,21 @@ export class VehiclesService {
return vehicleDataService.getTrims(pool, year, make, model); return vehicleDataService.getTrims(pool, year, make, model);
} }
async getDropdownOptions(
year: number,
make: string,
model: string,
trim: string,
engine?: string,
transmission?: string
): Promise<{ engines: string[]; transmissions: string[] }> {
const vehicleDataService = getVehicleDataService();
const pool = getPool();
logger.info('Fetching dropdown options via platform module', { year, make, model, trim, engine, transmission });
return vehicleDataService.getOptions(pool, year, make, model, trim, engine, transmission);
}
async getDropdownYears(): Promise<number[]> { async getDropdownYears(): Promise<number[]> {
const vehicleDataService = getVehicleDataService(); const vehicleDataService = getVehicleDataService();
const pool = getPool(); const pool = getPool();
@@ -210,54 +207,6 @@ export class VehiclesService {
return vehicleDataService.getYears(pool); return vehicleDataService.getYears(pool);
} }
async decodeVIN(vin: string): Promise<{
vin: string;
success: boolean;
year?: number;
make?: string;
model?: string;
trimLevel?: string;
engine?: string;
transmission?: string;
confidence?: number;
error?: string;
}> {
try {
logger.info('Decoding VIN', { vin });
// Use platform feature's VIN decode service
const vinDecodeService = getVINDecodeService();
const pool = getPool();
const result = await vinDecodeService.decodeVIN(pool, vin);
if (result.success && result.result) {
return {
vin,
success: true,
year: result.result.year ?? undefined,
make: result.result.make ?? undefined,
model: result.result.model ?? undefined,
trimLevel: result.result.trim_name ?? undefined,
engine: result.result.engine_description ?? undefined,
confidence: 85 // High confidence since we have good data
};
} else {
return {
vin,
success: false,
error: result.error || 'Unable to decode VIN'
};
}
} catch (error) {
logger.error('Failed to decode VIN', { vin, error });
return {
vin,
success: false,
error: 'VIN decode service unavailable'
};
}
}
private toResponse(vehicle: Vehicle): VehicleResponse { private toResponse(vehicle: Vehicle): VehicleResponse {
return { return {
id: vehicle.id, id: vehicle.id,

View File

@@ -22,19 +22,6 @@ jest.mock('../../../../core/plugins/auth.plugin', () => {
}; };
}); });
// Mock external VIN decoder
jest.mock('../../external/vpic/vpic.client', () => ({
vpicClient: {
decodeVIN: jest.fn().mockResolvedValue({
make: 'Honda',
model: 'Civic',
year: 2021,
engineType: '2.0L',
bodyType: 'Sedan',
rawData: []
})
}
}));
describe('Vehicles Integration Tests', () => { describe('Vehicles Integration Tests', () => {
beforeAll(async () => { beforeAll(async () => {
@@ -67,7 +54,7 @@ describe('Vehicles Integration Tests', () => {
}); });
describe('POST /api/vehicles', () => { describe('POST /api/vehicles', () => {
it('should create a new vehicle', async () => { it('should create a new vehicle with VIN', async () => {
const vehicleData = { const vehicleData = {
vin: '1HGBH41JXMN109186', vin: '1HGBH41JXMN109186',
nickname: 'My Test Car', nickname: 'My Test Car',
@@ -84,9 +71,6 @@ describe('Vehicles Integration Tests', () => {
id: expect.any(String), id: expect.any(String),
userId: 'test-user-123', userId: 'test-user-123',
vin: '1HGBH41JXMN109186', vin: '1HGBH41JXMN109186',
make: 'Honda',
model: 'Civic',
year: 2021,
nickname: 'My Test Car', nickname: 'My Test Car',
color: 'Blue', color: 'Blue',
odometerReading: 50000, odometerReading: 50000,
@@ -113,7 +97,8 @@ describe('Vehicles Integration Tests', () => {
it('should reject duplicate VIN for same user', async () => { it('should reject duplicate VIN for same user', async () => {
const vehicleData = { const vehicleData = {
vin: '1HGBH41JXMN109186', vin: '1HGBH41JXMN109186',
nickname: 'First Car' nickname: 'First Car',
licensePlate: 'ABC123'
}; };
// Create first vehicle // Create first vehicle
@@ -128,7 +113,7 @@ describe('Vehicles Integration Tests', () => {
.send({ ...vehicleData, nickname: 'Duplicate Car' }) .send({ ...vehicleData, nickname: 'Duplicate Car' })
.expect(400); .expect(400);
expect(response.body.error).toBe('Vehicle with this VIN already exists'); expect(response.body.message).toContain('already exists');
}); });
}); });

View File

@@ -12,14 +12,12 @@ import * as platformModule from '../../../platform';
jest.mock('../../data/vehicles.repository'); jest.mock('../../data/vehicles.repository');
jest.mock('../../../../core/config/redis'); jest.mock('../../../../core/config/redis');
jest.mock('../../../platform', () => ({ jest.mock('../../../platform', () => ({
getVINDecodeService: jest.fn(),
getVehicleDataService: jest.fn(), getVehicleDataService: jest.fn(),
getPool: jest.fn() getPool: jest.fn()
})); }));
const mockRepository = jest.mocked(VehiclesRepository); const mockRepository = jest.mocked(VehiclesRepository);
const mockCacheService = jest.mocked(cacheService); const mockCacheService = jest.mocked(cacheService);
const mockGetVINDecodeService = jest.mocked(platformModule.getVINDecodeService);
const mockGetVehicleDataService = jest.mocked(platformModule.getVehicleDataService); const mockGetVehicleDataService = jest.mocked(platformModule.getVehicleDataService);
const mockGetPool = jest.mocked(platformModule.getPool); const mockGetPool = jest.mocked(platformModule.getPool);
@@ -117,7 +115,7 @@ describe('VehiclesService', () => {
]); ]);
}); });
}); });
});
describe('createVehicle', () => { describe('createVehicle', () => {
const mockVehicleData = { const mockVehicleData = {
vin: '1HGBH41JXMN109186', vin: '1HGBH41JXMN109186',
@@ -126,22 +124,13 @@ describe('VehiclesService', () => {
odometerReading: 50000, odometerReading: 50000,
}; };
const mockVinDecodeResult = {
make: 'Honda',
model: 'Civic',
year: 2021,
engineType: '2.0L',
bodyType: 'Sedan',
rawData: [],
};
const mockCreatedVehicle = { const mockCreatedVehicle = {
id: 'vehicle-id-123', id: 'vehicle-id-123',
userId: 'user-123', userId: 'user-123',
vin: '1HGBH41JXMN109186', vin: '1HGBH41JXMN109186',
make: 'Honda', make: undefined,
model: 'Civic', model: undefined,
year: 2021, year: undefined,
nickname: 'My Car', nickname: 'My Car',
color: 'Blue', color: 'Blue',
licensePlate: undefined, licensePlate: undefined,
@@ -152,20 +141,7 @@ describe('VehiclesService', () => {
updatedAt: new Date('2024-01-01T00:00:00Z'), updatedAt: new Date('2024-01-01T00:00:00Z'),
}; };
it('should create a vehicle with VIN decoding', async () => { it('should create a vehicle with user-provided VIN', async () => {
const mockVinDecodeService = {
decodeVIN: jest.fn().mockResolvedValue({
success: true,
data: {
vin: '1HGBH41JXMN109186',
make: 'Honda',
model: 'Civic',
year: 2021
}
})
};
mockGetVINDecodeService.mockReturnValue(mockVinDecodeService as any);
repositoryInstance.findByUserAndVIN.mockResolvedValue(null); repositoryInstance.findByUserAndVIN.mockResolvedValue(null);
repositoryInstance.create.mockResolvedValue(mockCreatedVehicle); repositoryInstance.create.mockResolvedValue(mockCreatedVehicle);
mockCacheService.del.mockResolvedValue(undefined); mockCacheService.del.mockResolvedValue(undefined);
@@ -173,16 +149,13 @@ describe('VehiclesService', () => {
const result = await service.createVehicle(mockVehicleData, 'user-123'); const result = await service.createVehicle(mockVehicleData, 'user-123');
expect(repositoryInstance.findByUserAndVIN).toHaveBeenCalledWith('user-123', '1HGBH41JXMN109186'); expect(repositoryInstance.findByUserAndVIN).toHaveBeenCalledWith('user-123', '1HGBH41JXMN109186');
expect(mockVinDecodeService.decodeVIN).toHaveBeenCalledWith('mock-pool', '1HGBH41JXMN109186');
expect(repositoryInstance.create).toHaveBeenCalledWith({ expect(repositoryInstance.create).toHaveBeenCalledWith({
...mockVehicleData, ...mockVehicleData,
userId: 'user-123', userId: 'user-123',
make: 'Honda', make: undefined,
model: 'Civic', model: undefined,
year: 2021,
}); });
expect(result.id).toBe('vehicle-id-123'); expect(result.id).toBe('vehicle-id-123');
expect(result.make).toBe('Honda');
}); });
it('should reject invalid VIN format', async () => { it('should reject invalid VIN format', async () => {
@@ -196,31 +169,6 @@ describe('VehiclesService', () => {
await expect(service.createVehicle(mockVehicleData, 'user-123')).rejects.toThrow('Vehicle with this VIN already exists'); await expect(service.createVehicle(mockVehicleData, 'user-123')).rejects.toThrow('Vehicle with this VIN already exists');
}); });
it('should handle VIN decode failure gracefully', async () => {
const mockVinDecodeService = {
decodeVIN: jest.fn().mockResolvedValue({
success: false,
error: 'VIN decode failed'
})
};
mockGetVINDecodeService.mockReturnValue(mockVinDecodeService as any);
repositoryInstance.findByUserAndVIN.mockResolvedValue(null);
repositoryInstance.create.mockResolvedValue({ ...mockCreatedVehicle, make: undefined, model: undefined, year: undefined });
mockCacheService.del.mockResolvedValue(undefined);
const result = await service.createVehicle(mockVehicleData, 'user-123');
expect(repositoryInstance.create).toHaveBeenCalledWith({
...mockVehicleData,
userId: 'user-123',
make: undefined,
model: undefined,
year: undefined,
});
expect(result.make).toBeUndefined();
});
}); });
describe('getUserVehicles', () => { describe('getUserVehicles', () => {

View File

@@ -1,275 +0,0 @@
# Automotive Vehicle Selection Database - ETL Documentation
## Overview
This ETL pipeline creates a PostgreSQL database optimized for cascading dropdown vehicle selection:
**Year → Make → Model → Trim → Engine/Transmission**
## Database Schema
### Tables
1. **engines** - Simplified engine specifications
- id (Primary Key)
- name (Display format: "V8 3.5L", "L4 2.0L Turbo", "V6 6.2L Supercharged")
2. **transmissions** - Simplified transmission specifications
- id (Primary Key)
- type (Display format: "8-Speed Automatic", "6-Speed Manual", "CVT")
3. **vehicle_options** - Denormalized vehicle configurations
- Year, Make (Title Case: "Ford", "Acura", "Land Rover"), Model, Trim
- Foreign keys to engines and transmissions
- Optimized indexes for dropdown queries
### Views
- `available_years` - All distinct years
- `makes_by_year` - Makes grouped by year
- `models_by_year_make` - Models grouped by year/make
- `trims_by_year_make_model` - Trims grouped by year/make/model
- `complete_vehicle_configs` - Full vehicle details with engine/transmission
### Functions
- `get_makes_for_year(year)` - Returns makes for a specific year
- `get_models_for_year_make(year, make)` - Returns models for year/make
- `get_trims_for_year_make_model(year, make, model)` - Returns trims
- `get_options_for_vehicle(year, make, model, trim)` - Returns engine/transmission options
## Data Sources
### Primary Source
**makes-filter/*.json** (57 makes)
- Filtered manufacturer data
- Year/model/trim/engine hierarchy
- Engine specs as simple strings (e.g., "2.0L I4")
### Detailed Specs
**engines.json** (30,066+ records)
- Complete engine specifications
- Performance data, fuel economy
- Transmission details
**automobiles.json** (7,207 models)
- Model descriptions
- Used for hybrid backfill of recent years (2023-2025)
**brands.json** (124 brands)
- Brand metadata
- Used for brand name mapping
## ETL Process
### Step 1: Load Source Data
- Load `engines.json` (30,066 records)
- Load `brands.json` (124 brands)
- Load `automobiles.json` (7,207 models)
- Load all `makes-filter/*.json` files (55 files)
### Step 2: Transform Brand Names
- Convert ALL CAPS brand names to Title Case ("FORD" → "Ford")
- Preserve acronyms (BMW, GMC, KIA remain uppercase)
- Handle special cases (DeLorean, McLaren)
### Step 3: Process Engine Specifications
- Extract engine specs from engines.json
- Create simplified display names (e.g., "V8 3.5L Turbo")
- Normalize displacement (Cm3 → Liters) for matching
- Build engine cache with (displacement, configuration) keys
- Generate engines SQL with only id and name columns
### Step 4: Process Transmission Specifications
- Extract transmission specs from engines.json
- Create simplified display names (e.g., "8-Speed Automatic")
- Parse speed count and transmission type
- Build transmission cache for linking
- Generate transmissions SQL with only id and type columns
### Step 5: Process Makes-Filter Data
- Read all JSON files from `makes-filter/`
- Extract year/make/model/trim/engine combinations
- Match engine strings to detailed specs using displacement + configuration
- Link transmissions to vehicle records (98.9% success rate)
- Apply year filter (1980 and newer only)
- Build vehicle_options records
### Step 6: Hybrid Backfill
- Check `automobiles.json` for recent years (2023-2025)
- Add any missing year/make/model combinations
- Only backfill for filtered makes
- Link transmissions for backfilled records
- Limit to 3 engines per backfilled model
### Step 7: Generate SQL Output
- Write SQL files with proper escaping (newlines, quotes, special characters)
- Convert empty strings to NULL for data integrity
- Use batched inserts (1000 records per batch)
- Output to `output/` directory
## Running the ETL
### Prerequisites
- Docker container `mvp-postgres` running
- Python 3 (no additional dependencies required)
- JSON source files in project root
### Quick Start
```bash
# Step 1: Generate SQL files from JSON data
python3 etl_generate_sql.py
# Step 2: Import SQL files into database
./import_data.sh
```
### What Gets Generated
- `output/01_engines.sql` (~632KB, 30,066 records)
- `output/02_transmissions.sql` (~21KB, 828 records)
- `output/03_vehicle_options.sql` (~51MB, 1,122,644 records)
## Query Examples
### Get all available years
```sql
SELECT * FROM available_years;
```
### Get makes for 2024
```sql
SELECT * FROM get_makes_for_year(2024);
```
### Get models for 2025 Ford
```sql
SELECT * FROM get_models_for_year_make(2025, 'Ford');
```
### Get trims for 2025 Ford F-150
```sql
SELECT * FROM get_trims_for_year_make_model(2025, 'Ford', 'f-150');
```
### Get engine/transmission options for specific vehicle
```sql
SELECT * FROM get_options_for_vehicle(2025, 'Ford', 'f-150', 'XLT');
```
### Complete vehicle configurations
```sql
SELECT * FROM complete_vehicle_configs
WHERE year = 2025 AND make = 'Ford' AND model = 'f-150'
LIMIT 10;
```
## Performance Optimization
### Indexes Created
- `idx_vehicle_year` - Single column index on year
- `idx_vehicle_make` - Single column index on make
- `idx_vehicle_model` - Single column index on model
- `idx_vehicle_year_make` - Composite index for year/make queries
- `idx_vehicle_year_make_model` - Composite index for year/make/model queries
- `idx_vehicle_year_make_model_trim` - Composite index for full cascade
### Query Performance
Dropdown queries are optimized to return results in < 50ms for typical datasets.
## Data Matching Logic
### Brand Name Transformation
- Source data (brands.json) stores names in ALL CAPS: "FORD", "ACURA", "ALFA ROMEO"
- ETL converts to Title Case: "Ford", "Acura", "Alfa Romeo"
- Preserves acronyms: BMW, GMC, KIA, MINI, FIAT, RAM
- Special cases: DeLorean, McLaren
### Engine Matching
The ETL uses intelligent pattern matching to link simple engine strings from makes-filter to detailed specs:
1. **Parse engine string**: Extract displacement (e.g., "2.0L") and configuration (e.g., "I4")
2. **Normalize displacement**: Convert Cm3 to Liters ("3506 Cm3" → "3.5L")
3. **Match to cache**: Look up in engine cache by (displacement, configuration)
4. **Create display name**: Format as "V8 3.5L", "L4 2.0L Turbo", etc.
### Transmission Linking
- Transmission data is embedded in engines.json under "Transmission Specs"
- Each engine record includes gearbox type (e.g., "6-Speed Manual")
- ETL links transmissions to vehicle records based on engine match
- Success rate: 98.9% (1,109,510 of 1,122,644 records)
- Unlinked records: primarily electric vehicles without traditional transmissions
### Configuration Equivalents
- `I4` = `L4` = `INLINE-4` = `4 Inline`
- `V6` = `V-6`
- `V8` = `V-8`
## Filtered Makes (53 Total)
All brand names are stored in Title Case format for user-friendly display.
### American Brands (12)
Acura, Buick, Cadillac, Chevrolet, Chrysler, Dodge, Ford, GMC, Hummer, Jeep, Lincoln, RAM
### Luxury/Performance (13)
Aston Martin, Bentley, Ferrari, Lamborghini, Maserati, McLaren, Porsche, Rolls Royce, Tesla, Jaguar, Audi, BMW, Land Rover
### Japanese (8)
Honda, Infiniti, Lexus, Mazda, Mitsubishi, Nissan, Subaru, Toyota
### European (9)
Alfa Romeo, FIAT, MINI, Saab, Saturn, Scion, Smart, Volkswagen, Volvo
### Other (11)
Genesis, Geo, Hyundai, KIA, Lucid, Polestar, Rivian, Lotus, Mercury, Oldsmobile, Plymouth, Pontiac
## Troubleshooting
### Container Not Running
```bash
docker compose up -d
docker compose ps
```
### Database Connection Issues
Check connection parameters in `etl_vehicle_data.py`:
```python
DB_CONFIG = {
'host': 'localhost',
'database': 'motovaultpro',
'user': 'postgres',
'password': 'postgres',
'port': 5432
}
```
### Missing JSON Files
Ensure these files exist in project root:
- `engines.json`
- `automobiles.json`
- `brands.json`
- `makes-filter/*.json` (57 files)
### Python Dependencies
```bash
pip3 install psycopg2-binary
```
## Expected Results
After successful ETL:
- **Engines**: 30,066 records
- **Transmissions**: 828 records
- **Vehicle Options**: 1,122,644 configurations
- **Years**: 47 years (1980-2026)
- **Makes**: 53 manufacturers
- **Models**: 1,741 unique models
- **Transmission Linking**: 98.9% success rate
- **Output Files**: ~52MB total (632KB engines + 21KB transmissions + 51MB vehicles)
## Next Steps
1. Create API endpoints for dropdown queries
2. Add caching layer for frequently accessed queries
3. Implement full-text search for models
4. Add vehicle images and detailed specs display
5. Create admin interface for data management

View File

@@ -1,168 +0,0 @@
# Database Update: 1980+ Year Filter Applied
## Summary
The database has been successfully updated to exclude vehicles older than 1980.
---
## Changes Applied
### Before Filter
- **Total Vehicles:** 1,213,401
- **Year Range:** 1918-2026 (93 years)
- **Database Size:** 219MB
### After Filter (1980+)
- **Total Vehicles:** 1,122,644
- **Year Range:** 1980-2026 (47 years)
- **Records Filtered:** 90,757 vehicles removed
- **Reduction:** 7.5%
---
## Validation Results
**Year Range Verified:**
- Earliest Year: 1980
- Latest Year: 2026
- Total Years: 47
**No Pre-1980 Vehicles:**
- Vehicles before 1980: 0
**Data Integrity:**
- Engines: 30,066
- Makes: 53
- Models: 1,741
- All dropdown functions working correctly
---
## Technical Implementation
### 1. ETL Script Modified (`etl_generate_sql.py`)
Added year filter constant:
```python
# Year filter - only include vehicles 1980 or newer
self.min_year = 1980
```
Applied filter in two locations:
1. **process_makes_filter()** - Filters records from makes-filter JSON files
2. **hybrid_backfill()** - Ensures backfilled records also respect the filter
### 2. SQL Files Regenerated
- `output/01_engines.sql` - ~632KB (simplified to id and name only)
- `output/02_transmissions.sql` - ~21KB (new file, id and type only)
- `output/03_vehicle_options.sql` - ~51MB (reduced from 56MB)
- Total batches: 1,123 (reduced from 1,214)
### 3. Data Transformation
**Engine Names:** Simplified to user-friendly display format
- Example: "V8 3.5L Turbo", "L4 2.0L", "V6 6.2L Supercharged"
**Transmission Types:** Simplified to user-friendly display format
- Example: "8-Speed Automatic", "6-Speed Manual", "CVT"
**Brand Names:** Converted from ALL CAPS to Title Case
- Example: "FORD" → "Ford", "LAND ROVER" → "Land Rover"
- Acronyms preserved: BMW, GMC, KIA, MINI, FIAT, RAM
### 4. Database Re-imported
Successfully imported filtered data with zero pre-1980 vehicles.
---
## How to Change the Year Filter
To use a different year cutoff (e.g., 1990, 2000), edit `etl_generate_sql.py`:
```python
class VehicleSQLGenerator:
def __init__(self):
# Change this value to your desired minimum year
self.min_year = 1990 # Example: filter to 1990+
```
Then regenerate and re-import:
```bash
python3 etl_generate_sql.py
./import_data.sh
```
---
## Database Statistics (1980+)
| Metric | Count |
|--------|-------|
| **Engines** | 30,066 |
| **Transmissions** | 828 |
| **Vehicle Options** | 1,122,644 |
| **Years** | 47 (1980-2026) |
| **Makes** | 53 |
| **Models** | 1,741 |
| **Transmission Linking** | 98.9% success |
| **Database Size** | ~250MB |
---
## Available Years
Years available in database: 1980, 1981, 1982, ..., 2024, 2025, 2026
Total: 47 consecutive years
---
## Impact on Dropdown Queries
All dropdown cascade queries remain fully functional:
```sql
-- Get years (now starts at 1980)
SELECT * FROM available_years;
-- Get makes for 1980
SELECT * FROM get_makes_for_year(1980);
-- Get makes for 2025
SELECT * FROM get_makes_for_year(2025);
```
No changes required to API or query logic.
---
## Files Updated
| File | Change |
|------|--------|
| `etl_generate_sql.py` | Added min_year filter, brand name transformation, simplified display formats |
| `output/01_engines.sql` | Regenerated with simplified format (id, name only) |
| `output/02_transmissions.sql` | New file with transmission data (id, type only) |
| `output/03_vehicle_options.sql` | Regenerated (90K fewer records, transmission linking added) |
| Database `engines` table | Re-imported with simplified schema |
| Database `transmissions` table | Newly created and populated |
| Database `vehicle_options` table | Re-imported with 1980+ filter and transmission links |
---
## Next Steps
The database is **ready for use** with the 1980+ filter applied.
If you need to:
- **Change the year filter:** Edit `min_year` in `etl_generate_sql.py` and re-run
- **Restore all years:** Set `min_year = 0` and re-run
- **Add more filters:** Modify the filter logic in `process_makes_filter()` method
---
*Filter applied: 2025-11-10*
*Minimum year: 1980*

View File

@@ -1,285 +0,0 @@
# Automotive Vehicle Selection Database - Implementation Summary
## Status: ✅ COMPLETED & OPTIMIZED
The ETL pipeline has been successfully implemented, optimized, and executed. The database is now populated with clean, user-friendly data ready for production use.
---
## Database Statistics
| Metric | Count |
|--------|-------|
| **Engines** | 30,066 |
| **Transmissions** | 828 |
| **Vehicle Options** | 1,122,644 |
| **Years** | 47 (1980-2026) |
| **Makes** | 53 |
| **Models** | 1,741 |
### Data Quality Metrics
- **Transmission Linking Success**: 98.9% (1,109,510 of 1,122,644 records)
- **Records with NULL Engine/Transmission**: 1.1% (11,951 records - primarily electric vehicles)
- **Year Filter Applied**: 1980 and newer only
---
## What Was Implemented
### 1. Database Schema (`migrations/001_create_vehicle_database.sql`)
**Tables:**
- `engines` - Simplified engine specifications (id, name)
- Names formatted as: "V8 3.5L", "L4 2.0L Turbo", "V6 6.2L Supercharged"
- `transmissions` - Simplified transmission specifications (id, type)
- Types formatted as: "8-Speed Automatic", "6-Speed Manual", "CVT"
- `vehicle_options` - Denormalized table optimized for dropdown queries (year, make, model, trim, engine_id, transmission_id)
- Make names in Title Case: "Acura", "Ford", "BMW" (not ALL CAPS)
**Views:**
- `available_years` - All distinct years
- `makes_by_year` - Makes grouped by year
- `models_by_year_make` - Models grouped by year/make
- `trims_by_year_make_model` - Trims grouped by year/make/model
- `complete_vehicle_configs` - Full vehicle details with engine info
**Functions:**
- `get_makes_for_year(year)` - Returns available makes for a specific year
- `get_models_for_year_make(year, make)` - Returns models for year/make combination
- `get_trims_for_year_make_model(year, make, model)` - Returns trims for specific vehicle
- `get_options_for_vehicle(year, make, model, trim)` - Returns engine/transmission options
**Indexes:**
- Single column indexes on year, make, model, trim
- Composite indexes for optimal cascade query performance:
- `idx_vehicle_year_make`
- `idx_vehicle_year_make_model`
- `idx_vehicle_year_make_model_trim`
### 2. ETL Script (`etl_generate_sql.py`)
A Python script that processes JSON source files and generates SQL import files:
**Data Sources Processed:**
- `engines.json` (30,066 records) - Detailed engine specifications
- `automobiles.json` (7,207 records) - Vehicle models
- `brands.json` (124 records) - Brand information
- `makes-filter/*.json` (55 files) - Filtered manufacturer data
**ETL Process:**
1. **Extract** - Loads all JSON source files
2. **Transform**
- Converts brand names from ALL CAPS to Title Case ("FORD" → "Ford")
- Creates simplified engine display names (e.g., "V8 3.5L Turbo")
- Extracts configuration (V8, I4, L6), displacement, and aspiration
- Handles missing displacement by parsing from engine name
- Creates simplified transmission display names (e.g., "8-Speed Automatic")
- Extracts speed count and type (Manual, Automatic, CVT, Dual-Clutch)
- Normalizes displacement units (Cm3 → Liters) for matching
- Matches simple engine strings (e.g., "2.0L I4") to detailed specs
- Links transmissions to vehicle records (98.9% success rate)
- Filters vehicles to 1980 and newer only
- Performs hybrid backfill for recent years (2023-2025)
3. **Load** - Generates clean, optimized SQL import files
- Proper SQL escaping (newlines, quotes, special characters)
- Empty strings converted to NULL for data integrity
- Batched inserts for optimal performance
**Output Files:**
- `output/01_engines.sql` (~632KB, 30,066 records) - Only id and name columns
- `output/02_transmissions.sql` (~21KB, 828 records) - Only id and type columns
- `output/03_vehicle_options.sql` (~51MB, 1,122,644 records)
### 3. Import Script (`import_data.sh`)
Bash script that:
1. Runs database schema migration
2. Imports engines from SQL file
3. Imports transmissions from SQL file
4. Imports vehicle options from SQL file
5. Validates imported data with queries
---
## How to Use the Database
### Running the ETL Pipeline
```bash
# Step 1: Generate SQL files from JSON data
python3 etl_generate_sql.py
# Step 2: Import SQL files into database
./import_data.sh
```
### Example Dropdown Queries
**Get available years:**
```sql
SELECT * FROM available_years;
```
**Get makes for 2025:**
```sql
SELECT * FROM get_makes_for_year(2025);
```
**Get Ford models for 2025:**
```sql
SELECT * FROM get_models_for_year_make(2025, 'Ford');
```
**Get trims for 2025 Ford F-150:**
```sql
SELECT * FROM get_trims_for_year_make_model(2025, 'Ford', 'f-150');
```
**Get complete vehicle configuration:**
```sql
SELECT * FROM complete_vehicle_configs
WHERE year = 2025 AND make = 'Ford' AND model = 'f-150'
LIMIT 10;
```
### Accessing the Database
```bash
# Via Docker exec
docker exec -it mvp-postgres psql -U postgres -d motovaultpro
# Direct SQL query
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT * FROM available_years;"
```
---
## Data Flow: Year → Make → Model → Trim → Engine
The database is designed to support cascading dropdowns for vehicle selection:
1. **User selects Year** → Query: `get_makes_for_year(year)`
2. **User selects Make** → Query: `get_models_for_year_make(year, make)`
3. **User selects Model** → Query: `get_trims_for_year_make_model(year, make, model)`
4. **User selects Trim** → Query: `get_options_for_vehicle(year, make, model, trim)`
Each query is optimized with composite indexes for sub-50ms response times.
---
## Files Created
| File | Description | Size |
|------|-------------|------|
| `migrations/001_create_vehicle_database.sql` | Database schema | ~8KB |
| `etl_generate_sql.py` | ETL script (generates SQL files) | ~20KB |
| `import_data.sh` | Import script | ~2KB |
| `output/01_engines.sql` | Engine data | 34MB |
| `output/03_vehicle_options.sql` | Vehicle options data | 56MB |
| `ETL_README.md` | Detailed documentation | ~8KB |
| `IMPLEMENTATION_SUMMARY.md` | This file | ~5KB |
---
## Key Design Decisions
### 1. SQL File Generation (Not Direct DB Connection)
- **Why:** Avoids dependency installation in Docker container
- **Benefit:** Clean separation of ETL and import processes
- **Trade-off:** Requires intermediate storage (90MB of SQL files)
### 2. Denormalized vehicle_options Table
- **Why:** Optimized for read-heavy dropdown queries
- **Benefit:** Single table queries with composite indexes = fast lookups
- **Trade-off:** Some data duplication (1.2M records)
### 3. Hybrid Backfill for Recent Years
- **Why:** makes-filter data may not include latest 2023-2025 models
- **Benefit:** Database includes most recent vehicle data
- **Trade-off:** Slight data inconsistency (backfilled records marked with "Base" trim)
### 4. Engine Matching by Displacement + Configuration
- **Why:** makes-filter has simple strings ("2.0L I4"), engines.json has detailed specs
- **Benefit:** Links dropdown data to rich engine specifications
- **Trade-off:** ~0 matches if displacement/config formats don't align perfectly
---
## Known Limitations
1. **Electric Vehicles Have NULL Engine/Transmission IDs (1.1%)**
- Occurs when engine string from makes-filter doesn't match traditional displacement patterns
- Example: Tesla models with "Electric" motors don't have displacement specs
- Affects 11,951 of 1,122,644 records
- Future enhancement: Add electric motor specifications
2. **Model Names Have Inconsistencies**
- Some models use underscores (`bronco_sport` vs `Bronco Sport`)
- Model name casing varies between sources
- Future enhancement: Normalize model names to Title Case
3. **Engine Configuration Variations**
- Some engines show "4 Inline" while others show "L4" or "I4"
- All refer to inline 4-cylinder but use different notation
- Source data inconsistency from autoevolution.com
---
## Next Steps / Recommendations
### Immediate
1. ✅ Database is functional and ready for API integration
2. ✅ Dropdown queries are working and optimized
### Short Term
1. **Clean up model names** - Remove HTML entities, normalize formatting
2. **Add transmission data** - Find alternative source or manual entry
3. **Filter year range** - Add view for "modern vehicles" (e.g., 2000+)
4. **Add vehicle images** - Link to photo URLs from automobiles.json
### Medium Term
1. **Create REST API** - Build endpoints for dropdown queries
2. **Add caching layer** - Redis/Memcached for frequently accessed data
3. **Full-text search** - PostgreSQL FTS for model name searching
4. **Admin interface** - CRUD operations for data management
### Long Term
1. **Real-time updates** - Webhook/API to sync with autoevolution.com
2. **User preferences** - Save favorite vehicles, comparison features
3. **Analytics** - Track popular makes/models, search patterns
4. **Mobile optimization** - Optimize queries for mobile app usage
---
## Performance Notes
- **Index Coverage:** All dropdown queries use composite indexes
- **Expected Query Time:** < 50ms for typical dropdown query
- **Database Size:** ~250MB with all data and indexes
- **Batch Insert Performance:** 1000 records per batch = optimal
---
## Testing Checklist
- [x] Schema migration runs successfully
- [x] Engines import (30,066 records)
- [x] Vehicle options import (1,213,401 records)
- [x] available_years view returns data
- [x] get_makes_for_year() function works
- [x] get_models_for_year_make() function works
- [x] get_trims_for_year_make_model() function works
- [x] Composite indexes created
- [x] Foreign key relationships established
- [x] Year range validated (1918-2026)
- [x] Make count validated (53 makes)
---
## Conclusion
The automotive vehicle selection database is **complete and operational**. The database contains over 1.2 million vehicle configurations spanning 93 years and 53 manufacturers, optimized for cascading dropdown queries with sub-50ms response times.
The ETL pipeline is **production-ready** and can be re-run at any time to refresh data from updated JSON sources. All scripts are documented and executable with a single command.
**Status: ✅ READY FOR API DEVELOPMENT**

View File

@@ -1,117 +0,0 @@
# Quick Start Guide - Automotive Vehicle Database
## Database Status: ✅ OPERATIONAL
- **30,066** engines
- **828** transmissions
- **1,122,644** vehicle configurations
- **47** years (1980-2026)
- **53** makes
- **1,741** models
- **98.9%** transmission linking success
---
## Access the Database
```bash
docker exec -it mvp-postgres psql -U postgres -d motovaultpro
```
---
## Essential Queries
### 1. Get All Available Years
```sql
SELECT * FROM available_years;
```
### 2. Get Makes for a Specific Year
```sql
SELECT * FROM get_makes_for_year(2024);
```
### 3. Get Models for Year + Make
```sql
SELECT * FROM get_models_for_year_make(2024, 'Ford');
```
### 4. Get Trims for Year + Make + Model
```sql
SELECT * FROM get_trims_for_year_make_model(2024, 'Ford', 'f-150');
```
### 5. Get Complete Vehicle Details
```sql
SELECT * FROM complete_vehicle_configs
WHERE year = 2024
AND make = 'Ford'
AND model = 'f-150'
LIMIT 10;
```
---
## Refresh the Database
```bash
# Re-generate SQL files from JSON source data
python3 etl_generate_sql.py
# Re-import into database
./import_data.sh
```
---
## Files Overview
| File | Purpose | Size |
|------|---------|------|
| `etl_generate_sql.py` | Generate SQL import files from JSON | ~20KB |
| `import_data.sh` | Import SQL files into database | ~2KB |
| `migrations/001_create_vehicle_database.sql` | Database schema | ~8KB |
| `output/01_engines.sql` | Engine data (id, name only) | ~632KB |
| `output/02_transmissions.sql` | Transmission data (id, type only) | ~21KB |
| `output/03_vehicle_options.sql` | Vehicle configurations | ~51MB |
| **Total Output** | | **~52MB** |
---
## Database Schema
```
engines
├── id (PK)
└── name (e.g., "V8 3.5L Turbo", "L4 2.0L")
transmissions
├── id (PK)
└── type (e.g., "8-Speed Automatic", "6-Speed Manual")
vehicle_options
├── id (PK)
├── year (1980-2026)
├── make (Title Case: "Ford", "Acura", "Land Rover")
├── model
├── trim
├── engine_id (FK → engines)
└── transmission_id (FK → transmissions)
```
---
## Performance
- **Query Time:** < 50ms (composite indexes)
- **Database Size:** ~250MB (with indexes)
- **SQL Import Files:** ~52MB total
- **Batch Insert Size:** 1,000 records per batch
---
## Support
- **Full Documentation:** See `ETL_README.md`
- **Implementation Details:** See `IMPLEMENTATION_SUMMARY.md`

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1,653 +0,0 @@
#!/usr/bin/env python3
"""
ETL Script for Automotive Vehicle Selection Database
Generates SQL import files from local scraped data only (no network).
Output is constrained to a configurable year window (default 20002026).
"""
import json
import os
import re
from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple
class VehicleSQLGenerator:
def __init__(self):
self.makes_filter_dir = Path("makes-filter")
self.engines_data: List[Dict] = []
self.automobiles_data: List[Dict] = []
self.brands_data: List[Dict] = []
# Year window (configurable)
self.min_year = int(os.getenv("MIN_YEAR", "2000"))
self.max_year = int(os.getenv("MAX_YEAR", "2026"))
# Output SQL files
self.engines_sql_file = "output/01_engines.sql"
self.transmissions_sql_file = "output/02_transmissions.sql"
self.vehicles_sql_file = "output/03_vehicle_options.sql"
# Data structures populated during ETL
self.brand_name_map: Dict[str, str] = {}
self.known_models_by_make: Dict[str, Set[str]] = defaultdict(set)
# Baseline records from makes-filter with trims/engines per year/make/model
self.baseline_records: List[Dict] = []
self.evidence_by_model: Dict[Tuple[str, str], List[Dict]] = defaultdict(list)
self.vehicle_records: List[Dict] = []
# Dimension maps (populated after vehicle records are built)
self.engine_name_to_id: Dict[str, int] = {}
self.trans_name_to_id: Dict[str, int] = {}
# ------------------------------------------------------------------
# Loading and helper utilities
# ------------------------------------------------------------------
def load_json_files(self):
print("\n📂 Loading source JSON files...")
with open("engines.json", "r", encoding="utf-8") as f:
self.engines_data = json.load(f)
print(f" ✓ Loaded {len(self.engines_data):,} engine records")
with open("automobiles.json", "r", encoding="utf-8") as f:
self.automobiles_data = json.load(f)
print(f" ✓ Loaded {len(self.automobiles_data):,} automobile records")
with open("brands.json", "r", encoding="utf-8") as f:
self.brands_data = json.load(f)
print(f" ✓ Loaded {len(self.brands_data):,} brand records")
self.build_brand_name_map()
def build_brand_name_map(self):
keep_uppercase = {
"BMW",
"GMC",
"AC",
"MG",
"KIA",
"MINI",
"FIAT",
"RAM",
"KTM",
"FSO",
"ARO",
"TVR",
"NIO",
}
special_cases = {"delorean": "DeLorean", "mclaren": "McLaren"}
for brand in self.brands_data:
raw = brand.get("name", "").strip()
if not raw:
continue
slug = raw.lower().replace(" ", "_")
if slug in special_cases:
canonical = special_cases[slug]
elif raw in keep_uppercase:
canonical = raw
else:
canonical = raw.title()
self.brand_name_map[slug] = canonical
self.brand_name_map[raw.lower()] = canonical
def get_canonical_make_name(self, name: str) -> str:
slug = name.lower().replace(" ", "_")
if slug in self.brand_name_map:
return self.brand_name_map[slug]
spaced = slug.replace("_", " ")
if spaced in self.brand_name_map:
return self.brand_name_map[spaced]
return spaced.title()
def format_model_name(self, model_slug: str) -> str:
if not model_slug:
return ""
return model_slug.replace("_", " ").strip().title()
def classify_fuel_label(self, fuel: str) -> str:
fuel_lower = (fuel or "").lower()
if "electric" in fuel_lower:
return "Electric"
if "diesel" in fuel_lower:
return "Diesel"
if "hybrid" in fuel_lower:
return "Hybrid"
return "Gas"
def normalize_engine_display(self, display: Optional[str]) -> Optional[str]:
if not display:
return None
cleaned = display.strip()
return cleaned if cleaned else None
# ------------------------------------------------------------------
# Parsing helpers for evidence building
# ------------------------------------------------------------------
def parse_year_range_from_name(self, name: str) -> Optional[Tuple[int, int]]:
"""
Extract year or year range from automobile name.
Examples:
"CHEVROLET Corvette C4 Convertible 1984-1996 ..." -> (1984, 1996)
"2021-Present" -> (2021, self.max_year)
"2024 ..." -> (2024, 2024)
"""
match = re.search(r"(19|20)\d{2}(?:-(\d{4}|Present))?", name)
if not match:
return None
start = int(match.group(0).split("-")[0])
end_part = None
if "-" in match.group(0):
end_str = match.group(0).split("-")[1]
if end_str.lower() == "present":
end_part = self.max_year
else:
end_part = int(end_str)
end = end_part if end_part else start
return (max(start, self.min_year), min(end, self.max_year))
def split_model_and_trim(
self, make: str, candidate: str, known_models: Set[str]
) -> Tuple[Optional[str], Optional[str]]:
"""
Split the automobile title (with years removed) into model + trim by matching
the longest known model prefix. Returns (model, trim).
"""
candidate_clean = candidate.strip()
if not candidate_clean:
return (None, None)
# Try longest model name first
for model in sorted(known_models, key=len, reverse=True):
pattern = re.compile(rf"^{re.escape(model)}\b", re.IGNORECASE)
match = pattern.match(candidate_clean)
if match:
remaining = candidate_clean[match.end() :].strip()
model_name = model
trim_name = remaining if remaining else "Base"
return (model_name, trim_name)
# If nothing matched, give up to avoid inventing models
return (None, None)
# ------------------------------------------------------------------
# Engine / transmission formatting helpers
# ------------------------------------------------------------------
def extract_engine_specs(self, engine_record: Dict) -> Dict:
specs = engine_record.get("specs", {})
engine_specs = specs.get("Engine Specs", {})
trans_specs = specs.get("Transmission Specs", {})
return {
"name": engine_record.get("name", ""),
"displacement": engine_specs.get("Displacement:", ""),
"configuration": engine_specs.get("Cylinders:", ""),
"horsepower": engine_specs.get("Power:", ""),
"torque": engine_specs.get("Torque:", ""),
"fuel_type": engine_specs.get("Fuel:", ""),
"fuel_system": engine_specs.get("Fuel System:", ""),
"aspiration": engine_specs.get("Aspiration:", ""),
"transmission_type": trans_specs.get("Gearbox:", ""),
"drive_type": trans_specs.get("Drive Type:", ""),
"specs_json": specs,
}
def normalize_displacement(self, disp_str: str) -> Optional[str]:
if not disp_str:
return None
disp_str = disp_str.strip()
if disp_str.upper().endswith("L"):
match = re.search(r"(\d+\.?\d*)", disp_str)
if match:
liters = float(match.group(1))
return f"{liters:.1f}L"
cm3_match = re.search(r"(\d+)\s*Cm3", disp_str, re.IGNORECASE)
if cm3_match:
cm3 = int(cm3_match.group(1))
liters = cm3 / 1000.0
return f"{liters:.1f}L"
return None
def format_engine_display(self, specs: Dict) -> Optional[str]:
parts: List[str] = []
config = specs.get("configuration", "").strip()
if config:
parts.append(config.upper())
disp = self.normalize_displacement(specs.get("displacement", ""))
if not disp:
name = specs.get("name", "")
disp_match = re.search(r"(\d+\.?\d*)\s*L", name, re.IGNORECASE)
if disp_match:
disp = f"{float(disp_match.group(1)):.1f}L"
if disp:
parts.append(disp)
aspiration = specs.get("aspiration", "").strip()
fuel_system = specs.get("fuel_system", "").strip()
combined = f"{aspiration} {fuel_system}".lower()
if combined and "naturally aspirated" not in combined:
if "turbo" in combined:
parts.append("Turbo")
elif "supercharg" in combined:
parts.append("Supercharged")
if not parts:
return None
return " ".join(parts)
def format_transmission_display(self, trans_type: str, speeds: Optional[str]) -> str:
trans_clean = trans_type.strip() if trans_type else ""
if trans_clean:
trans_lower = trans_clean.lower()
if "cvt" in trans_lower:
return "CVT"
speed_text = None
if speeds:
speed_text = f"{speeds}-Speed"
elif trans_clean:
speed_match = re.search(r"(\d+)[- ]?[Ss]peed", trans_clean)
if speed_match:
speed_text = f"{speed_match.group(1)}-Speed"
kind = None
if trans_clean:
lower = trans_clean.lower()
if "manual" in lower:
kind = "Manual"
elif "automatic" in lower or "auto" in lower:
kind = "Automatic"
elif "direct" in lower or "dct" in lower:
kind = "Dual-Clutch"
if speed_text and kind:
return f"{speed_text} {kind}"
if kind:
return kind
if speed_text:
return f"{speed_text} Automatic"
return "Automatic"
def normalize_engine_string(self, engine: str) -> Optional[str]:
if not engine:
return None
eng = engine.strip()
if not eng:
return None
return eng
# ------------------------------------------------------------------
# Phase 1: Build baseline from makes-filter (year/make/model + trims/engines)
# ------------------------------------------------------------------
def build_known_models(self):
print("\n📖 Building known models from makes-filter...")
if not self.makes_filter_dir.exists():
raise FileNotFoundError("makes-filter directory not found")
for json_file in sorted(self.makes_filter_dir.glob("*.json")):
make_name = self.get_canonical_make_name(json_file.stem)
with open(json_file, "r", encoding="utf-8") as f:
make_data = json.load(f)
for _, year_entries in make_data.items():
for year_entry in year_entries:
year = int(year_entry.get("year", 0))
if year < self.min_year or year > self.max_year:
continue
for model in year_entry.get("models", []):
model_name = self.format_model_name(model.get("name", ""))
if model_name:
self.known_models_by_make[make_name].add(model_name)
total_models = sum(len(v) for v in self.known_models_by_make.values())
print(f" ✓ Collected {total_models} model names across makes")
def build_baseline_records(self):
print("\n🧩 Building baseline records (year/make/model + trims/engines)...")
records: List[Dict] = []
for json_file in sorted(self.makes_filter_dir.glob("*.json")):
make_name = self.get_canonical_make_name(json_file.stem)
with open(json_file, "r", encoding="utf-8") as f:
make_data = json.load(f)
for _, year_entries in make_data.items():
for year_entry in year_entries:
year = int(year_entry.get("year", 0))
if year < self.min_year or year > self.max_year:
continue
for model in year_entry.get("models", []):
model_name = self.format_model_name(model.get("name", ""))
if not model_name:
continue
engines = [self.normalize_engine_string(e) for e in model.get("engines", [])]
engines = [e for e in engines if e]
submodels = model.get("submodels", [])
if not submodels:
submodels = ["Base"]
trims_payload = []
for trim in submodels:
trim_name = self.format_model_name(trim)
trims_payload.append({"trim": trim_name or "Base", "engines": engines})
records.append({"year": year, "make": make_name, "model": model_name, "trims": trims_payload})
self.baseline_records = sorted(records, key=lambda r: (r["year"], r["make"].lower(), r["model"].lower()))
print(f" ✓ Baseline records: {len(self.baseline_records):,}")
# ------------------------------------------------------------------
# Phase 2: Build evidence from automobiles + engines
# ------------------------------------------------------------------
def build_automobile_evidence(self):
print("\n🔎 Building automobile evidence (trims/years/engines/transmissions)...")
brand_lookup = {b.get("id"): self.get_canonical_make_name(b.get("name", "")) for b in self.brands_data}
# Build quick index: automobile_id -> engines
engines_by_auto: Dict[int, List[Dict]] = defaultdict(list)
for engine in self.engines_data:
auto_id = engine.get("automobile_id")
if auto_id:
engines_by_auto[auto_id].append(engine)
for auto in self.automobiles_data:
auto_id = auto.get("id")
brand_id = auto.get("brand_id")
make = brand_lookup.get(brand_id)
if not make:
continue
year_range = self.parse_year_range_from_name(auto.get("name", ""))
if not year_range:
continue
year_start, year_end = year_range
if year_end < self.min_year or year_start > self.max_year:
continue
known_models = self.known_models_by_make.get(make, set())
if not known_models:
continue
name_clean = auto.get("name", "")
# Remove make prefix if present
name_clean = re.sub(rf"^{re.escape(make)}\s+", "", name_clean, flags=re.IGNORECASE)
# Remove year substring
name_clean = re.sub(r"(19|20)\d{2}(-\d{4}|-Present)?", "", name_clean).strip()
model, trim = self.split_model_and_trim(make, self.format_model_name(name_clean), known_models)
if not model:
continue
trim = self.format_model_name(trim or "Base")
engine_displays: Set[str] = set()
fuel_labels: Set[str] = set()
transmission_displays: Set[str] = set()
for engine_record in engines_by_auto.get(auto_id, []):
specs = self.extract_engine_specs(engine_record)
display = self.normalize_engine_display(self.format_engine_display(specs))
fuel_label = self.classify_fuel_label(specs.get("fuel_type", ""))
fuel_labels.add(fuel_label)
if display:
engine_displays.add(display)
trans_type = specs.get("transmission_type", "")
speed_match = re.search(r"(\d+)", trans_type) if trans_type else None
speeds = speed_match.group(1) if speed_match else None
trans_display = self.format_transmission_display(trans_type, speeds)
if trans_display:
transmission_displays.add(trans_display)
self.evidence_by_model[(make, model)].append(
{
"year_start": year_start,
"year_end": year_end,
"trim": trim,
"engines": engine_displays,
"transmissions": transmission_displays,
"fuel_labels": fuel_labels,
}
)
total_entries = sum(len(v) for v in self.evidence_by_model.values())
print(f" ✓ Evidence entries: {total_entries:,}")
# ------------------------------------------------------------------
# Phase 3: Build vehicle records combining baseline + evidence
# ------------------------------------------------------------------
def build_vehicle_records(self):
print("\n🚗 Building vehicle option records...")
records: List[Dict] = []
for baseline in self.baseline_records:
year = baseline["year"]
make = baseline["make"]
model = baseline["model"]
trims_payload = baseline["trims"]
evidence_entries = self.evidence_by_model.get((make, model), [])
applicable = [e for e in evidence_entries if e["year_start"] <= year <= e["year_end"]]
# Build evidence map per trim (case-insensitive match)
evidence_by_trim: Dict[str, Dict[str, Set[str]]] = defaultdict(lambda: {"engines": set(), "transmissions": set(), "fuel_labels": set()})
for entry in applicable:
trim_key = entry["trim"].lower()
evidence_by_trim[trim_key]["engines"].update(entry["engines"])
evidence_by_trim[trim_key]["transmissions"].update(entry["transmissions"])
evidence_by_trim[trim_key]["fuel_labels"].update(entry["fuel_labels"])
# Include evidence-only trims not present in makes-filter (if any)
all_trim_keys = set()
for t in trims_payload:
all_trim_keys.add(t["trim"].lower())
for trim_key in evidence_by_trim.keys():
all_trim_keys.add(trim_key)
if not all_trim_keys:
all_trim_keys.add("base")
for trim_key in sorted(all_trim_keys):
# If evidence exists for this trim but none cover this year, skip (avoids impossible year/trim combos)
evidence_for_trim = [e for e in evidence_entries if e["trim"].lower() == trim_key]
if evidence_for_trim and trim_key not in evidence_by_trim:
continue
# Determine trim display
trim_display = None
for t in trims_payload:
if t["trim"].lower() == trim_key:
trim_display = t["trim"]
break
if not trim_display:
# Use evidence trim name if not in makes-filter
for entry in applicable:
if entry["trim"].lower() == trim_key:
trim_display = entry["trim"]
break
if not trim_display:
trim_display = "Base"
# Engines: start from makes-filter engines for this trim
engines_set: Set[str] = set()
for t in trims_payload:
if t["trim"].lower() == trim_key:
for eng in t.get("engines", []):
norm = self.normalize_engine_string(eng)
if norm:
engines_set.add(norm)
# Overlay evidence engines for this trim
if trim_key in evidence_by_trim:
engines_set.update(evidence_by_trim[trim_key]["engines"])
# Fuel labels from evidence for fallback
fuel_labels = evidence_by_trim.get(trim_key, {}).get("fuel_labels", set())
if engines_set:
engine_names = sorted(engines_set)
else:
fallback_fuel = None
if fuel_labels:
if "Electric" in fuel_labels:
fallback_fuel = "Electric"
elif "Diesel" in fuel_labels:
fallback_fuel = "Diesel"
elif "Hybrid" in fuel_labels:
fallback_fuel = "Hybrid"
engine_names = [fallback_fuel or "Gas"]
transmissions_set: Set[str] = set()
if trim_key in evidence_by_trim:
transmissions_set.update(evidence_by_trim[trim_key]["transmissions"])
trans_names = sorted(transmissions_set) if transmissions_set else ["Manual", "Automatic"]
for engine_name in engine_names:
for trans_name in trans_names:
records.append(
{
"year": year,
"make": make,
"model": model,
"trim": trim_display,
"engine_name": engine_name,
"trans_name": trans_name,
}
)
# Deduplicate fact rows
unique_set = set()
deduped_records = []
for r in records:
key = (r["year"], r["make"].lower(), r["model"].lower(), r["trim"].lower(), r["engine_name"].lower(), r["trans_name"].lower())
if key in unique_set:
continue
unique_set.add(key)
deduped_records.append(r)
self.vehicle_records = deduped_records
print(f" ✓ Vehicle records after dedupe: {len(self.vehicle_records):,}")
# ------------------------------------------------------------------
# Phase 4: Assign dimension IDs and write SQL
# ------------------------------------------------------------------
def assign_dimension_ids(self):
engine_names = sorted({r["engine_name"] for r in self.vehicle_records})
trans_names = sorted({r["trans_name"] for r in self.vehicle_records})
self.engine_name_to_id = {name: idx + 1 for idx, name in enumerate(engine_names)}
self.trans_name_to_id = {name: idx + 1 for idx, name in enumerate(trans_names)}
def write_engines_sql(self):
os.makedirs("output", exist_ok=True)
with open(self.engines_sql_file, "w", encoding="utf-8") as f:
f.write("-- Engines data import\n-- Generated by ETL script\n\nBEGIN;\n\n")
values = []
for name, idx in sorted(self.engine_name_to_id.items(), key=lambda x: x[1]):
values.append(f"({idx},'{self.sql_escape_literal(name)}')")
f.write("INSERT INTO engines (id, name) VALUES\n")
f.write(",\n".join(values))
f.write(";\n\n")
f.write(f"SELECT setval('engines_id_seq', {len(self.engine_name_to_id)});\n\nCOMMIT;\n")
def write_transmissions_sql(self):
os.makedirs("output", exist_ok=True)
with open(self.transmissions_sql_file, "w", encoding="utf-8") as f:
f.write("-- Transmissions data import\n-- Generated by ETL script\n\nBEGIN;\n\n")
values = []
for name, idx in sorted(self.trans_name_to_id.items(), key=lambda x: x[1]):
values.append(f"({idx},'{self.sql_escape_literal(name)}')")
f.write("INSERT INTO transmissions (id, type) VALUES\n")
f.write(",\n".join(values))
f.write(";\n\n")
f.write(f"SELECT setval('transmissions_id_seq', {len(self.trans_name_to_id)});\n\nCOMMIT;\n")
def write_vehicle_options_sql(self):
os.makedirs("output", exist_ok=True)
with open(self.vehicles_sql_file, "w", encoding="utf-8") as f:
f.write("-- Vehicle options data import\n-- Generated by ETL script\n\nBEGIN;\n\n")
batch_size = 1000
total = len(self.vehicle_records)
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
batch = self.vehicle_records[start:end]
f.write("INSERT INTO vehicle_options (year, make, model, trim, engine_id, transmission_id) VALUES\n")
values = []
for record in batch:
engine_id = self.engine_name_to_id[record["engine_name"]]
trans_id = self.trans_name_to_id[record["trans_name"]]
values.append(
f"({record['year']},'{self.sql_escape_literal(record['make'])}','{self.sql_escape_literal(record['model'])}','{self.sql_escape_literal(record['trim'])}',{engine_id},{trans_id})"
)
f.write(",\n".join(values))
f.write(";\n\n")
f.write("COMMIT;\n")
# ------------------------------------------------------------------
# Utility
# ------------------------------------------------------------------
def sql_escape_literal(self, value: str) -> str:
return value.replace("\\", "\\\\").replace("'", "''")
# ------------------------------------------------------------------
# Stats
# ------------------------------------------------------------------
def generate_stats(self):
stats = {
"min_year": min(r["year"] for r in self.vehicle_records) if self.vehicle_records else None,
"max_year": max(r["year"] for r in self.vehicle_records) if self.vehicle_records else None,
"vehicle_records": len(self.vehicle_records),
"engines": len(self.engine_name_to_id),
"transmissions": len(self.trans_name_to_id),
"makes": len({r["make"] for r in self.vehicle_records}),
"models": len({(r["make"], r["model"]) for r in self.vehicle_records}),
}
with open("output/stats.txt", "w", encoding="utf-8") as f:
f.write("=" * 60 + "\n")
f.write("ETL Statistics\n")
f.write("=" * 60 + "\n\n")
for key, value in stats.items():
formatted = f"{value:,}" if isinstance(value, int) else value
f.write(f"{key.replace('_', ' ').title()}: {formatted}\n")
print("\n📊 Statistics:")
for key, value in stats.items():
formatted = f"{value:,}" if isinstance(value, int) else value
print(f" {key.replace('_', ' ').title()}: {formatted}")
# ------------------------------------------------------------------
# Run
# ------------------------------------------------------------------
def run(self):
try:
print("=" * 60)
print("🚀 Automotive Vehicle ETL - SQL Generator")
print(f" Year Window: {self.min_year}{self.max_year}")
print("=" * 60)
self.load_json_files()
self.build_known_models()
self.build_baseline_records()
self.build_automobile_evidence()
self.build_vehicle_records()
self.assign_dimension_ids()
self.write_engines_sql()
self.write_transmissions_sql()
self.write_vehicle_options_sql()
self.generate_stats()
print("\n" + "=" * 60)
print("✅ SQL Files Generated Successfully!")
print("=" * 60)
print("\nGenerated files:")
print(f" - {self.engines_sql_file}")
print(f" - {self.transmissions_sql_file}")
print(f" - {self.vehicles_sql_file}")
print(f" - output/stats.txt")
print("\nNext step: Import SQL files into database")
print(" ./import_data.sh")
except Exception as e:
print(f"\n❌ ETL Pipeline Failed: {e}")
import traceback
traceback.print_exc()
raise
if __name__ == "__main__":
VehicleSQLGenerator().run()

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,506 +0,0 @@
{
"aston_martin": [
{
"year": "2023",
"models": [
{
"name": "vantage",
"engines": [
"4.0L V8",
"5.2L V12"
],
"submodels": [
"AMR",
"V12",
"Base"
]
}
]
},
{
"year": "2020",
"models": [
{
"name": "db11",
"engines": [
"4.0L V8"
],
"submodels": []
},
{
"name": "dbs",
"engines": [
"5.2L V12"
],
"submodels": []
},
{
"name": "vantage",
"engines": [
"4.0L V8",
"5.2L V12"
],
"submodels": [
"AMR",
"V12",
"Base"
]
}
]
},
{
"year": "2019",
"models": [
{
"name": "vantage",
"engines": [
"4.0L V8",
"5.2L V12"
],
"submodels": [
"AMR",
"V12",
"Base"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "rapide",
"engines": [
"6.0L V12"
],
"submodels": []
}
]
},
{
"year": "2017",
"models": [
{
"name": "v12_vantage",
"engines": [
"6.0L V12"
],
"submodels": [
"Base",
"S"
]
},
{
"name": "vanquish",
"engines": [
"6.0L V12"
],
"submodels": [
"Carbon",
"Base",
"Volante"
]
}
]
},
{
"year": "2016",
"models": [
{
"name": "rapide",
"engines": [
"6.0L V12"
],
"submodels": []
},
{
"name": "v12_vantage",
"engines": [
"6.0L V12"
],
"submodels": [
"Base",
"S"
]
},
{
"name": "vanquish",
"engines": [
"6.0L V12"
],
"submodels": [
"Carbon",
"Base",
"Volante"
]
}
]
},
{
"year": "2015",
"models": [
{
"name": "db9",
"engines": [
"6.0L V12"
],
"submodels": [
"Volante",
"Base"
]
},
{
"name": "rapide",
"engines": [
"6.0L V12"
],
"submodels": []
},
{
"name": "v12_vantage",
"engines": [
"6.0L V12"
],
"submodels": [
"Base",
"S"
]
},
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
},
{
"name": "vanquish",
"engines": [
"6.0L V12"
],
"submodels": [
"Carbon",
"Base",
"Volante"
]
}
]
},
{
"year": "2014",
"models": [
{
"name": "db9",
"engines": [
"6.0L V12"
],
"submodels": [
"Volante",
"Base"
]
},
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
},
{
"name": "vanquish",
"engines": [
"6.0L V12"
],
"submodels": [
"Carbon",
"Base",
"Volante"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2011",
"models": [
{
"name": "v12_vantage",
"engines": [
"6.0L V12"
],
"submodels": [
"Base",
"S"
]
},
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2010",
"models": [
{
"name": "db9",
"engines": [
"6.0L V12"
],
"submodels": [
"Volante",
"Base"
]
},
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "db9",
"engines": [
"6.0L V12"
],
"submodels": [
"Volante",
"Base"
]
},
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "db9",
"engines": [
"6.0L V12"
],
"submodels": [
"Volante",
"Base"
]
},
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "v8_vantage",
"engines": [
"4.3L V8",
"4.7L V8"
],
"submodels": [
"GT",
"S",
"Base"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "db9",
"engines": [
"6.0L V12"
],
"submodels": [
"Volante",
"Base"
]
},
{
"name": "vantage",
"engines": [
"4.0L V8",
"5.2L V12"
],
"submodels": [
"AMR",
"V12",
"Base"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "db7",
"engines": [
"6.0L V12"
],
"submodels": [
"Vantage Volante",
"Vantage"
]
}
]
},
{
"year": "2001",
"models": [
{
"name": "db7",
"engines": [
"6.0L V12"
],
"submodels": [
"Vantage Volante",
"Vantage"
]
}
]
},
{
"year": "1993",
"models": [
{
"name": "virage",
"engines": [
"5.3L V8"
],
"submodels": [
"Volante"
]
}
]
},
{
"year": "1990",
"models": [
{
"name": "virage",
"engines": [
"5.3L V8"
],
"submodels": [
"Volante"
]
}
]
},
{
"year": "1983",
"models": [
{
"name": "v-8",
"engines": [
"5.3L V8"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,427 +0,0 @@
{
"bentley": [
{
"year": "2023",
"models": [
{
"name": "flying_spur",
"engines": [
"2.9L V6 MILD HYBRID EV- (MHEV)",
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Hybrid",
"V8",
"W12",
"S Hybrid",
"Base"
]
}
]
},
{
"year": "2022",
"models": [
{
"name": "flying_spur",
"engines": [
"2.9L V6 MILD HYBRID EV- (MHEV)",
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Hybrid",
"V8",
"W12",
"S Hybrid",
"Base"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
},
{
"name": "flying_spur",
"engines": [
"2.9L V6 MILD HYBRID EV- (MHEV)",
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Hybrid",
"V8",
"W12",
"S Hybrid",
"Base"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "bentayga",
"engines": [
"6.0L W12"
],
"submodels": [
"W12 Signature",
"Black Edition"
]
},
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
}
]
},
{
"year": "2017",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
}
]
},
{
"year": "2016",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
},
{
"name": "flying_spur",
"engines": [
"2.9L V6 MILD HYBRID EV- (MHEV)",
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Hybrid",
"V8",
"W12",
"S Hybrid",
"Base"
]
},
{
"name": "mulsanne",
"engines": [
"6.8L V8"
],
"submodels": [
"Base",
"Speed"
]
}
]
},
{
"year": "2014",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
},
{
"name": "mulsanne",
"engines": [
"6.8L V8"
],
"submodels": [
"Base",
"Speed"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
},
{
"name": "flying_spur",
"engines": [
"2.9L V6 MILD HYBRID EV- (MHEV)",
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Hybrid",
"V8",
"W12",
"S Hybrid",
"Base"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "arnage",
"engines": [
"4.4L V8",
"6.8L V8"
],
"submodels": [
"Base",
"R"
]
},
{
"name": "continental",
"engines": [
"4.0L V8",
"6.0L W12 FLEX",
"6.0L W12"
],
"submodels": [
"Base",
"GTC",
"Flying Spur Speed",
"GT V8 S",
"GTC V8 S",
"Flying Spur",
"GT",
"GT Speed"
]
}
]
},
{
"year": "1999",
"models": [
{
"name": "arnage",
"engines": [
"4.4L V8",
"6.8L V8"
],
"submodels": [
"Base",
"R"
]
}
]
},
{
"year": "1997",
"models": [
{
"name": "brooklands",
"engines": [
"6.8L V8"
],
"submodels": []
}
]
},
{
"year": "1996",
"models": [
{
"name": "azure",
"engines": [],
"submodels": []
}
]
},
{
"year": "1989",
"models": [
{
"name": "turbo_r",
"engines": [
"6.8L V8"
],
"submodels": []
}
]
},
{
"year": "1963",
"models": [
{
"name": "s3_series",
"engines": [
"6.2L V8"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,607 +0,0 @@
{
"ferrari": [
{
"year": "2024",
"models": [
{
"name": "296_gts",
"engines": [
"3.0L V6 PLUG-IN HYBRID EV- (PHEV)"
],
"submodels": []
}
]
},
{
"year": "2022",
"models": [
{
"name": "f8_spider",
"engines": [
"3.9L V8"
],
"submodels": []
}
]
},
{
"year": "2019",
"models": [
{
"name": "portofino",
"engines": [
"3.9L V8"
],
"submodels": []
}
]
},
{
"year": "2018",
"models": [
{
"name": "488_spider",
"engines": [
"3.9L V8"
],
"submodels": []
}
]
},
{
"year": "2017",
"models": [
{
"name": "gtc4lusso",
"engines": [
"6.3L V12"
],
"submodels": []
}
]
},
{
"year": "2016",
"models": [
{
"name": "488_gtb",
"engines": [
"3.9L V8"
],
"submodels": []
},
{
"name": "ff",
"engines": [
"6.3L V12"
],
"submodels": []
}
]
},
{
"year": "2015",
"models": [
{
"name": "458_italia",
"engines": [
"4.5L V8"
],
"submodels": [
"Base"
]
},
{
"name": "458_spider",
"engines": [
"4.5L V8"
],
"submodels": []
},
{
"name": "california_t",
"engines": [
"3.8L V8",
"3.9L V8"
],
"submodels": []
},
{
"name": "f12_berlinetta",
"engines": [
"6.3L V12"
],
"submodels": []
}
]
},
{
"year": "2014",
"models": [
{
"name": "458_italia",
"engines": [
"4.5L V8"
],
"submodels": [
"Base"
]
},
{
"name": "california",
"engines": [
"4.3L V8"
],
"submodels": []
},
{
"name": "laferrari",
"engines": [
"6.3L V12"
],
"submodels": []
}
]
},
{
"year": "2013",
"models": [
{
"name": "458_italia",
"engines": [
"4.5L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "458_italia",
"engines": [
"4.5L V8"
],
"submodels": [
"Base"
]
},
{
"name": "ff",
"engines": [
"6.3L V12"
],
"submodels": []
}
]
},
{
"year": "2010",
"models": [
{
"name": "458_italia",
"engines": [
"4.5L V8"
],
"submodels": [
"Base"
]
},
{
"name": "california",
"engines": [
"4.3L V8"
],
"submodels": []
}
]
},
{
"year": "2009",
"models": [
{
"name": "599_gtb",
"engines": [
"6.0L V12"
],
"submodels": []
}
]
},
{
"year": "2008",
"models": [
{
"name": "599_gtb",
"engines": [
"6.0L V12"
],
"submodels": []
},
{
"name": "f430",
"engines": [
"4.3L V8"
],
"submodels": [
"Spider",
"Base"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "f430",
"engines": [
"4.3L V8"
],
"submodels": [
"Spider",
"Base"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "612_scaglietti",
"engines": [
"5.7L V12"
],
"submodels": []
},
{
"name": "f430",
"engines": [
"4.3L V8"
],
"submodels": [
"Spider",
"Base"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "f430",
"engines": [
"4.3L V8"
],
"submodels": [
"Spider",
"Base"
]
},
{
"name": "superamerica",
"engines": [
"5.7L V12"
],
"submodels": []
}
]
},
{
"year": "2004",
"models": [
{
"name": "360",
"engines": [
"3.6L V8"
],
"submodels": [
"Challenge Stradale",
"Modena",
"Spider"
]
},
{
"name": "575_m_maranello",
"engines": [
"5.7L V12"
],
"submodels": []
},
{
"name": "enzo",
"engines": [],
"submodels": []
}
]
},
{
"year": "2003",
"models": [
{
"name": "360",
"engines": [
"3.6L V8"
],
"submodels": [
"Challenge Stradale",
"Modena",
"Spider"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "360",
"engines": [
"3.6L V8"
],
"submodels": [
"Challenge Stradale",
"Modena",
"Spider"
]
}
]
},
{
"year": "2001",
"models": [
{
"name": "360",
"engines": [
"3.6L V8"
],
"submodels": [
"Challenge Stradale",
"Modena",
"Spider"
]
}
]
},
{
"year": "2000",
"models": [
{
"name": "360",
"engines": [
"3.6L V8"
],
"submodels": [
"Challenge Stradale",
"Modena",
"Spider"
]
}
]
},
{
"year": "1998",
"models": [
{
"name": "456_gt",
"engines": [],
"submodels": []
}
]
},
{
"year": "1997",
"models": [
{
"name": "550_maranello",
"engines": [
"5.5L V12"
],
"submodels": []
},
{
"name": "f355_spider",
"engines": [
"3.5L V8"
],
"submodels": []
}
]
},
{
"year": "1996",
"models": [
{
"name": "f355_spider",
"engines": [
"3.5L V8"
],
"submodels": []
}
]
},
{
"year": "1995",
"models": [
{
"name": "f355_berlinetta",
"engines": [],
"submodels": []
}
]
},
{
"year": "1992",
"models": [
{
"name": "348_tb",
"engines": [
"3.4L V8"
],
"submodels": []
}
]
},
{
"year": "1991",
"models": [
{
"name": "mondial_t",
"engines": [
"3.4L V8"
],
"submodels": []
},
{
"name": "testarossa",
"engines": [
"4.9L H12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1990",
"models": [
{
"name": "348_ts",
"engines": [
"3.4L V8"
],
"submodels": []
}
]
},
{
"year": "1987",
"models": [
{
"name": "328_gts",
"engines": [
"3.2L V8"
],
"submodels": []
},
{
"name": "mondial_3_2",
"engines": [
"3.2L V8"
],
"submodels": []
},
{
"name": "testarossa",
"engines": [
"4.9L H12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1985",
"models": [
{
"name": "308_gts",
"engines": [
"3.0L V8"
],
"submodels": [
"Base",
"Quattrovalvole"
]
}
]
},
{
"year": "1983",
"models": [
{
"name": "308_gts",
"engines": [
"3.0L V8"
],
"submodels": [
"Base",
"Quattrovalvole"
]
}
]
},
{
"year": "1980",
"models": [
{
"name": "308_gts",
"engines": [
"3.0L V8"
],
"submodels": [
"Base",
"Quattrovalvole"
]
}
]
},
{
"year": "1977",
"models": [
{
"name": "308_gtb",
"engines": [
"3.0L V8"
],
"submodels": []
}
]
},
{
"year": "1972",
"models": [
{
"name": "365_gtc_4",
"engines": [
"4.4L V12"
],
"submodels": []
},
{
"name": "dino_246_gt",
"engines": [
"2.4L V6"
],
"submodels": []
}
]
},
{
"year": "1966",
"models": [
{
"name": "275_gtb",
"engines": [
"3.3L V12"
],
"submodels": []
},
{
"name": "500_superfast",
"engines": [
"5.0L V12"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,414 +0,0 @@
{
"genesis": [
{
"year": "2024",
"models": [
{
"name": "g90",
"engines": [
"3.3L V6",
"3.5L V6 MILD HYBRID EV- (MHEV)",
"5.0L V8"
],
"submodels": [
"E-Supercharger",
"Ultimate",
"Premium"
]
},
{
"name": "gv70",
"engines": [
"2.5L I4",
"3.5L V6"
],
"submodels": [
"Base",
"1.5 RS CVT Honda SENSING",
"Sport Plus",
"Prestige",
"Select",
"Advanced Plus",
"Sport Advanced",
"Advanced",
"Sport Prestige"
]
}
]
},
{
"year": "2023",
"models": [
{
"name": "g70",
"engines": [
"2.0L I4",
"3.3L V6"
],
"submodels": [
"Launch Edition",
"Base",
"Design",
"3.3T RWD",
"Advanced",
"Sport Prestige",
"Elite",
"Dynamic",
"Prestige",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
},
{
"name": "gv70",
"engines": [
"2.5L I4",
"3.5L V6"
],
"submodels": [
"Base",
"1.5 RS CVT Honda SENSING",
"Sport Plus",
"Prestige",
"Select",
"Advanced Plus",
"Sport Advanced",
"Advanced",
"Sport Prestige"
]
},
{
"name": "gv80",
"engines": [
"2.5L I4",
"3.5L V6"
],
"submodels": [
"Advanced",
"Advanced+",
"Prestige"
]
}
]
},
{
"year": "2022",
"models": [
{
"name": "g70",
"engines": [
"2.0L I4",
"3.3L V6"
],
"submodels": [
"Launch Edition",
"Base",
"Design",
"3.3T RWD",
"Advanced",
"Sport Prestige",
"Elite",
"Dynamic",
"Prestige",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
},
{
"name": "gv70",
"engines": [
"2.5L I4",
"3.5L V6"
],
"submodels": [
"Base",
"1.5 RS CVT Honda SENSING",
"Sport Plus",
"Prestige",
"Select",
"Advanced Plus",
"Sport Advanced",
"Advanced",
"Sport Prestige"
]
},
{
"name": "gv80",
"engines": [
"2.5L I4",
"3.5L V6"
],
"submodels": [
"Advanced",
"Advanced+",
"Prestige"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "g70",
"engines": [
"2.0L I4",
"3.3L V6"
],
"submodels": [
"Launch Edition",
"Base",
"Design",
"3.3T RWD",
"Advanced",
"Sport Prestige",
"Elite",
"Dynamic",
"Prestige",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
},
{
"name": "gv80",
"engines": [
"2.5L I4",
"3.5L V6"
],
"submodels": [
"Advanced",
"Advanced+",
"Prestige"
]
}
]
},
{
"year": "2020",
"models": [
{
"name": "g70",
"engines": [
"2.0L I4",
"3.3L V6"
],
"submodels": [
"Launch Edition",
"Base",
"Design",
"3.3T RWD",
"Advanced",
"Sport Prestige",
"Elite",
"Dynamic",
"Prestige",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
}
]
},
{
"year": "2019",
"models": [
{
"name": "g70",
"engines": [
"2.0L I4",
"3.3L V6"
],
"submodels": [
"Launch Edition",
"Base",
"Design",
"3.3T RWD",
"Advanced",
"Sport Prestige",
"Elite",
"Dynamic",
"Prestige",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
},
{
"name": "g90",
"engines": [
"3.3L V6",
"3.5L V6 MILD HYBRID EV- (MHEV)",
"5.0L V8"
],
"submodels": [
"E-Supercharger",
"Ultimate",
"Premium"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
},
{
"name": "g90",
"engines": [
"3.3L V6",
"3.5L V6 MILD HYBRID EV- (MHEV)",
"5.0L V8"
],
"submodels": [
"E-Supercharger",
"Ultimate",
"Premium"
]
}
]
},
{
"year": "2017",
"models": [
{
"name": "g80",
"engines": [
"2.5L I4",
"3.3L V6",
"3.5L V6",
"3.8L V6",
"5.0L V8"
],
"submodels": [
"Base",
"Advanced",
"Sport Prestige",
"5.0",
"Prestige",
"1.5 RS CVT Honda SENSING",
"3.8"
]
},
{
"name": "g90",
"engines": [
"3.3L V6",
"3.5L V6 MILD HYBRID EV- (MHEV)",
"5.0L V8"
],
"submodels": [
"E-Supercharger",
"Ultimate",
"Premium"
]
}
]
}
]
}

View File

@@ -1,383 +0,0 @@
{
"geo": [
{
"year": "1997",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1996",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1995",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1994",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1993",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "storm",
"engines": [
"1.6L I4"
],
"submodels": [
"2+2",
"2+2 GSi"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1992",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "storm",
"engines": [
"1.6L I4"
],
"submodels": [
"2+2",
"2+2 GSi"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1991",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "storm",
"engines": [
"1.6L I4"
],
"submodels": [
"2+2",
"2+2 GSi"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1990",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "prizm",
"engines": [
"1.6L I4",
"1.8L I4"
],
"submodels": [
"GSi",
"LSi",
"Base"
]
},
{
"name": "storm",
"engines": [
"1.6L I4"
],
"submodels": [
"2+2",
"2+2 GSi"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
},
{
"year": "1989",
"models": [
{
"name": "metro",
"engines": [
"1.0L L3",
"1.3L I4"
],
"submodels": [
"XFi",
"LSi",
"Base"
]
},
{
"name": "tracker",
"engines": [
"1.6L I4"
],
"submodels": [
"LSi",
"Base"
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,269 +0,0 @@
{
"hummer": [
{
"year": "2010",
"models": [
{
"name": "h3",
"engines": [
"3.5L L5",
"3.7L L5",
"5.3L V8"
],
"submodels": [
"Championship Series",
"X",
"Alpha",
"Adventure",
"Luxury",
"Base"
]
},
{
"name": "h3t",
"engines": [
"3.7L L5",
"5.3L V8 FLEX",
"5.3L V8"
],
"submodels": [
"Base",
"Alpha"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
},
{
"name": "h3",
"engines": [
"3.5L L5",
"3.7L L5",
"5.3L V8"
],
"submodels": [
"Championship Series",
"X",
"Alpha",
"Adventure",
"Luxury",
"Base"
]
},
{
"name": "h3t",
"engines": [
"3.7L L5",
"5.3L V8 FLEX",
"5.3L V8"
],
"submodels": [
"Base",
"Alpha"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
},
{
"name": "h3",
"engines": [
"3.5L L5",
"3.7L L5",
"5.3L V8"
],
"submodels": [
"Championship Series",
"X",
"Alpha",
"Adventure",
"Luxury",
"Base"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
},
{
"name": "h3",
"engines": [
"3.5L L5",
"3.7L L5",
"5.3L V8"
],
"submodels": [
"Championship Series",
"X",
"Alpha",
"Adventure",
"Luxury",
"Base"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
},
{
"name": "h3",
"engines": [
"3.5L L5",
"3.7L L5",
"5.3L V8"
],
"submodels": [
"Championship Series",
"X",
"Alpha",
"Adventure",
"Luxury",
"Base"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
}
]
},
{
"year": "2004",
"models": [
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
}
]
},
{
"year": "2003",
"models": [
{
"name": "h1",
"engines": [
"6.5L V8"
],
"submodels": [
"Base"
]
},
{
"name": "h2",
"engines": [
"6.0L V8",
"6.2L V8 FLEX",
"6.2L V8"
],
"submodels": [
"Special Edition",
"Adventure",
"Base",
"Luxury"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "h1",
"engines": [
"6.5L V8"
],
"submodels": [
"Base"
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,221 +0,0 @@
{
"lamborghini": [
{
"year": "2023",
"models": [
{
"name": "urus",
"engines": [
"4.0L V8"
],
"submodels": [
"Base",
"Performante"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "huracan",
"engines": [
"5.2L V10"
],
"submodels": []
}
]
},
{
"year": "2018",
"models": [
{
"name": "urus",
"engines": [
"4.0L V8"
],
"submodels": [
"Base",
"Performante"
]
}
]
},
{
"year": "2011",
"models": [
{
"name": "gallardo",
"engines": [
"5.0L V10",
"5.2L V10"
],
"submodels": [
"Spyder",
"LP550-2",
"Superleggera",
"Base"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "gallardo",
"engines": [
"5.0L V10",
"5.2L V10"
],
"submodels": [
"Spyder",
"LP550-2",
"Superleggera",
"Base"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "gallardo",
"engines": [
"5.0L V10",
"5.2L V10"
],
"submodels": [
"Spyder",
"LP550-2",
"Superleggera",
"Base"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "gallardo",
"engines": [
"5.0L V10",
"5.2L V10"
],
"submodels": [
"Spyder",
"LP550-2",
"Superleggera",
"Base"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "gallardo",
"engines": [
"5.0L V10",
"5.2L V10"
],
"submodels": [
"Spyder",
"LP550-2",
"Superleggera",
"Base"
]
},
{
"name": "murcielago",
"engines": [
"6.2L V12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2004",
"models": [
{
"name": "murcielago",
"engines": [
"6.2L V12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2003",
"models": [
{
"name": "murcielago",
"engines": [
"6.2L V12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "murcielago",
"engines": [
"6.2L V12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1992",
"models": [
{
"name": "diablo",
"engines": [],
"submodels": []
}
]
},
{
"year": "1988",
"models": [
{
"name": "countach",
"engines": [
"5.2L V12"
],
"submodels": []
}
]
},
{
"year": "1974",
"models": [
{
"name": "urraco",
"engines": [
"2.5L V8"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,829 +0,0 @@
{
"lotus": [
{
"year": "2024",
"models": [
{
"name": "emira",
"engines": [
"2.0L I4",
"3.5L V6"
],
"submodels": [
"First Edition",
"Base"
]
}
]
},
{
"year": "2023",
"models": [
{
"name": "emira",
"engines": [
"2.0L I4",
"3.5L V6"
],
"submodels": [
"First Edition",
"Base"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "evora_gt",
"engines": [
"3.5L V6"
],
"submodels": []
}
]
},
{
"year": "2020",
"models": [
{
"name": "evora_gt",
"engines": [
"3.5L V6"
],
"submodels": []
}
]
},
{
"year": "2017",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
}
]
},
{
"year": "2016",
"models": [
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
}
]
},
{
"year": "2015",
"models": [
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
}
]
},
{
"year": "2014",
"models": [
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
},
{
"name": "exige",
"engines": [
"1.8L I4"
],
"submodels": [
"S",
"Base",
"S 240"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
}
]
},
{
"year": "2011",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
}
]
},
{
"year": "2010",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "evora",
"engines": [
"3.5L V6"
],
"submodels": [
"S SR",
"Sports Racer",
"400",
"S",
"Base"
]
},
{
"name": "exige",
"engines": [
"1.8L I4"
],
"submodels": [
"S",
"Base",
"S 240"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "exige",
"engines": [
"1.8L I4"
],
"submodels": [
"S",
"Base",
"S 240"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "exige",
"engines": [
"1.8L I4"
],
"submodels": [
"S",
"Base",
"S 240"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "europa_s",
"engines": [],
"submodels": []
},
{
"name": "exige",
"engines": [
"1.8L I4"
],
"submodels": [
"S",
"Base",
"S 240"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
},
{
"name": "exige",
"engines": [
"1.8L I4"
],
"submodels": [
"S",
"Base",
"S 240"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "2004",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "2003",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "2001",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "2000",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "1999",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "1998",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "1997",
"models": [
{
"name": "elise",
"engines": [
"1.8L I4"
],
"submodels": [
"R",
"111 S",
"250 Cup",
"111s",
"111R",
"111",
"1.8",
"SC",
"Base"
]
}
]
},
{
"year": "1994",
"models": [
{
"name": "esprit",
"engines": [
"2.2L I4"
],
"submodels": [
"S4",
"Turbo SE",
"Turbo"
]
}
]
},
{
"year": "1991",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
}
]
},
{
"year": "1990",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
},
{
"name": "esprit",
"engines": [
"2.2L I4"
],
"submodels": [
"S4",
"Turbo SE",
"Turbo"
]
}
]
},
{
"year": "1989",
"models": [
{
"name": "esprit",
"engines": [
"2.2L I4"
],
"submodels": [
"S4",
"Turbo SE",
"Turbo"
]
}
]
},
{
"year": "1987",
"models": [
{
"name": "esprit",
"engines": [
"2.2L I4"
],
"submodels": [
"S4",
"Turbo SE",
"Turbo"
]
}
]
},
{
"year": "1972",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
}
]
},
{
"year": "1971",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
}
]
},
{
"year": "1969",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
}
]
},
{
"year": "1967",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
}
]
},
{
"year": "1966",
"models": [
{
"name": "elan",
"engines": [
"1.6L I4"
],
"submodels": [
"S4",
"2",
"2S 130",
"Base"
]
}
]
},
{
"year": "1964",
"models": [
{
"name": "seven",
"engines": [],
"submodels": []
}
]
},
{
"year": "1962",
"models": [
{
"name": "super_seven",
"engines": [
"1.5L I4"
],
"submodels": []
}
]
}
]
}

View File

@@ -1,14 +0,0 @@
{
"lucid": [
{
"year": "2023",
"models": [
{
"name": "air",
"engines": [],
"submodels": []
}
]
}
]
}

View File

@@ -1,692 +0,0 @@
{
"maserati": [
{
"year": "2023",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
}
]
},
{
"year": "2022",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
},
{
"name": "levante",
"engines": [
"3.0L V6"
],
"submodels": [
"S",
"Modena",
"Base"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
}
]
},
{
"year": "2020",
"models": [
{
"name": "levante",
"engines": [
"3.0L V6"
],
"submodels": [
"S",
"Modena",
"Base"
]
}
]
},
{
"year": "2019",
"models": [
{
"name": "levante",
"engines": [
"3.0L V6"
],
"submodels": [
"S",
"Modena",
"Base"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
},
{
"name": "levante",
"engines": [
"3.0L V6"
],
"submodels": [
"S",
"Modena",
"Base"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2017",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
},
{
"name": "levante",
"engines": [
"3.0L V6"
],
"submodels": [
"S",
"Modena",
"Base"
]
}
]
},
{
"year": "2016",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2015",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
},
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
}
]
},
{
"year": "2014",
"models": [
{
"name": "ghibli",
"engines": [
"3.0L V6"
],
"submodels": [
"Modena",
"S",
"Base",
"Modena Q4",
"S Q4"
]
},
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
}
]
},
{
"year": "2011",
"models": [
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
}
]
},
{
"year": "2010",
"models": [
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "granturismo",
"engines": [
"4.2L V8",
"4.7L V8"
],
"submodels": [
"Base",
"S",
"MC",
"1.5 RS CVT Honda SENSING"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "gransport",
"engines": [
"4.2L V8"
],
"submodels": [
"Base"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "gransport",
"engines": [
"4.2L V8"
],
"submodels": [
"Base"
]
},
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "coupe",
"engines": [
"4.2L V8"
],
"submodels": [
"GT"
]
}
]
},
{
"year": "2004",
"models": [
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
},
{
"name": "spyder",
"engines": [
"4.2L V8"
],
"submodels": [
"GT"
]
}
]
},
{
"year": "2003",
"models": [
{
"name": "coupe",
"engines": [
"4.2L V8"
],
"submodels": [
"GT"
]
},
{
"name": "spyder",
"engines": [
"4.2L V8"
],
"submodels": [
"GT"
]
}
]
},
{
"year": "2001",
"models": [
{
"name": "3200gt",
"engines": [],
"submodels": [
"Base"
]
}
]
},
{
"year": "2000",
"models": [
{
"name": "3200gt",
"engines": [],
"submodels": [
"Base"
]
}
]
},
{
"year": "1999",
"models": [
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "1980",
"models": [
{
"name": "quattroporte",
"engines": [
"3.0L V6",
"3.2L V8",
"3.8L V8",
"4.2L V8",
"4.7L V8",
"4.9L V8"
],
"submodels": [
"Executive GT",
"Sport GT S",
"Evoluzione",
"GTS",
"Sport GT",
"S Q4",
"Base"
]
}
]
},
{
"year": "1964",
"models": [
{
"name": "sebring",
"engines": [
"3.5L L6"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,159 +0,0 @@
{
"mclaren": [
{
"year": "2024",
"models": [
{
"name": "artura",
"engines": [
"3.0L V6 PLUG-IN HYBRID EV- (PHEV)"
],
"submodels": []
}
]
},
{
"year": "2022",
"models": [
{
"name": "720s",
"engines": [
"4.0L V8"
],
"submodels": [
"Spider",
"Base"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "gt",
"engines": [
"4.0L V8"
],
"submodels": []
}
]
},
{
"year": "2019",
"models": [
{
"name": "600lt",
"engines": [
"3.8L V8"
],
"submodels": []
},
{
"name": "720s",
"engines": [
"4.0L V8"
],
"submodels": [
"Spider",
"Base"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "570s",
"engines": [
"3.8L V8"
],
"submodels": []
},
{
"name": "720s",
"engines": [
"4.0L V8"
],
"submodels": [
"Spider",
"Base"
]
}
]
},
{
"year": "2017",
"models": [
{
"name": "570gt",
"engines": [
"3.8L V8"
],
"submodels": []
}
]
},
{
"year": "2016",
"models": [
{
"name": "650s",
"engines": [
"3.8L V8"
],
"submodels": []
}
]
},
{
"year": "2015",
"models": [
{
"name": "650s",
"engines": [
"3.8L V8"
],
"submodels": []
}
]
},
{
"year": "2014",
"models": [
{
"name": "mp4-12c",
"engines": [
"3.8L V8"
],
"submodels": []
}
]
},
{
"year": "2013",
"models": [
{
"name": "mp4-12c",
"engines": [
"3.8L V8"
],
"submodels": []
}
]
},
{
"year": "2012",
"models": [
{
"name": "mp4-12c",
"engines": [
"3.8L V8"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,26 +0,0 @@
{
"polestar": [
{
"year": "2022",
"models": [
{
"name": "polestar_2",
"engines": [],
"submodels": []
}
]
},
{
"year": "2021",
"models": [
{
"name": "polestar_1",
"engines": [
"2.0L I4 PLUG-IN HYBRID EV- (PHEV)"
],
"submodels": []
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,14 +0,0 @@
{
"rivian": [
{
"year": "2024",
"models": [
{
"name": "r1t",
"engines": [],
"submodels": []
}
]
}
]
}

View File

@@ -1,171 +0,0 @@
{
"rolls_royce": [
{
"year": "2023",
"models": [
{
"name": "ghost",
"engines": [
"6.7L V12"
],
"submodels": []
}
]
},
{
"year": "2006",
"models": [
{
"name": "phantom",
"engines": [
"6.7L V12"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1997",
"models": [
{
"name": "silver_spur",
"engines": [
"6.8L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1991",
"models": [
{
"name": "silver_spirit",
"engines": [
"6.8L V8"
],
"submodels": []
},
{
"name": "silver_spur",
"engines": [
"6.8L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1990",
"models": [
{
"name": "silver_spirit",
"engines": [
"6.8L V8"
],
"submodels": []
}
]
},
{
"year": "1986",
"models": [
{
"name": "silver_spirit",
"engines": [
"6.8L V8"
],
"submodels": []
}
]
},
{
"year": "1985",
"models": [
{
"name": "silver_spur",
"engines": [
"6.8L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1976",
"models": [
{
"name": "corniche",
"engines": [
"6.8L V8"
],
"submodels": []
}
]
},
{
"year": "1974",
"models": [
{
"name": "silver_shadow",
"engines": [
"6.8L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1972",
"models": [
{
"name": "silver_shadow",
"engines": [
"6.8L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1971",
"models": [
{
"name": "silver_shadow",
"engines": [
"6.8L V8"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "1960",
"models": [
{
"name": "phantom",
"engines": [
"6.7L V12"
],
"submodels": [
"Base"
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,505 +0,0 @@
{
"scion": [
{
"year": "2016",
"models": [
{
"name": "fr-s",
"engines": [
"2.0L H4"
],
"submodels": [
"Series 10",
"Base"
]
},
{
"name": "ia",
"engines": [
"1.5L I4"
],
"submodels": []
},
{
"name": "im",
"engines": [
"1.8L I4"
],
"submodels": []
},
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
}
]
},
{
"year": "2015",
"models": [
{
"name": "fr-s",
"engines": [
"2.0L H4"
],
"submodels": [
"Series 10",
"Base"
]
},
{
"name": "iq",
"engines": [
"1.3L I4"
],
"submodels": [
"Base"
]
},
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2014",
"models": [
{
"name": "fr-s",
"engines": [
"2.0L H4"
],
"submodels": [
"Series 10",
"Base"
]
},
{
"name": "iq",
"engines": [
"1.3L I4"
],
"submodels": [
"Base"
]
},
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "fr-s",
"engines": [
"2.0L H4"
],
"submodels": [
"Series 10",
"Base"
]
},
{
"name": "iq",
"engines": [
"1.3L I4"
],
"submodels": [
"Base"
]
},
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "iq",
"engines": [
"1.3L I4"
],
"submodels": [
"Base"
]
},
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2011",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2010",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xd",
"engines": [
"1.8L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xa",
"engines": [
"1.5L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "tc",
"engines": [
"2.4L I4",
"2.5L I4"
],
"submodels": [
"Spec",
"Base"
]
},
{
"name": "xa",
"engines": [
"1.5L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2004",
"models": [
{
"name": "xa",
"engines": [
"1.5L I4"
],
"submodels": [
"Base"
]
},
{
"name": "xb",
"engines": [
"1.5L I4",
"2.4L I4"
],
"submodels": [
"Base"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "xa",
"engines": [
"1.5L I4"
],
"submodels": [
"Base"
]
}
]
}
]
}

View File

@@ -1,945 +0,0 @@
{
"smart": [
{
"year": "2020",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2017",
"models": [
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2016",
"models": [
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2015",
"models": [
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2014",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2011",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2010",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2009",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2008",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2007",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2006",
"models": [
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
},
{
"name": "roadster",
"engines": [
"0.7L L3"
],
"submodels": [
"Coupe",
"Roadster",
"Roadster Coupe",
"Base"
]
}
]
},
{
"year": "2005",
"models": [
{
"name": "cabrio",
"engines": [
"0.7L L3"
],
"submodels": [
"Passion"
]
},
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
},
{
"name": "roadster",
"engines": [
"0.7L L3"
],
"submodels": [
"Coupe",
"Roadster",
"Roadster Coupe",
"Base"
]
}
]
},
{
"year": "2004",
"models": [
{
"name": "cabrio",
"engines": [
"0.7L L3"
],
"submodels": [
"Passion"
]
},
{
"name": "city-coupe",
"engines": [
"0.7L L3"
],
"submodels": [
"Base",
"Passion"
]
},
{
"name": "forfour",
"engines": [
"0.9L L3",
"1.0L L3",
"1.5L I4"
],
"submodels": [
"Base",
"W 453",
"Prime",
"Prime Premium",
"Passion",
"Pulse"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
},
{
"name": "roadster",
"engines": [
"0.7L L3"
],
"submodels": [
"Coupe",
"Roadster",
"Roadster Coupe",
"Base"
]
}
]
},
{
"year": "2003",
"models": [
{
"name": "city-coupe",
"engines": [
"0.7L L3"
],
"submodels": [
"Base",
"Passion"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
},
{
"name": "roadster",
"engines": [
"0.7L L3"
],
"submodels": [
"Coupe",
"Roadster",
"Roadster Coupe",
"Base"
]
}
]
},
{
"year": "2002",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2001",
"models": [
{
"name": "city_coupe",
"engines": [],
"submodels": [
"Pulse",
"Passion"
]
},
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "2000",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
},
{
"year": "1999",
"models": [
{
"name": "fortwo",
"engines": [
"0.7L L3",
"0.8L L3",
"0.9L L3",
"1.0L L3"
],
"submodels": [
"Turbo",
"Iceshine",
"Bo Concept",
"EQ",
"Electric Drive",
"Brabus Cabrio",
"Black",
"1.5 RS CVT Honda SENSING",
"Proxy",
"GrandStyle",
"CDI",
"Brabus",
"Prime",
"MHD",
"Pulse",
"Passion Cabrio",
"Pure",
"Base",
"Passion"
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,376 +0,0 @@
{
"tesla": [
{
"year": "2024",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "y",
"engines": [],
"submodels": [
"Mayor",
"Performance",
"Long Range"
]
}
]
},
{
"year": "2023",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "y",
"engines": [],
"submodels": [
"Mayor",
"Performance",
"Long Range"
]
}
]
},
{
"year": "2022",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
},
{
"name": "x",
"engines": [],
"submodels": [
"75D",
"Long Range Plus",
"Plaid",
"100D",
"90D"
]
},
{
"name": "y",
"engines": [],
"submodels": [
"Mayor",
"Performance",
"Long Range"
]
}
]
},
{
"year": "2021",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
},
{
"name": "y",
"engines": [],
"submodels": [
"Mayor",
"Performance",
"Long Range"
]
}
]
},
{
"year": "2020",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
},
{
"name": "x",
"engines": [],
"submodels": [
"75D",
"Long Range Plus",
"Plaid",
"100D",
"90D"
]
},
{
"name": "y",
"engines": [],
"submodels": [
"Mayor",
"Performance",
"Long Range"
]
}
]
},
{
"year": "2019",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "x",
"engines": [],
"submodels": [
"75D",
"Long Range Plus",
"Plaid",
"100D",
"90D"
]
}
]
},
{
"year": "2018",
"models": [
{
"name": "3",
"engines": [],
"submodels": [
"Long Range AWD",
"Performance",
"Standard Plus",
"Base",
"Long Range"
]
},
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
},
{
"name": "x",
"engines": [],
"submodels": [
"75D",
"Long Range Plus",
"Plaid",
"100D",
"90D"
]
}
]
},
{
"year": "2017",
"models": [
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
}
]
},
{
"year": "2016",
"models": [
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
},
{
"name": "x",
"engines": [],
"submodels": [
"75D",
"Long Range Plus",
"Plaid",
"100D",
"90D"
]
}
]
},
{
"year": "2015",
"models": [
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
}
]
},
{
"year": "2013",
"models": [
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
}
]
},
{
"year": "2012",
"models": [
{
"name": "s",
"engines": [],
"submodels": [
"70D",
"85D",
"60",
"75D",
"100D",
"Long Range Plus",
"Base",
"Plaid",
"90D"
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,632 +0,0 @@
-- Engines data import
-- Generated by ETL script
BEGIN;
INSERT INTO engines (id, name) VALUES
(1,'0.6L -2'),
(2,'0.6L -2 BATTERY W/RANGE EXT (BEV REX)'),
(3,'0.6L L2'),
(4,'0.7L L3'),
(5,'0.8L L3'),
(6,'0.9L L3'),
(7,'1.0L I4'),
(8,'1.0L L3'),
(9,'1.0L L3 FULL HYBRID EV- (FHEV)'),
(10,'1.2L H4'),
(11,'1.2L I4'),
(12,'1.2L L3'),
(13,'1.2L L3 FULL HYBRID EV- (FHEV)'),
(14,'1.3L H4'),
(15,'1.3L I4'),
(16,'1.3L I4 ELECTRIC'),
(17,'1.3L I4 FULL HYBRID EV- (FHEV)'),
(18,'1.3L L3'),
(19,'1.3L R2'),
(20,'1.4L I4'),
(21,'1.4L I4 FULL HYBRID EV- (FHEV)'),
(22,'1.4L I4 PLUG-IN HYBRID EV- (PHEV)'),
(23,'1.5L'),
(24,'1.5L H4'),
(25,'1.5L I4'),
(26,'1.5L I4 ELECTRIC'),
(27,'1.5L I4 FULL HYBRID EV- (FHEV)'),
(28,'1.5L I4 MILD HYBRID EV- (MHEV)'),
(29,'1.5L I4 PLUG-IN HYBRID EV- (PHEV)'),
(30,'1.5L L3'),
(31,'1.5L L3 PLUG-IN HYBRID EV- (PHEV)'),
(32,'1.6L H4'),
(33,'1.6L I4'),
(34,'1.6L I4 CNG'),
(35,'1.6L I4 FLEX'),
(36,'1.6L I4 FULL HYBRID EV- (FHEV)'),
(37,'1.6L I4 PLUG-IN HYBRID EV- (PHEV)'),
(38,'1.6L L3'),
(39,'1.6L Turbo'),
(40,'1.7L H4'),
(41,'1.7L I4'),
(42,'1.7L I4 CNG'),
(43,'1.8L H4'),
(44,'1.8L I4'),
(45,'1.8L I4 CNG'),
(46,'1.8L I4 ELECTRIC'),
(47,'1.8L I4 FLEX'),
(48,'1.8L I4 FULL HYBRID EV- (FHEV)'),
(49,'1.8L I4 PLUG-IN HYBRID EV- (PHEV)'),
(50,'1.9L H4'),
(51,'1.9L I4'),
(52,'2 0.9L'),
(53,'2-ROTOR ROTARY 1.3L'),
(54,'2.0L H4'),
(55,'2.0L H4 FULL HYBRID EV- (FHEV)'),
(56,'2.0L H4 PLUG-IN HYBRID EV- (PHEV)'),
(57,'2.0L H6'),
(58,'2.0L I4'),
(59,'2.0L I4 BI-FUEL'),
(60,'2.0L I4 FLEX'),
(61,'2.0L I4 FULL HYBRID EV- (FHEV)'),
(62,'2.0L I4 MILD HYBRID EV- (MHEV)'),
(63,'2.0L I4 PLUG-IN HYBRID EV- (PHEV)'),
(64,'2.0L I6'),
(65,'2.0L L5'),
(66,'2.0L Turbo'),
(67,'2.0L V4'),
(68,'2.1L H4'),
(69,'2.1L I4'),
(70,'2.2L H4'),
(71,'2.2L H6'),
(72,'2.2L I4'),
(73,'2.2L I4 BI-FUEL'),
(74,'2.2L I4 CNG'),
(75,'2.2L I4 FLEX'),
(76,'2.2L I5'),
(77,'2.2L I6'),
(78,'2.3L H6'),
(79,'2.3L I4'),
(80,'2.3L I4 FULL HYBRID EV- (FHEV)'),
(81,'2.3L L5'),
(82,'2.3L V6'),
(83,'2.4L H4'),
(84,'2.4L I4'),
(85,'2.4L I4 FLEX'),
(86,'2.4L I4 FULL HYBRID EV- (FHEV)'),
(87,'2.4L I4 MILD HYBRID EV- (MHEV)'),
(88,'2.4L I4 PLUG-IN HYBRID EV- (PHEV)'),
(89,'2.4L L5'),
(90,'2.4L L6'),
(91,'2.4L Turbo'),
(92,'2.5L'),
(93,'2.5L H4'),
(94,'2.5L H6'),
(95,'2.5L I4'),
(96,'2.5L I4 CNG'),
(97,'2.5L I4 FLEX'),
(98,'2.5L I4 FULL HYBRID EV- (FHEV)'),
(99,'2.5L I4 LPG'),
(100,'2.5L I4 MILD HYBRID EV- (MHEV)'),
(101,'2.5L I4 PLUG-IN HYBRID EV- (PHEV)'),
(102,'2.5L I5'),
(103,'2.5L I6'),
(104,'2.5L L5'),
(105,'2.5L V6'),
(106,'2.6L I4'),
(107,'2.7L H6'),
(108,'2.7L I4'),
(109,'2.7L I6'),
(110,'2.7L L5'),
(111,'2.7L V6'),
(112,'2.7L V6 FLEX'),
(113,'2.8L I4'),
(114,'2.8L I6'),
(115,'2.8L L6'),
(116,'2.8L V6'),
(117,'2.9L H6'),
(118,'2.9L I4'),
(119,'2.9L L6'),
(120,'2.9L V6'),
(121,'2.9L V6 MILD HYBRID EV- (MHEV)'),
(122,'2.9L V6 PLUG-IN HYBRID EV- (PHEV)'),
(123,'3 0.9L Turbo'),
(124,'3 1.0L'),
(125,'3 1.2L'),
(126,'3.0L H6'),
(127,'3.0L I4'),
(128,'3.0L I6'),
(129,'3.0L I6 FULL HYBRID EV- (FHEV)'),
(130,'3.0L I6 MILD HYBRID EV- (MHEV)'),
(131,'3.0L I6 PLUG-IN HYBRID EV- (PHEV)'),
(132,'3.0L L6'),
(133,'3.0L L6 MILD HYBRID EV- (MHEV)'),
(134,'3.0L L6 PLUG-IN HYBRID EV- (PHEV)'),
(135,'3.0L V6'),
(136,'3.0L V6 FLEX'),
(137,'3.0L V6 FULL HYBRID EV- (FHEV)'),
(138,'3.0L V6 MILD HYBRID EV- (MHEV)'),
(139,'3.0L V6 PLUG-IN HYBRID EV- (PHEV)'),
(140,'3.0L V8'),
(141,'3.1L V6'),
(142,'3.2L H6'),
(143,'3.2L I6'),
(144,'3.2L L5'),
(145,'3.2L L6'),
(146,'3.2L V6'),
(147,'3.2L V8'),
(148,'3.3L H6'),
(149,'3.3L L6'),
(150,'3.3L V6'),
(151,'3.3L V6 CNG'),
(152,'3.3L V6 FLEX'),
(153,'3.3L V6 FULL HYBRID EV- (FHEV)'),
(154,'3.4L H6'),
(155,'3.4L I4'),
(156,'3.4L V6'),
(157,'3.4L V6 MILD HYBRID EV- (MHEV)'),
(158,'3.4L V8'),
(159,'3.5L I5'),
(160,'3.5L I6'),
(161,'3.5L L5'),
(162,'3.5L Turbo'),
(163,'3.5L V6'),
(164,'3.5L V6 FLEX'),
(165,'3.5L V6 FULL HYBRID EV- (FHEV)'),
(166,'3.5L V6 MILD HYBRID EV- (MHEV)'),
(167,'3.5L V8'),
(168,'3.6L'),
(169,'3.6L H6'),
(170,'3.6L I6'),
(171,'3.6L L6'),
(172,'3.6L V6'),
(173,'3.6L V6 BI-FUEL'),
(174,'3.6L V6 FLEX'),
(175,'3.6L V6 MILD HYBRID EV- (MHEV)'),
(176,'3.6L V8'),
(177,'3.7L H6'),
(178,'3.7L I5'),
(179,'3.7L L5'),
(180,'3.7L L6'),
(181,'3.7L V6'),
(182,'3.7L V6 CNG'),
(183,'3.7L V6 FLEX'),
(184,'3.7L V6 LPG'),
(185,'3.7L V8'),
(186,'3.8L H6'),
(187,'3.8L I6'),
(188,'3.8L V6'),
(189,'3.8L V6 FLEX'),
(190,'3.8L V8'),
(191,'3.9L L6'),
(192,'3.9L V6'),
(193,'3.9L V6 FLEX'),
(194,'3.9L V8'),
(195,'4 1.0L'),
(196,'4 1.2L'),
(197,'4 1.3L'),
(198,'4 1.4L'),
(199,'4 1.4L Turbo'),
(200,'4 1.6L'),
(201,'4 1.6L Turbo'),
(202,'4 1.8L'),
(203,'4 1.8L Turbo'),
(204,'4 2.0L'),
(205,'4 2.4L'),
(206,'4 2.5L'),
(207,'4 IN-LINE 1.8L'),
(208,'4 INLINE 1.5L'),
(209,'4 INLINE 2.4L'),
(210,'4.0L H6'),
(211,'4.0L I4'),
(212,'4.0L L6'),
(213,'4.0L V6'),
(214,'4.0L V6 FLEX'),
(215,'4.0L V8'),
(216,'4.0L V8 MILD HYBRID EV- (MHEV)'),
(217,'4.0L V8 PLUG-IN HYBRID EV- (PHEV)'),
(218,'4.0L W8'),
(219,'4.1L I6'),
(220,'4.1L L6'),
(221,'4.1L V6'),
(222,'4.1L V8'),
(223,'4.2L I6'),
(224,'4.2L L6'),
(225,'4.2L V6'),
(226,'4.2L V8'),
(227,'4.3L V6'),
(228,'4.3L V6 FLEX'),
(229,'4.3L V8'),
(230,'4.4L V8'),
(231,'4.4L V8 MILD HYBRID EV- (MHEV)'),
(232,'4.4L V8 PLUG-IN HYBRID EV- (PHEV)'),
(233,'4.5L L6'),
(234,'4.5L V8'),
(235,'4.6L V8'),
(236,'4.6L V8 CNG'),
(237,'4.6L V8 FLEX'),
(238,'4.7L V8'),
(239,'4.7L V8 FLEX'),
(240,'4.8L V8'),
(241,'4.8L V8 FLEX'),
(242,'4.9L L6'),
(243,'4.9L L6 BI-FUEL'),
(244,'4.9L V10'),
(245,'4.9L V8'),
(246,'5 2.0L Turbo'),
(247,'5 2.4L Turbo'),
(248,'5.0L V10'),
(249,'5.0L V12'),
(250,'5.0L V8'),
(251,'5.0L V8 FLEX'),
(252,'5.1L V8'),
(253,'5.2L V10'),
(254,'5.2L V12'),
(255,'5.2L V8'),
(256,'5.2L V8 CNG'),
(257,'5.3L V8'),
(258,'5.3L V8 FLEX'),
(259,'5.3L V8 FULL HYBRID EV- (FHEV)'),
(260,'5.3L V8 MILD HYBRID EV- (MHEV)'),
(261,'5.4L V12'),
(262,'5.4L V8'),
(263,'5.4L V8 BI-FUEL'),
(264,'5.4L V8 CNG'),
(265,'5.4L V8 FLEX'),
(266,'5.4L V8 LPG'),
(267,'5.5L V8'),
(268,'5.6L V8'),
(269,'5.6L V8 FLEX'),
(270,'5.7L V12'),
(271,'5.7L V8'),
(272,'5.7L V8 CNG'),
(273,'5.7L V8 FLEX'),
(274,'5.7L V8 FULL HYBRID EV- (FHEV)'),
(275,'5.7L V8 MILD HYBRID EV- (MHEV)'),
(276,'5.8L V8'),
(277,'5.9L L6'),
(278,'5.9L V8'),
(279,'6 3.0L'),
(280,'6 3.0L Turbo'),
(281,'6 3.2L'),
(282,'6 3.5L'),
(283,'6 3.7L'),
(284,'6 4.3L'),
(285,'6.0L L6'),
(286,'6.0L V12'),
(287,'6.0L V8'),
(288,'6.0L V8 BI-FUEL'),
(289,'6.0L V8 CNG'),
(290,'6.0L V8 ELECTRIC/FLEX'),
(291,'6.0L V8 FLEX'),
(292,'6.0L V8 FULL HYBRID EV- (FHEV)'),
(293,'6.0L V8 FULL HYBRID EV-FLEX (FHEV)'),
(294,'6.0L W12'),
(295,'6.0L W12 FLEX'),
(296,'6.1L V8'),
(297,'6.2L V12'),
(298,'6.2L V8'),
(299,'6.2L V8 FLEX'),
(300,'6.3L V12'),
(301,'6.3L V8'),
(302,'6.3L W12'),
(303,'6.4L V8'),
(304,'6.5L V8'),
(305,'6.6L'),
(306,'6.6L V12'),
(307,'6.6L V8'),
(308,'6.6L V8 FLEX'),
(309,'6.7L L6'),
(310,'6.7L V12'),
(311,'6.7L V8'),
(312,'6.8L V10'),
(313,'6.8L V10 CNG'),
(314,'6.8L V10 LPG'),
(315,'6.8L V8'),
(316,'6.9L V8'),
(317,'7.0L V8'),
(318,'7.2L V8'),
(319,'7.3L V8'),
(320,'7.4L V8'),
(321,'7.5L V8'),
(322,'7.6L V8'),
(323,'7.7L V8'),
(324,'8 0.6L'),
(325,'8 4.8L'),
(326,'8 5.0L'),
(327,'8 5.3L'),
(328,'8 6.0L'),
(329,'8 6.2L'),
(330,'8 6.6L'),
(331,'8.0L V10'),
(332,'8.1L V8'),
(333,'8.2L V8'),
(334,'8.3L V10'),
(335,'8.4L V10'),
(336,'B4 1.5L'),
(337,'B4 1.6L'),
(338,'B4 2.0L'),
(339,'B4 2.0L Turbo'),
(340,'B4 2.4L'),
(341,'B4 2.4L Turbo'),
(342,'B4 2.5L'),
(343,'B4 2.5L Turbo'),
(344,'B4 3.6L'),
(345,'B6 2.5L'),
(346,'B6 3.0L'),
(347,'B6 3.0L Turbo'),
(348,'B6 3.1L'),
(349,'B6 3.4L'),
(350,'B6 3.6L'),
(351,'B6 3.8L Turbo'),
(352,'B6 4.0L'),
(353,'EV400'),
(354,'Electric'),
(355,'Gas'),
(356,'H4 1.6L'),
(357,'H4 1.6L Turbo'),
(358,'H4 2.0L'),
(359,'H4 2.0L Turbo'),
(360,'H4 2.5L'),
(361,'H4 2.5L Turbo'),
(362,'H6 3.0L'),
(363,'H6 3.0L Turbo'),
(364,'H6 3.4L'),
(365,'H6 3.6L'),
(366,'H6 3.6L Turbo'),
(367,'H6 3.8L'),
(368,'H6 3.8L Turbo'),
(369,'H6 4.0L'),
(370,'H6 4.0L Turbo'),
(371,'Hybrid'),
(372,'L10 6.8L'),
(373,'L2 0.6L'),
(374,'L2 0.9L'),
(375,'L2 0.9L Turbo'),
(376,'L3'),
(377,'L3 0.6L Turbo'),
(378,'L3 0.7L'),
(379,'L3 0.7L Turbo'),
(380,'L3 0.8L'),
(381,'L3 0.8L Turbo'),
(382,'L3 0.9L Turbo'),
(383,'L3 1.0L'),
(384,'L3 1.0L Turbo'),
(385,'L3 1.1L'),
(386,'L3 1.1L Turbo'),
(387,'L3 1.2L'),
(388,'L3 1.2L Turbo'),
(389,'L3 1.4L'),
(390,'L3 1.4L Turbo'),
(391,'L3 1.5L'),
(392,'L3 1.5L Turbo'),
(393,'L3 1.6L Turbo'),
(394,'L4 0.7L'),
(395,'L4 0.9L'),
(396,'L4 1.0L'),
(397,'L4 1.0L Turbo'),
(398,'L4 1.1L'),
(399,'L4 1.1L Turbo'),
(400,'L4 1.2L'),
(401,'L4 1.2L Turbo'),
(402,'L4 1.3L'),
(403,'L4 1.3L Turbo'),
(404,'L4 1.4L'),
(405,'L4 1.4L Turbo'),
(406,'L4 1.5L'),
(407,'L4 1.5L Turbo'),
(408,'L4 1.6L'),
(409,'L4 1.6L Turbo'),
(410,'L4 1.7L'),
(411,'L4 1.7L Turbo'),
(412,'L4 1.8L'),
(413,'L4 1.8L Supercharged'),
(414,'L4 1.8L Turbo'),
(415,'L4 1.9L'),
(416,'L4 1.9L Turbo'),
(417,'L4 2.0L'),
(418,'L4 2.0L Supercharged'),
(419,'L4 2.0L Turbo'),
(420,'L4 2.1L'),
(421,'L4 2.1L Turbo'),
(422,'L4 2.2L'),
(423,'L4 2.2L Turbo'),
(424,'L4 2.3L'),
(425,'L4 2.3L Turbo'),
(426,'L4 2.4L'),
(427,'L4 2.4L Turbo'),
(428,'L4 2.5L'),
(429,'L4 2.5L Supercharged'),
(430,'L4 2.5L Turbo'),
(431,'L4 2.6L'),
(432,'L4 2.7L'),
(433,'L4 2.7L Turbo'),
(434,'L4 2.8L'),
(435,'L4 2.8L Turbo'),
(436,'L4 2.9L'),
(437,'L4 20.0L Turbo'),
(438,'L4 3.0L'),
(439,'L4 3.0L Turbo'),
(440,'L4 3.2L'),
(441,'L4 3.2L Turbo'),
(442,'L4 3.5L'),
(443,'L4 3.6L'),
(444,'L4 3.9L'),
(445,'L4 4.2L Turbo'),
(446,'L5 0.5L Turbo'),
(447,'L5 1.4L Turbo'),
(448,'L5 2.0L'),
(449,'L5 2.0L Turbo'),
(450,'L5 2.3L'),
(451,'L5 2.3L Turbo'),
(452,'L5 2.4L'),
(453,'L5 2.4L Turbo'),
(454,'L5 2.5L'),
(455,'L5 2.5L Turbo'),
(456,'L5 2.7L'),
(457,'L5 3.1L'),
(458,'L5 3.2L Turbo'),
(459,'L5 3.5L'),
(460,'L5 3.6L'),
(461,'L5 3.7L'),
(462,'L6 1.9L'),
(463,'L6 2.0L'),
(464,'L6 2.2L'),
(465,'L6 2.5L'),
(466,'L6 2.6L Turbo'),
(467,'L6 2.8L'),
(468,'L6 2.9L'),
(469,'L6 2.9L Turbo'),
(470,'L6 3.0L'),
(471,'L6 3.0L Turbo'),
(472,'L6 3.1L'),
(473,'L6 3.2L'),
(474,'L6 3.6L'),
(475,'L6 3.7L Turbo'),
(476,'L6 4.0L'),
(477,'L6 4.2L'),
(478,'L6 4.2L Turbo'),
(479,'L6 4.8L Turbo'),
(480,'L6 5.3L'),
(481,'L6 6.7L'),
(482,'L7 3.0L Turbo'),
(483,'R2 1.3L'),
(484,'V* 4.2L Turbo'),
(485,'V10 4.9L'),
(486,'V10 4.9L Turbo'),
(487,'V10 5.0L'),
(488,'V10 5.2L'),
(489,'V10 8.0L'),
(490,'V10 8.3L'),
(491,'V10 8.4L'),
(492,'V12 5.2L Turbo'),
(493,'V12 5.7L'),
(494,'V12 5.9L'),
(495,'V12 5.9L Turbo'),
(496,'V12 6.0L'),
(497,'V12 6.2L'),
(498,'V12 6.3L'),
(499,'V12 6.3L Turbo'),
(500,'V12 6.5L'),
(501,'V4 2.5L Turbo'),
(502,'V5 2.3L'),
(503,'V5 2.5L'),
(504,'V6'),
(505,'V6 0.6L'),
(506,'V6 1.4L'),
(507,'V6 2.0L'),
(508,'V6 2.0L Turbo'),
(509,'V6 2.1L'),
(510,'V6 2.3L'),
(511,'V6 2.3L Turbo'),
(512,'V6 2.4L'),
(513,'V6 2.5L'),
(514,'V6 2.5L Turbo'),
(515,'V6 2.6L'),
(516,'V6 2.7L'),
(517,'V6 2.7L Turbo'),
(518,'V6 2.8L'),
(519,'V6 2.8L Turbo'),
(520,'V6 2.9L'),
(521,'V6 2.9L Turbo'),
(522,'V6 3.0L'),
(523,'V6 3.0L Supercharged'),
(524,'V6 3.0L Turbo'),
(525,'V6 3.1L'),
(526,'V6 3.2L'),
(527,'V6 3.2L Supercharged'),
(528,'V6 3.2L Turbo'),
(529,'V6 3.3L'),
(530,'V6 3.3L Turbo'),
(531,'V6 3.4L'),
(532,'V6 3.4L Turbo'),
(533,'V6 3.5L'),
(534,'V6 3.5L Supercharged'),
(535,'V6 3.5L Turbo'),
(536,'V6 3.6L'),
(537,'V6 3.6L Turbo'),
(538,'V6 3.7L'),
(539,'V6 3.7L Turbo'),
(540,'V6 3.8L'),
(541,'V6 3.8L Supercharged'),
(542,'V6 3.8L Turbo'),
(543,'V6 3.9L'),
(544,'V6 4.0L'),
(545,'V6 4.0L Turbo'),
(546,'V6 4.2L'),
(547,'V6 4.3L'),
(548,'V6 4.6L'),
(549,'V6 4.9L'),
(550,'V6 5.7L'),
(551,'V6 6.0L'),
(552,'V8'),
(553,'V8 0.0L'),
(554,'V8 0.0L Supercharged'),
(555,'V8 3.2L'),
(556,'V8 3.5L'),
(557,'V8 3.6L'),
(558,'V8 3.7L'),
(559,'V8 3.8L'),
(560,'V8 3.8L Turbo'),
(561,'V8 3.9L'),
(562,'V8 3.9L Turbo'),
(563,'V8 4.0L'),
(564,'V8 4.0L Turbo'),
(565,'V8 4.1L'),
(566,'V8 4.1L Turbo'),
(567,'V8 4.2L'),
(568,'V8 4.2L Turbo'),
(569,'V8 4.3L'),
(570,'V8 4.4L'),
(571,'V8 4.4L Supercharged'),
(572,'V8 4.4L Turbo'),
(573,'V8 4.5L'),
(574,'V8 4.5L Turbo'),
(575,'V8 4.6L'),
(576,'V8 4.6L Turbo'),
(577,'V8 4.7L'),
(578,'V8 4.8L'),
(579,'V8 4.8L Turbo'),
(580,'V8 4.9L'),
(581,'V8 5.0L'),
(582,'V8 5.0L Supercharged'),
(583,'V8 5.0L Turbo'),
(584,'V8 5.2L'),
(585,'V8 5.2L Supercharged'),
(586,'V8 5.2L Turbo'),
(587,'V8 5.3L'),
(588,'V8 5.3L Turbo'),
(589,'V8 5.4L'),
(590,'V8 5.4L Supercharged'),
(591,'V8 5.5L'),
(592,'V8 5.5L Turbo'),
(593,'V8 5.6L'),
(594,'V8 5.6L Turbo'),
(595,'V8 5.7L'),
(596,'V8 5.7L Turbo'),
(597,'V8 5.8L'),
(598,'V8 5.8L Supercharged'),
(599,'V8 5.9L'),
(600,'V8 6.0L'),
(601,'V8 6.0L Turbo'),
(602,'V8 6.1L'),
(603,'V8 6.2L'),
(604,'V8 6.2L Supercharged'),
(605,'V8 6.2L Turbo'),
(606,'V8 6.4L'),
(607,'V8 6.4L Turbo'),
(608,'V8 6.5L'),
(609,'V8 6.6L'),
(610,'V8 6.6L Turbo'),
(611,'V8 6.8L Turbo'),
(612,'V8 7.0L'),
(613,'V8 7.3L Turbo'),
(614,'V8 Turbo'),
(615,'VR5 2.3L'),
(616,'VR6 2.8L'),
(617,'VR6 3.6L'),
(618,'W12 6.0L'),
(619,'W12 6.0L Turbo'),
(620,'W12 6.3L'),
(621,'W12 6.3L Turbo'),
(622,'W8 4.0L');
SELECT setval('engines_id_seq', 622);
COMMIT;

View File

@@ -1,29 +0,0 @@
-- Transmissions data import
-- Generated by ETL script
BEGIN;
INSERT INTO transmissions (id, type) VALUES
(1,'1-Speed Automatic'),
(2,'10-Speed Automatic'),
(3,'2-Speed Automatic'),
(4,'3-Speed Automatic'),
(5,'4-Speed Automatic'),
(6,'4-Speed Manual'),
(7,'5-Speed Automatic'),
(8,'5-Speed Manual'),
(9,'6-Speed Automatic'),
(10,'6-Speed Dual-Clutch'),
(11,'6-Speed Manual'),
(12,'7-Speed Automatic'),
(13,'7-Speed Dual-Clutch'),
(14,'7-Speed Manual'),
(15,'8-Speed Automatic'),
(16,'9-Speed Automatic'),
(17,'Automatic'),
(18,'CVT'),
(19,'Manual');
SELECT setval('transmissions_id_seq', 19);
COMMIT;

File diff suppressed because it is too large Load Diff

View File

@@ -1,11 +0,0 @@
============================================================
ETL Statistics
============================================================
Min Year: 2,000
Max Year: 2,026
Vehicle Records: 1,675,335
Engines: 622
Transmissions: 19
Makes: 54
Models: 1,881

View File

@@ -1,112 +0,0 @@
#!/usr/bin/env python3
"""
Post-import QA validation for vehicle dropdown data.
Runs basic duplicate and range checks against the motovaultpro Postgres container.
"""
import os
import subprocess
import sys
def run_psql(query: str) -> str:
cmd = [
"docker",
"exec",
"mvp-postgres",
"psql",
"-U",
"postgres",
"-d",
"motovaultpro",
"-At",
"-c",
query,
]
return subprocess.check_output(cmd, text=True)
def check_container():
try:
subprocess.check_output(["docker", "ps"], text=True)
except Exception:
print("❌ Docker not available.")
sys.exit(1)
try:
containers = subprocess.check_output(
["docker", "ps", "--filter", "name=mvp-postgres", "--format", "{{.Names}}"],
text=True,
).strip()
if not containers:
print("❌ mvp-postgres container not running.")
sys.exit(1)
except Exception as exc:
print(f"❌ Failed to check containers: {exc}")
sys.exit(1)
def main():
check_container()
print("🔍 Running QA checks...\n")
queries = {
"engine_duplicate_names": """
SELECT COUNT(*) FROM (
SELECT LOWER(name) as n, COUNT(*) c
FROM engines
GROUP BY 1 HAVING COUNT(*) > 1
) t;
""",
"transmission_duplicate_types": """
SELECT COUNT(*) FROM (
SELECT LOWER(type) as t, COUNT(*) c
FROM transmissions
GROUP BY 1 HAVING COUNT(*) > 1
) t;
""",
"vehicle_option_duplicates": """
SELECT COUNT(*) FROM (
SELECT year, make, model, trim, engine_id, transmission_id, COUNT(*) c
FROM vehicle_options
GROUP BY 1,2,3,4,5,6 HAVING COUNT(*) > 1
) t;
""",
"year_range": """
SELECT MIN(year) || ' - ' || MAX(year) FROM vehicle_options;
""",
"counts": """
SELECT
(SELECT COUNT(*) FROM engines) AS engines,
(SELECT COUNT(*) FROM transmissions) AS transmissions,
(SELECT COUNT(*) FROM vehicle_options) AS vehicle_options;
""",
}
results = {}
for key, query in queries.items():
try:
results[key] = run_psql(query).strip()
except subprocess.CalledProcessError as exc:
print(f"❌ Query failed ({key}): {exc}")
sys.exit(1)
print(f"Engine duplicate names: {results['engine_duplicate_names']}")
print(f"Transmission duplicate types: {results['transmission_duplicate_types']}")
print(f"Vehicle option duplicates: {results['vehicle_option_duplicates']}")
print(f"Year range: {results['year_range']}")
print(f"Counts (engines, transmissions, vehicle_options): {results['counts']}")
if (
results["engine_duplicate_names"] == "0"
and results["transmission_duplicate_types"] == "0"
and results["vehicle_option_duplicates"] == "0"
):
print("\n✅ QA checks passed.")
else:
print("\n❌ QA checks found issues.")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""
Generate SQL import files from a VehAPI snapshot SQLite database.
Reads observed compatibility pairs from the snapshot (trim-filtered engine<->transmission pairs)
and produces:
- output/01_engines.sql
- output/02_transmissions.sql
- output/03_vehicle_options.sql
No legacy JSON or network calls are used. The snapshot path is provided via CLI flag.
"""
import argparse
import os
import sqlite3
from pathlib import Path
from typing import Dict, Iterable, List, Sequence
BATCH_SIZE = 1000
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Generate SQL files from a VehAPI snapshot (SQLite).",
)
parser.add_argument(
"--snapshot-path",
type=Path,
default=os.environ.get("SNAPSHOT_PATH"),
help="Path to snapshots/<date>/snapshot.sqlite produced by vehapi_fetch_snapshot.py (or env SNAPSHOT_PATH)",
)
parser.add_argument(
"--output-dir",
type=Path,
default=Path("output"),
help="Directory to write SQL output files (default: output)",
)
return parser.parse_args()
def load_pairs(snapshot_path: Path) -> List[sqlite3.Row]:
if not snapshot_path.exists():
raise FileNotFoundError(f"Snapshot not found: {snapshot_path}")
conn = sqlite3.connect(snapshot_path)
conn.row_factory = sqlite3.Row
try:
cursor = conn.execute(
"""
SELECT
year,
make,
model,
trim,
engine_display,
engine_canon,
engine_bucket,
trans_display,
trans_canon,
trans_bucket
FROM pairs
ORDER BY year, make, model, trim, engine_canon, trans_canon
"""
)
rows = cursor.fetchall()
except sqlite3.Error as exc:
raise RuntimeError(f"Failed to read pairs from snapshot: {exc}") from exc
finally:
conn.close()
if not rows:
raise ValueError("Snapshot contains no rows in pairs table.")
return rows
def choose_engine_label(engine_display: str, engine_bucket: str, engine_canon: str) -> str:
"""
Use VehAPI display string when present, otherwise fall back to the bucket label,
and finally to the canonical key to avoid empty names.
"""
if engine_display:
return engine_display
if engine_bucket:
return engine_bucket
return engine_canon
def choose_trans_label(trans_display: str, trans_bucket: str, trans_canon: str) -> str:
if trans_display:
return trans_display
if trans_bucket:
return trans_bucket
return trans_canon
def build_engine_dimension(rows: Sequence[sqlite3.Row]) -> Dict[str, Dict]:
engines: Dict[str, Dict] = {}
for row in rows:
canon = row["engine_canon"]
if canon is None or canon == "":
raise ValueError(f"Missing engine_canon for row: {dict(row)}")
if canon in engines:
continue
engines[canon] = {
"id": len(engines) + 1,
"name": choose_engine_label(row["engine_display"], row["engine_bucket"], canon),
"fuel_type": row["engine_bucket"] or None,
}
return engines
def build_transmission_dimension(rows: Sequence[sqlite3.Row]) -> Dict[str, Dict]:
transmissions: Dict[str, Dict] = {}
for row in rows:
canon = row["trans_canon"]
if canon is None or canon == "":
raise ValueError(f"Missing trans_canon for row: {dict(row)}")
if canon in transmissions:
continue
transmissions[canon] = {
"id": len(transmissions) + 1,
"type": choose_trans_label(row["trans_display"], row["trans_bucket"], canon),
}
return transmissions
def build_vehicle_options(
rows: Sequence[sqlite3.Row],
engine_map: Dict[str, Dict],
trans_map: Dict[str, Dict],
) -> List[Dict]:
options: List[Dict] = []
for row in rows:
engine_canon = row["engine_canon"]
trans_canon = row["trans_canon"]
options.append(
{
"year": int(row["year"]),
"make": row["make"],
"model": row["model"],
"trim": row["trim"],
"engine_id": engine_map[engine_canon]["id"],
"transmission_id": trans_map[trans_canon]["id"],
}
)
return options
def sql_value(value):
if value is None:
return "NULL"
if isinstance(value, str):
return "'" + value.replace("'", "''") + "'"
return str(value)
def chunked(seq: Iterable[Dict], size: int) -> Iterable[List[Dict]]:
chunk: List[Dict] = []
for item in seq:
chunk.append(item)
if len(chunk) >= size:
yield chunk
chunk = []
if chunk:
yield chunk
def write_insert_file(
path: Path,
table: str,
columns: Sequence[str],
rows: Sequence[Dict],
):
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
f.write(f"-- Auto-generated by etl_generate_sql.py\n")
if not rows:
f.write(f"-- No rows for {table}\n")
return
for batch in chunked(rows, BATCH_SIZE):
values_sql = ",\n".join(
"(" + ",".join(sql_value(row[col]) for col in columns) + ")"
for row in batch
)
f.write(f"INSERT INTO {table} ({', '.join(columns)}) VALUES\n{values_sql};\n\n")
def main():
args = parse_args()
snapshot_path: Path = args.snapshot_path
output_dir: Path = args.output_dir
if snapshot_path is None:
raise SystemExit("Snapshot path is required. Pass --snapshot-path or set SNAPSHOT_PATH.")
print(f"Reading snapshot: {snapshot_path}")
rows = load_pairs(snapshot_path)
years = sorted({int(row["year"]) for row in rows})
print(f" Loaded {len(rows):,} observed engine<->transmission pairs across {len(years)} years")
engines = build_engine_dimension(rows)
transmissions = build_transmission_dimension(rows)
vehicle_options = build_vehicle_options(rows, engines, transmissions)
print(f"Engines: {len(engines):,}")
print(f"Transmissions: {len(transmissions):,}")
print(f"Vehicle options (observed pairs): {len(vehicle_options):,}")
write_insert_file(
output_dir / "01_engines.sql",
"engines",
["id", "name", "fuel_type"],
engines.values(),
)
write_insert_file(
output_dir / "02_transmissions.sql",
"transmissions",
["id", "type"],
transmissions.values(),
)
write_insert_file(
output_dir / "03_vehicle_options.sql",
"vehicle_options",
["year", "make", "model", "trim", "engine_id", "transmission_id"],
vehicle_options,
)
print("\nSQL files generated:")
print(f" - {output_dir / '01_engines.sql'}")
print(f" - {output_dir / '02_transmissions.sql'}")
print(f" - {output_dir / '03_vehicle_options.sql'}")
print(f"\nYear coverage: {years[0]}-{years[-1]}")
if __name__ == "__main__":
main()

View File

@@ -1,37 +1,37 @@
#!/bin/bash #!/bin/bash
# Import generated SQL files into PostgreSQL database # Offline import of generated SQL files into PostgreSQL (no network).
# Run this after etl_generate_sql.py has created the SQL files
set -e set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
echo "==========================================" echo "=========================================="
echo "📥 Automotive Database Import" echo "📥 Automotive Database Import (offline)"
echo "==========================================" echo "=========================================="
echo "" echo ""
# Check if Docker container is running require_file() {
if [ ! -f "$1" ]; then
echo "❌ Missing required file: $1"
exit 1
fi
}
if ! docker ps --filter "name=mvp-postgres" --format "{{.Names}}" | grep -q "mvp-postgres"; then if ! docker ps --filter "name=mvp-postgres" --format "{{.Names}}" | grep -q "mvp-postgres"; then
echo "❌ Error: mvp-postgres container is not running" echo "❌ Error: mvp-postgres container is not running"
exit 1 exit 1
fi fi
echo "✓ Docker container mvp-postgres is running" require_file "output/01_engines.sql"
echo "" require_file "output/02_transmissions.sql"
require_file "output/03_vehicle_options.sql"
# Check if output directory exists
if [ ! -d "output" ]; then
echo "❌ Error: output directory not found"
echo "Please run etl_generate_sql.py first to generate SQL files"
exit 1
fi
# Run schema migration first
echo "📋 Step 1: Running database schema migration..." echo "📋 Step 1: Running database schema migration..."
docker exec -i mvp-postgres psql -U postgres -d motovaultpro < migrations/001_create_vehicle_database.sql docker exec -i mvp-postgres psql -U postgres -d motovaultpro < migrations/001_create_vehicle_database.sql
echo "✓ Schema migration completed" echo "✓ Schema migration completed"
echo "" echo ""
# Truncate tables for a clean rerun
echo "🧹 Step 2: Truncating existing data..." echo "🧹 Step 2: Truncating existing data..."
docker exec -i mvp-postgres psql -U postgres -d motovaultpro <<'EOF' docker exec -i mvp-postgres psql -U postgres -d motovaultpro <<'EOF'
TRUNCATE TABLE vehicle_options RESTART IDENTITY CASCADE; TRUNCATE TABLE vehicle_options RESTART IDENTITY CASCADE;
@@ -41,38 +41,31 @@ EOF
echo "✓ Tables truncated" echo "✓ Tables truncated"
echo "" echo ""
# Import engines
echo "📥 Step 3: Importing engines..." echo "📥 Step 3: Importing engines..."
docker exec -i mvp-postgres psql -U postgres -d motovaultpro < output/01_engines.sql docker exec -i mvp-postgres psql -U postgres -d motovaultpro < output/01_engines.sql
echo "✓ Engines imported" echo "✓ Engines imported"
echo "" echo ""
# Import transmissions
echo "📥 Step 4: Importing transmissions..." echo "📥 Step 4: Importing transmissions..."
docker exec -i mvp-postgres psql -U postgres -d motovaultpro < output/02_transmissions.sql docker exec -i mvp-postgres psql -U postgres -d motovaultpro < output/02_transmissions.sql
echo "✓ Transmissions imported" echo "✓ Transmissions imported"
echo "" echo ""
# Import vehicle options echo "📥 Step 5: Importing vehicle options (observed pairs only)..."
echo "📥 Step 5: Importing vehicle options (this may take a minute)..."
docker exec -i mvp-postgres psql -U postgres -d motovaultpro < output/03_vehicle_options.sql docker exec -i mvp-postgres psql -U postgres -d motovaultpro < output/03_vehicle_options.sql
echo "✓ Vehicle options imported" echo "✓ Vehicle options imported"
echo "" echo ""
# Verify data
echo "==========================================" echo "=========================================="
echo "✅ Import completed successfully!" echo "✅ Import completed"
echo "==========================================" echo "=========================================="
echo "" echo ""
echo "🔍 Database verification:" echo "🔍 Database verification:"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT COUNT(*) as engine_count FROM engines;" docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT COUNT(*) as engines FROM engines;"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT COUNT(*) as transmission_count FROM transmissions;" docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT COUNT(*) as transmissions FROM transmissions;"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT COUNT(*) as vehicle_count FROM vehicle_options;" docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT COUNT(*) as vehicle_options FROM vehicle_options;"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT MIN(year) as min_year, MAX(year) as max_year FROM vehicle_options;" docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT MIN(year) as min_year, MAX(year) as max_year FROM vehicle_options;"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT DISTINCT year FROM vehicle_options ORDER BY year LIMIT 5;"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT DISTINCT year FROM vehicle_options ORDER BY year DESC LIMIT 5;"
echo "" echo ""
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT * FROM available_years;" echo "✓ Database ready for dropdown use."
echo ""
echo "📊 Sample query - 2024 makes:"
docker exec mvp-postgres psql -U postgres -d motovaultpro -c "SELECT * FROM get_makes_for_year(2024) LIMIT 10;"
echo ""
echo "✓ Database is ready for use!"

View File

@@ -72,6 +72,8 @@ CREATE INDEX idx_vehicle_trim ON vehicle_options(trim);
CREATE INDEX idx_vehicle_year_make ON vehicle_options(year, make); CREATE INDEX idx_vehicle_year_make ON vehicle_options(year, make);
CREATE INDEX idx_vehicle_year_make_model ON vehicle_options(year, make, model); CREATE INDEX idx_vehicle_year_make_model ON vehicle_options(year, make, model);
CREATE INDEX idx_vehicle_year_make_model_trim ON vehicle_options(year, make, model, trim); CREATE INDEX idx_vehicle_year_make_model_trim ON vehicle_options(year, make, model, trim);
CREATE INDEX idx_vehicle_year_make_model_trim_engine ON vehicle_options(year, make, model, trim, engine_id);
CREATE INDEX idx_vehicle_year_make_model_trim_trans ON vehicle_options(year, make, model, trim, transmission_id);
-- Views for dropdown queries -- Views for dropdown queries
@@ -189,6 +191,91 @@ BEGIN
END; END;
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
-- Helper functions for trim-level options and pair-safe filtering
CREATE OR REPLACE FUNCTION get_transmissions_for_vehicle(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR)
RETURNS TABLE(
transmission_id INTEGER,
transmission_type VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
t.id,
t.type
FROM vehicle_options vo
JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
ORDER BY t.type ASC;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION get_engines_for_vehicle(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR)
RETURNS TABLE(
engine_id INTEGER,
engine_name VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
e.id,
e.name
FROM vehicle_options vo
JOIN engines e ON vo.engine_id = e.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
ORDER BY e.name ASC;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION get_transmissions_for_vehicle_engine(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR, p_engine_name VARCHAR)
RETURNS TABLE(
transmission_id INTEGER,
transmission_type VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
t.id,
t.type
FROM vehicle_options vo
JOIN engines e ON vo.engine_id = e.id
JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
AND e.name = p_engine_name
ORDER BY t.type ASC;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION get_engines_for_vehicle_trans(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR, p_trans_type VARCHAR)
RETURNS TABLE(
engine_id INTEGER,
engine_name VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
e.id,
e.name
FROM vehicle_options vo
JOIN engines e ON vo.engine_id = e.id
JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
AND t.type = p_trans_type
ORDER BY e.name ASC;
END;
$$ LANGUAGE plpgsql;
COMMENT ON TABLE vehicle_options IS 'Denormalized table optimized for cascading dropdown queries'; COMMENT ON TABLE vehicle_options IS 'Denormalized table optimized for cascading dropdown queries';
COMMENT ON TABLE engines IS 'Engine specifications with detailed technical data'; COMMENT ON TABLE engines IS 'Engine specifications with detailed technical data';
COMMENT ON TABLE transmissions IS 'Transmission specifications'; COMMENT ON TABLE transmissions IS 'Transmission specifications';

View File

@@ -0,0 +1,22 @@
-- Auto-generated by etl_generate_sql.py
INSERT INTO engines (id, name, fuel_type) VALUES
(1,'Gas','Gas'),
(2,'2.0L 150 hp I4','Gas'),
(3,'2.4L 201 hp I4','Gas'),
(4,'3.5L 290 hp V6','Gas'),
(5,'3.5L 273 hp V6','Gas'),
(6,'3.5L 310 hp V6','Gas'),
(7,'2.4L 206 hp I4','Gas'),
(8,'2.0L 220 hp I4','Gas'),
(9,'1.8L 170 hp I4','Gas'),
(10,'Diesel','Diesel'),
(11,'2.0L 150 hp I4 Diesel','Diesel'),
(12,'2.0L 220 hp I4 Flex Fuel Vehicle','Gas'),
(13,'3.0L 310 hp V6','Gas'),
(14,'3.0L 240 hp V6 Diesel','Diesel'),
(15,'4.0L 435 hp V8','Diesel'),
(16,'3.0L 333 hp V6','Gas'),
(17,'6.3L 500 hp W12','Gas'),
(18,'2.0L 200 hp I4','Gas'),
(19,'3.0L 272 hp V6','Gas');

View File

@@ -0,0 +1,13 @@
-- Auto-generated by etl_generate_sql.py
INSERT INTO transmissions (id, type) VALUES
(1,'Automatic'),
(2,'Manual'),
(3,'5-Speed Automatic'),
(4,'6-Speed Manual'),
(5,'6-Speed Automatic'),
(6,'8-Speed Dual Clutch'),
(7,'9-Speed Automatic'),
(8,'6-Speed Dual Clutch'),
(9,'8-Speed Automatic'),
(10,'Continuously Variable Transmission');

View File

@@ -0,0 +1,281 @@
-- Auto-generated by etl_generate_sql.py
INSERT INTO vehicle_options (year, make, model, trim, engine_id, transmission_id) VALUES
(2015,'Acura','ILX','2.0L',1,1),
(2015,'Acura','ILX','2.0L',1,2),
(2015,'Acura','ILX','2.0L FWD',2,3),
(2015,'Acura','ILX','2.0L FWD with Premium Package',2,3),
(2015,'Acura','ILX','2.0L FWD with Technology Package',2,3),
(2015,'Acura','ILX','2.0L Technology',1,1),
(2015,'Acura','ILX','2.0L Technology',1,2),
(2015,'Acura','ILX','2.0L w/Premium Package',1,1),
(2015,'Acura','ILX','2.0L w/Premium Package',1,2),
(2015,'Acura','ILX','2.4L FWD with Premium Package',2,3),
(2015,'Acura','ILX','2.4L FWD with Premium Package',3,4),
(2015,'Acura','ILX','2.4L w/Premium Package',1,1),
(2015,'Acura','ILX','2.4L w/Premium Package',1,2),
(2015,'Acura','ILX','FWD with Dynamic Package',2,3),
(2015,'Acura','MDX','3.5L',1,1),
(2015,'Acura','MDX','3.5L',1,2),
(2015,'Acura','MDX','3.5L Advance Pkg w/Entertainment Pkg',1,1),
(2015,'Acura','MDX','3.5L Advance Pkg w/Entertainment Pkg',1,2),
(2015,'Acura','MDX','3.5L Technology Package',1,1),
(2015,'Acura','MDX','3.5L Technology Package',1,2),
(2015,'Acura','MDX','3.5L Technology Pkg/w Entertainment Pkg',1,1),
(2015,'Acura','MDX','3.5L Technology Pkg/w Entertainment Pkg',1,2),
(2015,'Acura','MDX','3.5L w/Technology & Entertainment Pkgs',1,1),
(2015,'Acura','MDX','3.5L w/Technology & Entertainment Pkgs',1,2),
(2015,'Acura','MDX','FWD',4,5),
(2015,'Acura','MDX','FWD with Advance and Entertainment Package',4,5),
(2015,'Acura','MDX','FWD with Technology Package',4,5),
(2015,'Acura','MDX','FWD with Technology and Entertainment Package',4,5),
(2015,'Acura','MDX','SH-AWD',4,5),
(2015,'Acura','MDX','SH-AWD with Advance and Entertainment Package',4,5),
(2015,'Acura','MDX','SH-AWD with Elite Package',4,5),
(2015,'Acura','MDX','SH-AWD with Navigation',4,5),
(2015,'Acura','MDX','SH-AWD with Technology Package',4,5),
(2015,'Acura','MDX','SH-AWD with Technology and Entertainment Package',4,5),
(2015,'Acura','RDX','AWD',5,5),
(2015,'Acura','RDX','AWD with Technology Package',5,5),
(2015,'Acura','RDX','Base',1,1),
(2015,'Acura','RDX','Base',1,2),
(2015,'Acura','RDX','FWD',5,5),
(2015,'Acura','RDX','FWD with Technology Package',5,5),
(2015,'Acura','RDX','Technology Package',1,1),
(2015,'Acura','RDX','Technology Package',1,2),
(2015,'Acura','RLX','Advance Package',1,1),
(2015,'Acura','RLX','Advance Package',1,2),
(2015,'Acura','RLX','Base',1,1),
(2015,'Acura','RLX','Base',1,2),
(2015,'Acura','RLX','FWD',6,5),
(2015,'Acura','RLX','FWD',1,1),
(2015,'Acura','RLX','FWD',1,2),
(2015,'Acura','RLX','FWD with Advance Package',6,5),
(2015,'Acura','RLX','FWD with Elite Package',6,5),
(2015,'Acura','RLX','FWD with Krell Audio Package',6,5),
(2015,'Acura','RLX','FWD with Navigation',6,5),
(2015,'Acura','RLX','FWD with Technology Package',6,5),
(2015,'Acura','RLX','Navigation',1,1),
(2015,'Acura','RLX','Navigation',1,2),
(2015,'Acura','RLX','Technology Package',1,1),
(2015,'Acura','RLX','Technology Package',1,2),
(2015,'Acura','RLX Hybrid Sport','SH-AWD',1,1),
(2015,'Acura','RLX Hybrid Sport','SH-AWD',1,2),
(2015,'Acura','TLX','Base',1,1),
(2015,'Acura','TLX','Base',1,2),
(2015,'Acura','TLX','FWD',7,6),
(2015,'Acura','TLX','FWD with Technology Package',7,6),
(2015,'Acura','TLX','SH-AWD with Elite Package',4,7),
(2015,'Acura','TLX','Tech',1,1),
(2015,'Acura','TLX','Tech',1,2),
(2015,'Acura','TLX','V6',1,1),
(2015,'Acura','TLX','V6',1,2),
(2015,'Acura','TLX','V6 Advance',1,1),
(2015,'Acura','TLX','V6 Advance',1,2),
(2015,'Acura','TLX','V6 FWD',4,7),
(2015,'Acura','TLX','V6 FWD with Advance Package',4,7),
(2015,'Acura','TLX','V6 FWD with Technology Package',4,7),
(2015,'Acura','TLX','V6 SH-AWD',4,7),
(2015,'Acura','TLX','V6 SH-AWD with Advance Package',4,7),
(2015,'Acura','TLX','V6 SH-AWD with Technology Package',4,7),
(2015,'Acura','TLX','V6 Tech',1,1),
(2015,'Acura','TLX','V6 Tech',1,2),
(2015,'Acura','TLX','V6 with Elite Package',4,7),
(2015,'Audi','A3','1.8T Komfort Sedan FWD',8,8),
(2015,'Audi','A3','1.8T Premium',1,1),
(2015,'Audi','A3','1.8T Premium',1,2),
(2015,'Audi','A3','1.8T Premium Cabriolet FWD',9,8),
(2015,'Audi','A3','1.8T Premium Plus',1,1),
(2015,'Audi','A3','1.8T Premium Plus',1,2),
(2015,'Audi','A3','1.8T Premium Plus Cabriolet FWD',9,8),
(2015,'Audi','A3','1.8T Premium Plus Sedan FWD',9,8),
(2015,'Audi','A3','1.8T Premium Sedan FWD',9,8),
(2015,'Audi','A3','1.8T Prestige',1,1),
(2015,'Audi','A3','1.8T Prestige',1,2),
(2015,'Audi','A3','1.8T Prestige Cabriolet FWD',9,8),
(2015,'Audi','A3','1.8T Prestige Sedan FWD',9,8),
(2015,'Audi','A3','1.8T Prestige Sedan FWD',8,8),
(2015,'Audi','A3','1.8T Progressiv Sedan FWD',8,8),
(2015,'Audi','A3','2.0 TDI Premium',10,1),
(2015,'Audi','A3','2.0 TDI Premium',10,2),
(2015,'Audi','A3','2.0 TDI Premium Plus',10,1),
(2015,'Audi','A3','2.0 TDI Premium Plus',10,2),
(2015,'Audi','A3','2.0 TDI Premium Plus Sedan FWD',11,8),
(2015,'Audi','A3','2.0 TDI Premium Sedan FWD',11,8),
(2015,'Audi','A3','2.0 TDI Prestige',10,1),
(2015,'Audi','A3','2.0 TDI Prestige',10,2),
(2015,'Audi','A3','2.0 TDI Prestige Sedan FWD',11,8),
(2015,'Audi','A3','2.0T Premium',1,1),
(2015,'Audi','A3','2.0T Premium',1,2),
(2015,'Audi','A3','2.0T Premium Plus',1,1),
(2015,'Audi','A3','2.0T Premium Plus',1,2),
(2015,'Audi','A3','2.0T Prestige',1,1),
(2015,'Audi','A3','2.0T Prestige',1,2),
(2015,'Audi','A3','2.0T quattro Komfort Cabriolet AWD',8,8),
(2015,'Audi','A3','2.0T quattro Komfort Sedan AWD',8,8),
(2015,'Audi','A3','2.0T quattro Premium Cabriolet AWD',8,8),
(2015,'Audi','A3','2.0T quattro Premium Plus Cabriolet AWD',8,8),
(2015,'Audi','A3','2.0T quattro Premium Plus Sedan AWD',8,8),
(2015,'Audi','A3','2.0T quattro Premium Sedan AWD',8,8),
(2015,'Audi','A3','2.0T quattro Prestige Cabriolet AWD',8,8),
(2015,'Audi','A3','2.0T quattro Prestige Sedan AWD',8,8),
(2015,'Audi','A3','2.0T quattro Progressiv Cabriolet AWD',8,8),
(2015,'Audi','A3','2.0T quattro Progressiv Sedan AWD',8,8),
(2015,'Audi','A3','2.0T quattro Technik Cabriolet AWD',8,8),
(2015,'Audi','A3','2.0T quattro Technik FWD',1,1),
(2015,'Audi','A3','2.0T quattro Technik FWD',1,2),
(2015,'Audi','A3','2.0T quattro Technik Sedan AWD',8,8),
(2015,'Audi','A3','TDI Komfort Sedan FWD',8,8),
(2015,'Audi','A3','TDI Progressiv Sedan FWD',8,8),
(2015,'Audi','A3','TDI Technik Sedan FWD',8,8),
(2015,'Audi','A4','2.0T FrontTrak Komfort FWD',8,4),
(2015,'Audi','A4','2.0T FrontTrak Komfort FWD',8,9),
(2015,'Audi','A4','2.0T Premium',1,1),
(2015,'Audi','A4','2.0T Premium',1,2),
(2015,'Audi','A4','2.0T Premium FWD',8,10),
(2015,'Audi','A4','2.0T Premium Plus',1,1),
(2015,'Audi','A4','2.0T Premium Plus',1,2),
(2015,'Audi','A4','2.0T Premium Plus FWD',8,10),
(2015,'Audi','A4','2.0T Premium Plus Sedan FWD',8,10),
(2015,'Audi','A4','2.0T Premium Sedan FWD',8,10),
(2015,'Audi','A4','2.0T Prestige',1,1),
(2015,'Audi','A4','2.0T Prestige',1,2),
(2015,'Audi','A4','2.0T Prestige FWD',8,4),
(2015,'Audi','A4','2.0T Prestige FWD',8,9),
(2015,'Audi','A4','2.0T Prestige Sedan FWD',8,10),
(2015,'Audi','A4','2.0T quattro Komfort AWD',8,4),
(2015,'Audi','A4','2.0T quattro Komfort AWD',8,9),
(2015,'Audi','A4','2.0T quattro Premium AWD',8,4),
(2015,'Audi','A4','2.0T quattro Premium AWD',8,9),
(2015,'Audi','A4','2.0T quattro Premium Plus AWD',8,4),
(2015,'Audi','A4','2.0T quattro Premium Plus AWD',8,9),
(2015,'Audi','A4','2.0T quattro Premium Plus Sedan AWD',8,4),
(2015,'Audi','A4','2.0T quattro Premium Plus Sedan AWD',8,9),
(2015,'Audi','A4','2.0T quattro Premium Sedan AWD',8,4),
(2015,'Audi','A4','2.0T quattro Premium Sedan AWD',8,9),
(2015,'Audi','A4','2.0T quattro Prestige AWD',8,4),
(2015,'Audi','A4','2.0T quattro Prestige AWD',8,9),
(2015,'Audi','A4','2.0T quattro Prestige Sedan AWD',8,4),
(2015,'Audi','A4','2.0T quattro Prestige Sedan AWD',8,9),
(2015,'Audi','A4','2.0T quattro Progressiv AWD',8,4),
(2015,'Audi','A4','2.0T quattro Progressiv AWD',8,9),
(2015,'Audi','A4','2.0T quattro Technik AWD',8,4),
(2015,'Audi','A4','2.0T quattro Technik AWD',8,9),
(2015,'Audi','A4 Allroad','2.0T quattro Komfort AWD',12,9),
(2015,'Audi','A4 Allroad','2.0T quattro Premium AWD',8,9),
(2015,'Audi','A4 Allroad','2.0T quattro Premium AWD',12,9),
(2015,'Audi','A4 Allroad','2.0T quattro Premium Plus AWD',8,9),
(2015,'Audi','A4 Allroad','2.0T quattro Premium Plus AWD',12,9),
(2015,'Audi','A4 Allroad','2.0T quattro Prestige AWD',8,9),
(2015,'Audi','A4 Allroad','2.0T quattro Prestige AWD',12,9),
(2015,'Audi','A4 Allroad','2.0T quattro Progressiv AWD',12,9),
(2015,'Audi','A4 Allroad','2.0T quattro Technik AWD',12,9),
(2015,'Audi','A5','2.0T Premium',1,1),
(2015,'Audi','A5','2.0T Premium',1,2),
(2015,'Audi','A5','2.0T Premium Plus',1,1),
(2015,'Audi','A5','2.0T Premium Plus',1,2),
(2015,'Audi','A5','2.0T quattro Komfort Coupe AWD',8,4),
(2015,'Audi','A5','2.0T quattro Komfort Coupe AWD',8,9),
(2015,'Audi','A5','2.0T quattro Premium Cabriolet AWD',8,9),
(2015,'Audi','A5','2.0T quattro Premium Cabriolet AWD',12,9),
(2015,'Audi','A5','2.0T quattro Premium Coupe AWD',8,4),
(2015,'Audi','A5','2.0T quattro Premium Coupe AWD',8,9),
(2015,'Audi','A5','2.0T quattro Premium Plus Cabriolet AWD',8,9),
(2015,'Audi','A5','2.0T quattro Premium Plus Cabriolet AWD',12,9),
(2015,'Audi','A5','2.0T quattro Premium Plus Coupe AWD',8,4),
(2015,'Audi','A5','2.0T quattro Premium Plus Coupe AWD',8,9),
(2015,'Audi','A5','2.0T quattro Prestige Cabriolet AWD',8,4),
(2015,'Audi','A5','2.0T quattro Prestige Cabriolet AWD',8,9),
(2015,'Audi','A5','2.0T quattro Prestige Coupe AWD',8,4),
(2015,'Audi','A5','2.0T quattro Prestige Coupe AWD',8,9),
(2015,'Audi','A5','2.0T quattro Progressiv Cabriolet AWD',8,4),
(2015,'Audi','A5','2.0T quattro Progressiv Cabriolet AWD',8,9),
(2015,'Audi','A5','2.0T quattro Progressiv Coupe AWD',8,4),
(2015,'Audi','A5','2.0T quattro Progressiv Coupe AWD',8,9),
(2015,'Audi','A5','2.0T quattro Progressiv Coupe AWD',1,1),
(2015,'Audi','A5','2.0T quattro Progressiv Coupe AWD',1,2),
(2015,'Audi','A5','2.0T quattro Technik Cabriolet AWD',8,4),
(2015,'Audi','A5','2.0T quattro Technik Cabriolet AWD',8,9),
(2015,'Audi','A5','2.0T quattro Technik Coupe AWD',8,4),
(2015,'Audi','A5','2.0T quattro Technik Coupe AWD',8,9),
(2015,'Audi','A6','2.0T Premium',1,1),
(2015,'Audi','A6','2.0T Premium',1,2),
(2015,'Audi','A6','2.0T Premium Plus Sedan FWD',8,10),
(2015,'Audi','A6','2.0T Premium Sedan FWD',8,10),
(2015,'Audi','A6','2.0T Premium Sedan FWD',13,9),
(2015,'Audi','A6','2.0T quattro Premium Plus Sedan AWD',8,9),
(2015,'Audi','A6','2.0T quattro Premium Sedan AWD',8,9),
(2015,'Audi','A6','2.0T quattro Progressiv Sedan AWD',13,9),
(2015,'Audi','A6','2.0T quattro Technik Sedan AWD',13,9),
(2015,'Audi','A6','3.0 TDI Premium Plus',10,1),
(2015,'Audi','A6','3.0 TDI Premium Plus',10,2),
(2015,'Audi','A6','3.0 TDI quattro Premium Plus Sedan AWD',14,9),
(2015,'Audi','A6','3.0 TDI quattro Prestige Sedan AWD',14,9),
(2015,'Audi','A6','3.0 TDI quattro Progressiv Sedan AWD',13,9),
(2015,'Audi','A6','3.0 TDI quattro Technik Sedan AWD',13,9),
(2015,'Audi','A6','3.0T Premium Plus',1,1),
(2015,'Audi','A6','3.0T Premium Plus',1,2),
(2015,'Audi','A6','3.0T Prestige',1,1),
(2015,'Audi','A6','3.0T Prestige',1,2),
(2015,'Audi','A6','3.0T quattro Premium Plus Sedan AWD',13,9),
(2015,'Audi','A6','3.0T quattro Prestige Sedan AWD',13,9),
(2015,'Audi','A6','3.0T quattro Progressiv Sedan AWD',13,9),
(2015,'Audi','A6','3.0T quattro Technik Sedan AWD',13,9),
(2015,'Audi','A7','3.0 TDI Premium Plus',10,1),
(2015,'Audi','A7','3.0 TDI Premium Plus',10,2),
(2015,'Audi','A7','3.0 TDI quattro Premium Plus AWD',14,9),
(2015,'Audi','A7','3.0 TDI quattro Premium Plus AWD',13,9),
(2015,'Audi','A7','3.0 TDI quattro Prestige AWD',14,9),
(2015,'Audi','A7','3.0 TDI quattro Progressiv AWD',13,9),
(2015,'Audi','A7','3.0 TDI quattro Technik AWD',13,9),
(2015,'Audi','A7','3.0T Premium Plus',1,1),
(2015,'Audi','A7','3.0T Premium Plus',1,2),
(2015,'Audi','A7','3.0T Prestige',1,1),
(2015,'Audi','A7','3.0T Prestige',1,2),
(2015,'Audi','A7','3.0T quattro Premium Plus AWD',13,9),
(2015,'Audi','A7','3.0T quattro Prestige AWD',13,9),
(2015,'Audi','A7','3.0T quattro Progressiv AWD',13,9),
(2015,'Audi','A7','3.0T quattro Technik AWD',13,9),
(2015,'Audi','A8','3.0 TDI quattro AWD',15,9),
(2015,'Audi','A8','3.0T',1,1),
(2015,'Audi','A8','3.0T',1,2),
(2015,'Audi','A8','3.0T quattro AWD',16,9),
(2015,'Audi','A8','4.0T',1,1),
(2015,'Audi','A8','4.0T',1,2),
(2015,'Audi','A8','4.0T quattro AWD',15,9),
(2015,'Audi','A8','L 3.0 TDI',10,1),
(2015,'Audi','A8','L 3.0 TDI',10,2),
(2015,'Audi','A8','L 3.0 TDI quattro AWD',14,9),
(2015,'Audi','A8','L 3.0T',1,1),
(2015,'Audi','A8','L 3.0T',1,2),
(2015,'Audi','A8','L 3.0T quattro AWD',16,9),
(2015,'Audi','A8','L 4.0T',1,1),
(2015,'Audi','A8','L 4.0T',1,2),
(2015,'Audi','A8','L 4.0T quattro AWD',15,9),
(2015,'Audi','A8','L W12 6.3',1,1),
(2015,'Audi','A8','L W12 6.3',1,2),
(2015,'Audi','A8','L W12 quattro AWD',15,9),
(2015,'Audi','A8','L W12 quattro AWD',17,9),
(2015,'Audi','Q3','2.0T Premium Plus',1,1),
(2015,'Audi','Q3','2.0T Premium Plus',1,2),
(2015,'Audi','Q3','2.0T Premium Plus FWD',18,5),
(2015,'Audi','Q3','2.0T Prestige',1,1),
(2015,'Audi','Q3','2.0T Prestige',1,2),
(2015,'Audi','Q3','2.0T Prestige FWD',18,5),
(2015,'Audi','Q3','2.0T Progressiv FWD',18,5),
(2015,'Audi','Q3','2.0T Technik FWD',18,5),
(2015,'Audi','Q3','2.0T quattro Premium Plus AWD',18,5),
(2015,'Audi','Q3','2.0T quattro Prestige AWD',18,5),
(2015,'Audi','Q3','3.0T quattro Progressiv AWD',18,5),
(2015,'Audi','Q3','3.0T quattro Technik AWD',18,5),
(2015,'Audi','Q5','2.0T Premium',1,1),
(2015,'Audi','Q5','2.0T Premium',1,2),
(2015,'Audi','Q5','2.0T Premium Plus',1,1),
(2015,'Audi','Q5','2.0T Premium Plus',1,2),
(2015,'Audi','Q5','2.0T quattro Komfort AWD',19,9),
(2015,'Audi','allroad','2.0T Premium',1,1),
(2015,'Audi','allroad','2.0T Premium',1,2),
(2015,'Audi','allroad','2.0T Premium Plus',1,1),
(2015,'Audi','allroad','2.0T Premium Plus',1,2),
(2015,'Audi','allroad','2.0T Prestige',1,1),
(2015,'Audi','allroad','2.0T Prestige',1,2);

190
data/vehicle-etl/qa_validate.py Executable file
View File

@@ -0,0 +1,190 @@
#!/usr/bin/env python3
"""
Post-import QA validation for vehicle dropdown data.
Runs basic duplicate and range checks against the motovaultpro Postgres container.
"""
import os
import subprocess
import sys
def run_psql(query: str) -> str:
cmd = [
"docker",
"exec",
"mvp-postgres",
"psql",
"-U",
"postgres",
"-d",
"motovaultpro",
"-At",
"-c",
query,
]
return subprocess.check_output(cmd, text=True)
def check_container():
try:
subprocess.check_output(["docker", "ps"], text=True)
except Exception:
print("❌ Docker not available.")
sys.exit(1)
try:
containers = subprocess.check_output(
["docker", "ps", "--filter", "name=mvp-postgres", "--format", "{{.Names}}"],
text=True,
).strip()
if not containers:
print("❌ mvp-postgres container not running.")
sys.exit(1)
except Exception as exc:
print(f"❌ Failed to check containers: {exc}")
sys.exit(1)
def check_invalid_combinations():
"""Verify known invalid combinations do not exist."""
invalid_combos = [
(1992, "Chevrolet", "Corvette", "Z06"), # Z06 started 2001
(2000, "Chevrolet", "Corvette", "35th Anniversary Edition"), # Was 1988
(2000, "Chevrolet", "Corvette", "Stingray"), # Stingray started 2014
(1995, "Ford", "Mustang", "Mach-E"), # Mach-E is 2021+
(2020, "Tesla", "Cybertruck", "Base"), # Not in production until later
]
issues = []
for year, make, model, trim in invalid_combos:
query = f"""
SELECT COUNT(*) FROM vehicle_options
WHERE year = {year}
AND make = '{make}'
AND model = '{model}'
AND trim = '{trim}'
"""
count = int(run_psql(query).strip())
if count > 0:
issues.append(f"Invalid combo found: {year} {make} {model} {trim}")
return issues
def check_trim_coverage():
"""Report on trim coverage statistics."""
query = """
SELECT
COUNT(DISTINCT (year, make, model)) as total_models,
COUNT(DISTINCT (year, make, model)) FILTER (WHERE trim = 'Base') as base_only,
COUNT(DISTINCT (year, make, model)) FILTER (WHERE trim != 'Base') as has_specific_trims
FROM vehicle_options
"""
result = run_psql(query).strip()
print(f"Trim coverage (total/base_only/has_specific_trims): {result}")
def main():
check_container()
print("🔍 Running QA checks...\n")
queries = {
"engine_duplicate_names": """
SELECT COUNT(*) FROM (
SELECT LOWER(name) as n, COUNT(*) c
FROM engines
GROUP BY 1 HAVING COUNT(*) > 1
) t;
""",
"transmission_duplicate_types": """
SELECT COUNT(*) FROM (
SELECT LOWER(type) as t, COUNT(*) c
FROM transmissions
GROUP BY 1 HAVING COUNT(*) > 1
) t;
""",
"vehicle_option_duplicates": """
SELECT COUNT(*) FROM (
SELECT year, make, model, trim, engine_id, transmission_id, COUNT(*) c
FROM vehicle_options
GROUP BY 1,2,3,4,5,6 HAVING COUNT(*) > 1
) t;
""",
"year_range": """
SELECT MIN(year) || ' - ' || MAX(year) FROM vehicle_options;
""",
"year_range_valid": """
SELECT COUNT(*) FROM (
SELECT 1 FROM vehicle_options WHERE year < 2015 OR year > 2022 LIMIT 1
) t;
""",
"counts": """
SELECT
(SELECT COUNT(*) FROM engines) AS engines,
(SELECT COUNT(*) FROM transmissions) AS transmissions,
(SELECT COUNT(*) FROM vehicle_options) AS vehicle_options;
""",
"cross_join_gaps": """
SELECT COUNT(*) FROM (
SELECT base.year, base.make, base.model, base.trim, e.engine_id, t.transmission_id
FROM (
SELECT DISTINCT year, make, model, trim FROM vehicle_options
) base
JOIN (
SELECT DISTINCT year, make, model, trim, engine_id FROM vehicle_options
) e ON base.year = e.year AND base.make = e.make AND base.model = e.model AND base.trim = e.trim
JOIN (
SELECT DISTINCT year, make, model, trim, transmission_id FROM vehicle_options
) t ON base.year = t.year AND base.make = t.make AND base.model = t.model AND base.trim = t.trim
EXCEPT
SELECT year, make, model, trim, engine_id, transmission_id FROM vehicle_options
) gap;
""",
}
results = {}
for key, query in queries.items():
try:
results[key] = run_psql(query).strip()
except subprocess.CalledProcessError as exc:
print(f"❌ Query failed ({key}): {exc}")
sys.exit(1)
issues_found = False
print(f"Engine duplicate names: {results['engine_duplicate_names']}")
print(f"Transmission duplicate types: {results['transmission_duplicate_types']}")
print(f"Vehicle option duplicates: {results['vehicle_option_duplicates']}")
print(f"Year range: {results['year_range']}")
print(f"Out-of-range years (should be 0): {results['year_range_valid']}")
print(f"Counts (engines, transmissions, vehicle_options): {results['counts']}")
print(f"Cross-join gaps (should be 0 to avoid impossible pairs): {results['cross_join_gaps']}")
if (
results["engine_duplicate_names"] != "0"
or results["transmission_duplicate_types"] != "0"
or results["vehicle_option_duplicates"] != "0"
or results["year_range_valid"] != "0"
or results["cross_join_gaps"] != "0"
):
issues_found = True
invalids = check_invalid_combinations()
if invalids:
issues_found = True
print("\n❌ Invalid combinations detected:")
for issue in invalids:
print(f" - {issue}")
else:
print("\n✅ No known invalid year/make/model/trim combos found.")
check_trim_coverage()
if not issues_found:
print("\n✅ QA checks passed.")
else:
print("\n❌ QA checks found issues.")
if __name__ == "__main__":
main()

View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More