egullickson/motovaultpro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Fix imports and database bugs. Removed legacy ETL code.

Browse Source
This commit is contained in:
Eric Gullickson
2025-12-27 12:07:24 -06:00
parent 0d9edbe761
commit bfb0c23ae1
30 changed files with 239174 additions and 4441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/_system/migrations/run-all.ts
View File

@@ -23,6 +23,7 @@ const MIGRATION_ORDER = [
'features/maintenance', // Depends on vehicles
'features/stations', // Independent
'features/admin', // Admin role management and oversight; depends on update_updated_at_column()
'features/backup', // Admin backup feature; depends on update_updated_at_column()
'features/notifications', // Depends on maintenance and documents
'features/user-profile', // User profile management; independent
];

386
backend/src/features/admin/scripts/bulk-import-catalog.ts Normal file
View File

@@ -0,0 +1,386 @@
/**
* Bulk Vehicle Catalog CSV Import
*
* Processes large CSV files (250k+ rows) using batch processing to avoid
* memory and timeout issues that occur in the web import.
*
* Usage (from inside container):
* ts-node src/features/admin/scripts/bulk-import-catalog.ts
*
* CSV Format:
* Required columns: year, make, model, trim
* Optional columns: engine_name, transmission_type
*/
import * as fs from 'fs';
import * as readline from 'readline';
import { pool } from '../../../core/config/database';
const BATCH_SIZE = 5000;
const CSV_PATH = '/tmp/catalog-import.csv';
interface ImportRow {
year: number;
make: string;
model: string;
trim: string;
engineName: string | null;
transmissionType: string | null;
}
interface ImportStats {
totalRows: number;
batchesProcessed: number;
errors: number;
startTime: Date;
}
/**
* Parse a CSV line handling quoted fields
*/
function parseCSVLine(line: string): string[] {
const result: string[] = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
if (char === '"') {
inQuotes = !inQuotes;
} else if (char === ',' && !inQuotes) {
result.push(current.trim());
current = '';
} else {
current += char;
}
}
result.push(current.trim());
return result;
}
/**
* Bulk get or create engines
* Returns map of engine_name -> engine_id
*/
async function getOrCreateEngines(
client: any,
engineNames: string[]
): Promise<Map<string, number>> {
if (engineNames.length === 0) {
return new Map();
}
// Build VALUES clause for bulk insert
const values: any[] = [];
const placeholders = engineNames
.map((name, idx) => {
values.push(name, 'Gas');
return `($${idx * 2 + 1}, $${idx * 2 + 2})`;
})
.join(', ');
const query = `
INSERT INTO engines (name, fuel_type)
VALUES ${placeholders}
ON CONFLICT (LOWER(name)) DO UPDATE
SET name = EXCLUDED.name
RETURNING id, name
`;
const result = await client.query(query, values);
const map = new Map<string, number>();
for (const row of result.rows) {
map.set(row.name, row.id);
}
return map;
}
/**
* Bulk get or create transmissions
* Returns map of transmission_type -> transmission_id
*/
async function getOrCreateTransmissions(
client: any,
transmissionTypes: string[]
): Promise<Map<string, number>> {
if (transmissionTypes.length === 0) {
return new Map();
}
// Build VALUES clause for bulk insert
const values: any[] = [];
const placeholders = transmissionTypes
.map((type, idx) => {
values.push(type);
return `($${idx + 1})`;
})
.join(', ');
const query = `
INSERT INTO transmissions (type)
VALUES ${placeholders}
ON CONFLICT (LOWER(type)) DO UPDATE
SET type = EXCLUDED.type
RETURNING id, type
`;
const result = await client.query(query, values);
const map = new Map<string, number>();
for (const row of result.rows) {
map.set(row.type, row.id);
}
return map;
}
/**
* Process a batch of rows
*/
async function processBatch(
client: any,
batch: ImportRow[],
stats: ImportStats
): Promise<void> {
if (batch.length === 0) {
return;
}
// Extract unique engines and transmissions
const uniqueEngines = new Set<string>();
const uniqueTransmissions = new Set<string>();
for (const row of batch) {
if (row.engineName) {
uniqueEngines.add(row.engineName);
}
if (row.transmissionType) {
uniqueTransmissions.add(row.transmissionType);
}
}
// Get/create engines and transmissions
const engineMap = await getOrCreateEngines(client, Array.from(uniqueEngines));
const transmissionMap = await getOrCreateTransmissions(
client,
Array.from(uniqueTransmissions)
);
// Build vehicle_options batch upsert
const values: any[] = [];
const placeholders = batch
.map((row, idx) => {
const engineId = row.engineName ? engineMap.get(row.engineName) || null : null;
const transmissionId = row.transmissionType
? transmissionMap.get(row.transmissionType) || null
: null;
values.push(
row.year,
row.make,
row.model,
row.trim,
engineId,
transmissionId
);
const base = idx * 6;
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6})`;
})
.join(', ');
const upsertQuery = `
INSERT INTO vehicle_options (year, make, model, trim, engine_id, transmission_id)
VALUES ${placeholders}
ON CONFLICT (year, make, model, trim, engine_id, transmission_id)
DO UPDATE SET
updated_at = NOW()
`;
await client.query(upsertQuery, values);
stats.totalRows += batch.length;
}
/**
* Main import function
*/
async function importCatalog(): Promise<void> {
const stats: ImportStats = {
totalRows: 0,
batchesProcessed: 0,
errors: 0,
startTime: new Date(),
};
console.log('='.repeat(60));
console.log('Vehicle Catalog Bulk Import');
console.log('='.repeat(60));
console.log(`CSV File: ${CSV_PATH}`);
console.log(`Batch Size: ${BATCH_SIZE}`);
console.log('');
// Validate file exists
if (!fs.existsSync(CSV_PATH)) {
console.error(`Error: CSV file not found at ${CSV_PATH}`);
process.exit(1);
}
const fileStream = fs.createReadStream(CSV_PATH);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
let headers: string[] = [];
let headerIndices: Record<string, number> = {};
let batch: ImportRow[] = [];
let isFirstLine = true;
for await (const line of rl) {
// Parse header row
if (isFirstLine) {
headers = parseCSVLine(line);
const headerLower = headers.map((h) => h.toLowerCase().trim());
// Validate required headers
const required = ['year', 'make', 'model', 'trim'];
for (const req of required) {
if (!headerLower.includes(req)) {
console.error(`Error: Missing required header: ${req}`);
process.exit(1);
}
}
// Build header index map
headerIndices = {
year: headerLower.indexOf('year'),
make: headerLower.indexOf('make'),
model: headerLower.indexOf('model'),
trim: headerLower.indexOf('trim'),
engineName: headerLower.indexOf('engine_name'),
transmissionType: headerLower.indexOf('transmission_type'),
};
isFirstLine = false;
continue;
}
// Parse data row
try {
const fields = parseCSVLine(line);
const row: ImportRow = {
year: parseInt(fields[headerIndices.year]),
make: fields[headerIndices.make]?.trim() || '',
model: fields[headerIndices.model]?.trim() || '',
trim: fields[headerIndices.trim]?.trim() || '',
engineName:
headerIndices.engineName >= 0
? fields[headerIndices.engineName]?.trim() || null
: null,
transmissionType:
headerIndices.transmissionType >= 0
? fields[headerIndices.transmissionType]?.trim() || null
: null,
};
batch.push(row);
// Process batch when full
if (batch.length >= BATCH_SIZE) {
const client = await pool.connect();
try {
await client.query('BEGIN');
await processBatch(client, batch, stats);
await client.query('COMMIT');
stats.batchesProcessed++;
const elapsed = (Date.now() - stats.startTime.getTime()) / 1000;
console.log(
`Batch ${stats.batchesProcessed}: ${stats.totalRows.toLocaleString()} rows processed (${elapsed.toFixed(1)}s)`
);
} catch (error: any) {
await client.query('ROLLBACK');
console.error(`Error processing batch ${stats.batchesProcessed + 1}:`, error.message);
stats.errors += batch.length;
} finally {
client.release();
}
batch = [];
}
} catch (error: any) {
stats.errors++;
console.error(`Error parsing row: ${error.message}`);
}
}
// Process remaining rows
if (batch.length > 0) {
const client = await pool.connect();
try {
await client.query('BEGIN');
await processBatch(client, batch, stats);
await client.query('COMMIT');
stats.batchesProcessed++;
const elapsed = (Date.now() - stats.startTime.getTime()) / 1000;
console.log(
`Batch ${stats.batchesProcessed}: ${stats.totalRows.toLocaleString()} rows processed (${elapsed.toFixed(1)}s)`
);
} catch (error: any) {
await client.query('ROLLBACK');
console.error(`Error processing final batch:`, error.message);
stats.errors += batch.length;
} finally {
client.release();
}
}
// Print summary
const totalElapsed = (Date.now() - stats.startTime.getTime()) / 1000;
console.log('');
console.log('='.repeat(60));
console.log('Import Summary');
console.log('='.repeat(60));
console.log(`Total rows processed: ${stats.totalRows.toLocaleString()}`);
console.log(`Batches processed: ${stats.batchesProcessed}`);
console.log(`Errors: ${stats.errors}`);
console.log(`Elapsed time: ${Math.floor(totalElapsed / 60)}m ${(totalElapsed % 60).toFixed(0)}s`);
console.log('');
// Verify counts
const client = await pool.connect();
try {
const voResult = await client.query('SELECT COUNT(*) FROM vehicle_options');
const engResult = await client.query('SELECT COUNT(*) FROM engines');
const transResult = await client.query('SELECT COUNT(*) FROM transmissions');
console.log('Database Verification:');
console.log(` vehicle_options: ${parseInt(voResult.rows[0].count).toLocaleString()}`);
console.log(` engines: ${parseInt(engResult.rows[0].count).toLocaleString()}`);
console.log(` transmissions: ${parseInt(transResult.rows[0].count).toLocaleString()}`);
} finally {
client.release();
}
console.log('');
console.log('Import completed successfully!');
console.log('='.repeat(60));
}
// Run import
importCatalog()
.then(() => {
pool.end();
process.exit(0);
})
.catch((error) => {
console.error('Fatal error:', error);
pool.end();
process.exit(1);
});

404
backend/src/features/platform/migrations/001_create_vehicle_lookup_schema.sql
View File

@@ -1,139 +1,293 @@
-- Create dedicated schema for normalized vehicle lookup data
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_namespace WHERE nspname = 'vehicles'
) THEN
EXECUTE 'CREATE SCHEMA vehicles';
END IF;
END;
$$;
-- Migration: Create Automotive Vehicle Selection Database
-- Optimized for dropdown cascade queries
-- Date: 2025-11-10
-- Create manufacturers table
CREATE TABLE IF NOT EXISTS vehicles.make (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(150) NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
-- Drop existing tables if they exist
DROP TABLE IF EXISTS vehicle_options CASCADE;
DROP TABLE IF EXISTS engines CASCADE;
DROP TABLE IF EXISTS transmissions CASCADE;
DROP INDEX IF EXISTS idx_vehicle_year;
DROP INDEX IF EXISTS idx_vehicle_make;
DROP INDEX IF EXISTS idx_vehicle_model;
DROP INDEX IF EXISTS idx_vehicle_trim;
DROP INDEX IF EXISTS idx_vehicle_composite;
-- Create engines table with detailed specifications
CREATE TABLE engines (
id SERIAL PRIMARY KEY,
name VARCHAR(255) NOT NULL,
displacement VARCHAR(50),
configuration VARCHAR(50),
horsepower VARCHAR(100),
torque VARCHAR(100),
fuel_type VARCHAR(100),
fuel_system VARCHAR(255),
aspiration VARCHAR(100),
specs_json JSONB,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Prevent duplicate engine display names (case-insensitive)
CREATE UNIQUE INDEX IF NOT EXISTS uq_engines_name_lower ON engines (LOWER(name));
CREATE INDEX idx_engines_displacement ON engines(displacement);
CREATE INDEX idx_engines_config ON engines(configuration);
-- Create transmissions table
CREATE TABLE transmissions (
id SERIAL PRIMARY KEY,
type VARCHAR(100) NOT NULL,
speeds VARCHAR(50),
drive_type VARCHAR(100),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Prevent duplicate transmission display names (case-insensitive)
CREATE UNIQUE INDEX IF NOT EXISTS uq_transmissions_type_lower ON transmissions (LOWER(type));
CREATE INDEX idx_transmissions_type ON transmissions(type);
-- Create denormalized vehicle_options table optimized for dropdown queries
CREATE TABLE vehicle_options (
id SERIAL PRIMARY KEY,
year INTEGER NOT NULL,
make VARCHAR(100) NOT NULL,
model VARCHAR(255) NOT NULL,
trim VARCHAR(255) NOT NULL,
engine_id INTEGER REFERENCES engines(id) ON DELETE SET NULL,
transmission_id INTEGER REFERENCES transmissions(id) ON DELETE SET NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Prevent duplicate vehicle option rows
CREATE UNIQUE INDEX IF NOT EXISTS uq_vehicle_options_full ON vehicle_options (
year, make, model, trim, engine_id, transmission_id
);
CREATE OR REPLACE FUNCTION vehicles.touch_updated_at()
RETURNS TRIGGER AS $$
-- Indexes for cascading dropdown performance
CREATE INDEX idx_vehicle_year ON vehicle_options(year);
CREATE INDEX idx_vehicle_make ON vehicle_options(make);
CREATE INDEX idx_vehicle_model ON vehicle_options(model);
CREATE INDEX idx_vehicle_trim ON vehicle_options(trim);
CREATE INDEX idx_vehicle_year_make ON vehicle_options(year, make);
CREATE INDEX idx_vehicle_year_make_model ON vehicle_options(year, make, model);
CREATE INDEX idx_vehicle_year_make_model_trim ON vehicle_options(year, make, model, trim);
CREATE INDEX idx_vehicle_year_make_model_trim_engine ON vehicle_options(year, make, model, trim, engine_id);
CREATE INDEX idx_vehicle_year_make_model_trim_trans ON vehicle_options(year, make, model, trim, transmission_id);
-- Full-text search index for admin catalog search
CREATE INDEX idx_vehicle_options_fts ON vehicle_options
USING gin(to_tsvector('english', year::text || ' ' || make || ' ' || model || ' ' || trim));
-- Index on engines.name for join performance during search
CREATE INDEX idx_engines_name ON engines(name);
-- Views for dropdown queries
-- View: Get all available years
CREATE OR REPLACE VIEW available_years AS
SELECT DISTINCT year
FROM vehicle_options
ORDER BY year DESC;
-- View: Get makes by year
CREATE OR REPLACE VIEW makes_by_year AS
SELECT DISTINCT year, make
FROM vehicle_options
ORDER BY year DESC, make ASC;
-- View: Get models by year and make
CREATE OR REPLACE VIEW models_by_year_make AS
SELECT DISTINCT year, make, model
FROM vehicle_options
ORDER BY year DESC, make ASC, model ASC;
-- View: Get trims by year, make, and model
CREATE OR REPLACE VIEW trims_by_year_make_model AS
SELECT DISTINCT year, make, model, trim
FROM vehicle_options
ORDER BY year DESC, make ASC, model ASC, trim ASC;
-- View: Get complete vehicle configurations with engine and transmission details
CREATE OR REPLACE VIEW complete_vehicle_configs AS
SELECT
vo.id,
vo.year,
vo.make,
vo.model,
vo.trim,
e.name AS engine_name,
e.displacement,
e.configuration,
e.horsepower,
e.torque,
e.fuel_type,
t.type AS transmission_type,
t.speeds AS transmission_speeds,
t.drive_type
FROM vehicle_options vo
LEFT JOIN engines e ON vo.engine_id = e.id
LEFT JOIN transmissions t ON vo.transmission_id = t.id
ORDER BY vo.year DESC, vo.make ASC, vo.model ASC, vo.trim ASC;
-- Function to get makes for a specific year
CREATE OR REPLACE FUNCTION get_makes_for_year(p_year INTEGER)
RETURNS TABLE(make VARCHAR) AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
RETURN QUERY
SELECT DISTINCT vehicle_options.make
FROM vehicle_options
WHERE vehicle_options.year = p_year
ORDER BY vehicle_options.make ASC;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS touch_make_updated_at ON vehicles.make;
CREATE TRIGGER touch_make_updated_at
BEFORE UPDATE ON vehicles.make
FOR EACH ROW
EXECUTE FUNCTION vehicles.touch_updated_at();
-- Function to get models for a specific year and make
CREATE OR REPLACE FUNCTION get_models_for_year_make(p_year INTEGER, p_make VARCHAR)
RETURNS TABLE(model VARCHAR) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT vehicle_options.model
FROM vehicle_options
WHERE vehicle_options.year = p_year
AND vehicle_options.make = p_make
ORDER BY vehicle_options.model ASC;
END;
$$ LANGUAGE plpgsql;
-- Create models table
CREATE TABLE IF NOT EXISTS vehicles.model (
id BIGSERIAL PRIMARY KEY,
make_id BIGINT NOT NULL REFERENCES vehicles.make(id) ON DELETE CASCADE,
name VARCHAR(150) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT vehicles_model_unique UNIQUE(make_id, name)
);
-- Function to get trims for a specific year, make, and model
CREATE OR REPLACE FUNCTION get_trims_for_year_make_model(p_year INTEGER, p_make VARCHAR, p_model VARCHAR)
RETURNS TABLE(trim_name VARCHAR) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT vehicle_options.trim
FROM vehicle_options
WHERE vehicle_options.year = p_year
AND vehicle_options.make = p_make
AND vehicle_options.model = p_model
ORDER BY vehicle_options.trim ASC;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS touch_model_updated_at ON vehicles.model;
CREATE TRIGGER touch_model_updated_at
BEFORE UPDATE ON vehicles.model
FOR EACH ROW
EXECUTE FUNCTION vehicles.touch_updated_at();
-- Function to get engine and transmission options for a specific vehicle
CREATE OR REPLACE FUNCTION get_options_for_vehicle(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR)
RETURNS TABLE(
engine_name VARCHAR,
engine_displacement VARCHAR,
engine_horsepower VARCHAR,
transmission_type VARCHAR,
transmission_speeds VARCHAR,
drive_type VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT
e.name,
e.displacement,
e.horsepower,
t.type,
t.speeds,
t.drive_type
FROM vehicle_options vo
LEFT JOIN engines e ON vo.engine_id = e.id
LEFT JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim;
END;
$$ LANGUAGE plpgsql;
-- Create model_year table
CREATE TABLE IF NOT EXISTS vehicles.model_year (
id BIGSERIAL PRIMARY KEY,
model_id BIGINT NOT NULL REFERENCES vehicles.model(id) ON DELETE CASCADE,
year INTEGER NOT NULL CHECK (year BETWEEN 1900 AND 2100),
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT vehicles_model_year_unique UNIQUE(model_id, year)
);
-- Helper functions for trim-level options and pair-safe filtering
CREATE OR REPLACE FUNCTION get_transmissions_for_vehicle(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR)
RETURNS TABLE(
transmission_id INTEGER,
transmission_type VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
t.id,
t.type
FROM vehicle_options vo
JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
ORDER BY t.type ASC;
END;
$$ LANGUAGE plpgsql;
CREATE INDEX IF NOT EXISTS idx_model_year_year ON vehicles.model_year(year DESC);
CREATE OR REPLACE FUNCTION get_engines_for_vehicle(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR)
RETURNS TABLE(
engine_id INTEGER,
engine_name VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
e.id,
e.name
FROM vehicle_options vo
JOIN engines e ON vo.engine_id = e.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
ORDER BY e.name ASC;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS touch_model_year_updated_at ON vehicles.model_year;
CREATE TRIGGER touch_model_year_updated_at
BEFORE UPDATE ON vehicles.model_year
FOR EACH ROW
EXECUTE FUNCTION vehicles.touch_updated_at();
CREATE OR REPLACE FUNCTION get_transmissions_for_vehicle_engine(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR, p_engine_name VARCHAR)
RETURNS TABLE(
transmission_id INTEGER,
transmission_type VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
t.id,
t.type
FROM vehicle_options vo
JOIN engines e ON vo.engine_id = e.id
JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
AND e.name = p_engine_name
ORDER BY t.type ASC;
END;
$$ LANGUAGE plpgsql;
-- Create trims table
CREATE TABLE IF NOT EXISTS vehicles.trim (
id BIGSERIAL PRIMARY KEY,
model_year_id BIGINT NOT NULL REFERENCES vehicles.model_year(id) ON DELETE CASCADE,
name VARCHAR(150) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT vehicles_trim_unique UNIQUE(model_year_id, name)
);
CREATE OR REPLACE FUNCTION get_engines_for_vehicle_trans(p_year INTEGER, p_make VARCHAR, p_model VARCHAR, p_trim VARCHAR, p_trans_type VARCHAR)
RETURNS TABLE(
engine_id INTEGER,
engine_name VARCHAR
) AS $$
BEGIN
RETURN QUERY
SELECT DISTINCT
e.id,
e.name
FROM vehicle_options vo
JOIN engines e ON vo.engine_id = e.id
JOIN transmissions t ON vo.transmission_id = t.id
WHERE vo.year = p_year
AND vo.make = p_make
AND vo.model = p_model
AND vo.trim = p_trim
AND t.type = p_trans_type
ORDER BY e.name ASC;
END;
$$ LANGUAGE plpgsql;
CREATE INDEX IF NOT EXISTS idx_trim_model_year ON vehicles.trim(model_year_id);
DROP TRIGGER IF EXISTS touch_trim_updated_at ON vehicles.trim;
CREATE TRIGGER touch_trim_updated_at
BEFORE UPDATE ON vehicles.trim
FOR EACH ROW
EXECUTE FUNCTION vehicles.touch_updated_at();
-- Create engines table
CREATE TABLE IF NOT EXISTS vehicles.engine (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(200) NOT NULL UNIQUE,
code VARCHAR(50),
displacement_l NUMERIC(5,2),
cylinders SMALLINT,
fuel_type VARCHAR(50),
aspiration VARCHAR(50),
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
DROP TRIGGER IF EXISTS touch_engine_updated_at ON vehicles.engine;
CREATE TRIGGER touch_engine_updated_at
BEFORE UPDATE ON vehicles.engine
FOR EACH ROW
EXECUTE FUNCTION vehicles.touch_updated_at();
-- Create trim-engine bridge table
CREATE TABLE IF NOT EXISTS vehicles.trim_engine (
trim_id BIGINT NOT NULL REFERENCES vehicles.trim(id) ON DELETE CASCADE,
engine_id BIGINT NOT NULL REFERENCES vehicles.engine(id) ON DELETE CASCADE,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (trim_id, engine_id)
);
-- Create transmissions table (static manual/automatic for now)
CREATE TABLE IF NOT EXISTS vehicles.transmission (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(50) NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
DROP TRIGGER IF EXISTS touch_transmission_updated_at ON vehicles.transmission;
CREATE TRIGGER touch_transmission_updated_at
BEFORE UPDATE ON vehicles.transmission
FOR EACH ROW
EXECUTE FUNCTION vehicles.touch_updated_at();
-- Optional bridge for future proofing (not yet populated)
CREATE TABLE IF NOT EXISTS vehicles.trim_transmission (
trim_id BIGINT NOT NULL REFERENCES vehicles.trim(id) ON DELETE CASCADE,
transmission_id BIGINT NOT NULL REFERENCES vehicles.transmission(id) ON DELETE CASCADE,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (trim_id, transmission_id)
);
-- Helpful indexes for cascading dropdown lookups
CREATE INDEX IF NOT EXISTS idx_model_make ON vehicles.model(make_id);
CREATE INDEX IF NOT EXISTS idx_trim_name ON vehicles.trim(LOWER(name));
CREATE INDEX IF NOT EXISTS idx_engine_name ON vehicles.engine(LOWER(name));
CREATE INDEX IF NOT EXISTS idx_trim_engine_engine ON vehicles.trim_engine(engine_id);
COMMENT ON TABLE vehicle_options IS 'Denormalized table optimized for cascading dropdown queries';
COMMENT ON TABLE engines IS 'Engine specifications with detailed technical data';
COMMENT ON TABLE transmissions IS 'Transmission specifications';
COMMENT ON VIEW available_years IS 'Returns all distinct years available in the database';
COMMENT ON VIEW makes_by_year IS 'Returns makes grouped by year for dropdown population';
COMMENT ON VIEW models_by_year_make IS 'Returns models grouped by year and make';
COMMENT ON VIEW trims_by_year_make_model IS 'Returns trims grouped by year, make, and model';
COMMENT ON VIEW complete_vehicle_configs IS 'Complete vehicle configurations with all details';

BIN
data/backups/motovaultpro_backup_2025-12-25T16-47-55.tar.gz
View File

Binary file not shown.

BIN
data/backups/motovaultpro_backup_2025-12-27T18-03-54.tar.gz Normal file
View File

Binary file not shown.

0
data/vehicle-etl/snapshots/.gitkeep → data/backups/test.txt
View File

74
data/vehicle-etl/ACURA.txt
View File

@@ -1,74 +0,0 @@
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
-----------------------------------------------------+-----------------------------------------------+-----------------------------------------------------------------------------------------------------------------+---------------+-----------------+--------------+---------------+-----------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+----------------------------------+--------------
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
| Vehicle Descriptor | 5J8YE1H0*SL | | | | 196 | 5J8YE1H0*SL | | | VehicleDescriptor | string | Decoding | Corrections |
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
| Additional Error Text | | | | | 156 | | | | AdditionalErrorText | string | Decoding | Corrections |
General | Vehicle Type | MULTIPURPOSE PASSENGER VEHICLE (MPV) | | | 5J8 | 39 | 7 | 2015-03-27 15:31:28.273 | 2105 | VehicleType | lookup | WMI | VehType |
General | Make | ACURA | 2074168 | 26929 | Y[ED]*H | 26 | 475 | | | Make | lookup | WMI, Pattern | pattern - model |
General | Manufacturer Name | AMERICAN HONDA MOTOR CO., INC. | | | 5J8 | 27 | 988 | | 2105 | Manufacturer | lookup | WMI | Manu. Name |
General | Model Year | 2025 | | | ***X*|Y | 29 | 2025 | | | ModelYear | int | WMI, Input | ModelYear |
General | Model | MDX | 2074168 | 26929 | Y[ED]*H | 28 | 2147 | 2024-05-21 10:17:20.287 | 2105 | Model | lookup | Pattern | Pattern |
General | Plant City | EAST LIBERTY | 2117059 | 27341 | *****|*L | 31 | EAST LIBERTY | 2024-08-27 07:36:52.53 | 2105 | PlantCity | string | Pattern | Pattern |
General | Plant State | OHIO | 2117056 | 27341 | *****|*[ALY] | 77 | OHIO | 2024-08-27 07:35:26.557 | 2105 | PlantState | string | Pattern | Pattern |
General | Plant Country | UNITED STATES (USA) | 2117055 | 27341 | *****|*[ALY] | 75 | 6 | 2024-08-27 07:35:26.54 | 2105 | PlantCountry | lookup | Pattern | Pattern |
General | Trim | SH-AWD A-Spec | 2074202 | 26929 | YE1H0 | 38 | SH-AWD A-Spec | 2024-05-21 10:24:09.057 | 2105 | Trim | string | Pattern | Pattern |
Exterior / Body | Body Class | Sport Utility Vehicle (SUV)/Multi-Purpose Vehicle (MPV) | 2074165 | 26929 | Y[ED]*H | 5 | 7 | 2024-05-21 10:17:20.143 | 2105 | BodyClass | lookup | Pattern | Pattern |
Exterior / Body | Doors | 5 | 2074166 | 26929 | Y[ED]*H | 14 | 5 | 2024-05-21 10:17:20.223 | 2105 | Doors | int | Pattern | Pattern |
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 1D: 5,001 - 6,000 lb (2,268 - 2,722 kg) | 2074167 | 26929 | Y[ED]*H | 25 | 13 | 2024-05-21 10:17:20.27 | 2105 | GVWR | lookup | Pattern | Pattern |
Exterior / Dimension | Gross Vehicle Weight Rating To | Class 1D: 5,001 - 6,000 lb (2,268 - 2,722 kg) | 2074180 | 26929 | Y[ED]*H | 190 | 13 | 2024-05-21 10:17:20.63 | 2105 | GVWR_to | lookup | Pattern | Pattern |
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
Mechanical / Transmission | Transmission Speeds | 10 | 2074171 | 26929 | Y[ED]*H | 63 | 10 | 2024-05-21 10:17:20.343 | 2105 | TransmissionSpeeds | int | Pattern | Pattern |
Mechanical / Transmission | Transmission Style | Automatic | 2074169 | 26929 | Y[ED]*H | 37 | 2 | 2024-05-21 10:17:20.303 | 2105 | TransmissionStyle | lookup | Pattern | Pattern |
Mechanical / Drivetrain | Drive Type | 4WD/4-Wheel Drive/4x4 | 2074181 | 26929 | YE1 | 15 | 2 | 2024-05-21 10:18:30.127 | 2105 | DriveType | lookup | Pattern | Pattern |
Engine | Other Engine Info | Direct Fuel Injection | 2074214 | 26929 | Y[ED][19] | 129 | Direct Fuel Injection | 2024-05-21 10:42:05.407 | 2105 | OtherEngineInfo | string | Pattern | Pattern |
Engine | Displacement (CC) | 3500.0 | 2074206 | 26929 | Y[ED][19] | 11 | 3500.0 | | 2105 | DisplacementCC | decimal | Pattern | Conversion 4: 3.5 * 1000 |
Engine | Displacement (CI) | 213.5831043315629938 | 2074206 | 26929 | Y[ED][19] | 12 | 213.5831043315629938 | | 2105 | DisplacementCI | decimal | Pattern | Conversion 7: 3.5 / 0.016387064 |
Engine | Engine Number of Cylinders | 6 | 2074205 | 26929 | Y[ED][19] | 9 | 6 | 2024-05-21 10:42:05.137 | 2105 | EngineCylinders | int | Pattern | Pattern |
Engine | Displacement (L) | 3.5 | 2074206 | 26929 | Y[ED][19] | 13 | 3.5 | 2024-05-21 10:42:05.157 | 2105 | DisplacementL | decimal | Pattern | Pattern |
Engine | Engine Stroke Cycles | 4 | 2074207 | 26929 | Y[ED][19] | 17 | 4 | 2024-05-21 10:42:05.287 | 2105 | EngineCycles | int | Pattern | Pattern |
Engine | Engine Model | J35Y5 | 2074208 | 26929 | Y[ED][19] | 18 | J35Y5 | 2024-05-21 10:42:05.303 | 2105 | EngineModel | string | Pattern | Pattern |
Engine | Fuel Type - Primary | Gasoline | 2074209 | 26929 | Y[ED][19] | 24 | 4 | 2024-05-21 10:42:05.32 | 2105 | FuelTypePrimary | lookup | Pattern | Pattern |
Engine | Valve Train Design | Single Overhead Cam (SOHC) | 2074210 | 26929 | Y[ED][19] | 62 | 4 | 2024-05-21 10:42:05.337 | 2105 | ValveTrainDesign | lookup | Pattern | Pattern |
Engine | Engine Configuration | V-Shaped | 2074211 | 26929 | Y[ED][19] | 64 | 2 | 2024-05-21 10:42:05.353 | 2105 | EngineConfiguration | lookup | Pattern | Pattern |
Engine | Engine Brake (hp) From | 290 | 2074212 | 26929 | Y[ED][19] | 71 | 290 | 2024-05-21 10:42:05.37 | 2105 | EngineHP | decimal | Pattern | Pattern |
Engine | Cooling Type | Water | 2074213 | 26929 | Y[ED][19] | 122 | 2 | 2024-05-21 10:42:05.39 | 2105 | CoolingType | lookup | Pattern | Pattern |
Engine | Engine Manufacturer | Honda | 2074215 | 26929 | Y[ED][19] | 146 | Honda | 2024-05-21 10:42:05.423 | 2105 | EngineManufacturer | string | Pattern | Pattern |
Passive Safety System | Other Restraint System Info | Front: Seat Belts, Mid/Rear: Seat Belt and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2074179 | 26929 | Y[ED]*H | 121 | Front: Seat Belts, Mid/Rear: Seat Belt and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2024-05-21 10:17:20.61 | 2105 | OtherRestraintSystemInfo | string | Pattern | Pattern |
Passive Safety System | Seat Belt Type | Manual | 2074174 | 26929 | Y[ED]*H | 79 | 1 | 2024-05-21 10:17:20.393 | 2105 | SeatBeltsAll | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st Row (Driver and Passenger) | 2074178 | 26929 | Y[ED]*H | 107 | 3 | 2024-05-21 10:17:20.593 | 2105 | AirBagLocSide | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Knee Air Bag Locations | 1st Row (Driver and Passenger) | 2074173 | 26929 | Y[ED]*H | 69 | 3 | 2024-05-21 10:17:20.377 | 2105 | AirBagLocKnee | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 2074172 | 26929 | Y[ED]*H | 65 | 3 | 2024-05-21 10:17:20.36 | 2105 | AirBagLocFront | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | 1st and 2nd and 3rd Rows | 2074170 | 26929 | Y[ED]*H | 55 | 5 | 2024-05-21 10:17:20.327 | 2105 | AirBagLocCurtain | lookup | Pattern | Pattern |
Active Safety System | Keyless Ignition | Standard | 16875 | 5601 | | 176 | 1 | 2025-04-03 14:07:35.83 | | KeylessIgnition | lookup | Pattern | Vehicle Specs |
Active Safety System | Electronic Stability Control (ESC) | Standard | 16875 | 5601 | | 99 | 1 | 2025-04-03 14:07:35.587 | | ESC | lookup | Pattern | Vehicle Specs |
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Direct | 16875 | 5601 | | 168 | 1 | 2025-04-03 14:07:35.767 | | TPMS | lookup | Pattern | Vehicle Specs |
Active Safety System | Event Data Recorder (EDR) | Standard | 16875 | 5601 | | 175 | 1 | 2025-04-03 14:07:35.817 | | EDR | lookup | Pattern | Vehicle Specs |
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Standard | 16875 | 5601 | | 172 | 1 | 2025-04-03 14:07:35.803 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs |
Active Safety System | Anti-lock Braking System (ABS) | Standard | 16875 | 5601 | | 86 | 1 | 2025-04-03 14:07:35.56 | | ABS | lookup | Pattern | Vehicle Specs |
Active Safety System | Traction Control | Standard | 16875 | 5601 | | 100 | 1 | 2025-04-03 14:07:35.6 | | TractionControl | lookup | Pattern | Vehicle Specs |
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Standard | 16875 | 5601 | | 81 | 1 | 2025-04-03 14:07:35.55 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 16875 | 5601 | | 101 | 1 | 2025-04-03 14:07:35.613 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 16875 | 5601 | | 170 | 1 | 2025-04-03 14:07:35.78 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 16875 | 5601 | | 171 | 1 | 2025-04-03 14:07:35.79 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 16875 | 5601 | | 87 | 1 | 2025-04-03 14:07:35.573 | | CIB | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Standard | 16875 | 5601 | | 103 | 1 | 2025-04-03 14:07:35.64 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 16875 | 5601 | | 102 | 1 | 2025-04-03 14:07:35.627 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Centering Assistance | Standard | 16875 | 5601 | | 194 | 1 | 2025-04-03 14:07:36.043 | | LaneCenteringAssistance | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Blind Spot Intervention (BSI) | Standard | 16875 | 5601 | | 193 | 1 | 2025-04-03 14:07:36.03 | | BlindSpotIntervention | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Standard | 16875 | 5601 | | 192 | 1 | 2025-04-03 14:07:36.01 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Standard | 16875 | 5601 | | 183 | 1 | 2025-04-03 14:07:35.897 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 16875 | 5601 | | 104 | 1 | 2025-04-03 14:07:35.753 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 16875 | 5601 | | 178 | 3 | 2025-04-03 14:07:35.863 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 16875 | 5601 | | 177 | 1 | 2025-04-03 14:07:35.843 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 16875 | 5601 | | 179 | 1 | 2025-04-03 14:07:35.88 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs |
(70 rows)

64
data/vehicle-etl/BMW.txt
View File

@@ -1,64 +0,0 @@
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
-----------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+----------------------------------+--------------
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
| Additional Error Text | | | | | 156 | | | | AdditionalErrorText | string | Decoding | Corrections |
| Vehicle Descriptor | 5YM13ET0*R9 | | | | 196 | 5YM13ET0*R9 | | | VehicleDescriptor | string | Decoding | Corrections |
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
General | Vehicle Type | MULTIPURPOSE PASSENGER VEHICLE (MPV) | | | 5YM | 39 | 7 | 2015-03-11 13:50:19.203 | 2020 | VehicleType | lookup | WMI | VehType |
General | Plant State | SOUTH CAROLINA | 1765033 | 23981 | *****|*9 | 77 | SOUTH CAROLINA | 2022-06-22 14:28:01.55 | 2020 | PlantState | string | Pattern | Pattern |
General | Plant Country | UNITED STATES (USA) | 1765032 | 23981 | *****|*9 | 75 | 6 | 2022-06-22 14:28:01.55 | 2020 | PlantCountry | lookup | Pattern | Pattern |
General | Plant City | GREER | 1765031 | 23981 | *****|*9 | 31 | GREER | 2022-06-22 14:28:01.55 | 2020 | PlantCity | string | Pattern | Pattern |
General | Manufacturer Name | BMW MANUFACTURER CORPORATION / BMW NORTH AMERICA | | | 5YM | 27 | 968 | | 2020 | Manufacturer | lookup | WMI | Manu. Name |
General | Make | BMW | 1895508 | 25151 | 13ET0 | 26 | 452 | | | Make | lookup | WMI, Pattern | pattern - model |
General | Model | X5 | 1895508 | 25151 | 13ET0 | 28 | 1717 | 2023-03-09 15:43:52.33 | 2020 | Model | lookup | Pattern | Pattern | f
General | Trim | X5 M Competition | 2031372 | 25151 | 13ET0 | 38 | X5 M Competition | 2024-02-20 12:44:30.22 | 2020 | Trim | string | Pattern | Pattern | f
General | Model Year | 2024 | | | ***X*|Y | 29 | 2024 | | | ModelYear | int | WMI, Input | ModelYear |
Exterior / Body | Doors | 4 | 1895505 | 25151 | 13ET0 | 14 | 4 | 2023-03-09 15:43:52.28 | 2020 | Doors | int | Pattern | Pattern | f
Exterior / Body | Body Class | Sport Utility Vehicle (SUV)/Multi-Purpose Vehicle (MPV) | 1895502 | 25151 | 13ET0 | 5 | 7 | 2023-03-09 15:43:52.227 | 2020 | BodyClass | lookup | Pattern | Pattern | f
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 2E: 6,001 - 7,000 lb (2,722 - 3,175 kg) | 1895507 | 25151 | 13ET0 | 25 | 14 | 2023-03-09 15:43:52.313 | 2020 | GVWR | lookup | Pattern | Pattern | f
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
Engine | Engine Number of Cylinders | 8 | 1895503 | 25151 | 13ET0 | 9 | 8 | 2023-03-09 15:43:52.243 | 2020 | EngineCylinders | int | Pattern | Pattern | f
Engine | Displacement (CC) | 4400.0 | 1895504 | 25151 | 13ET0 | 11 | 4400.0 | | 2020 | DisplacementCC | decimal | Pattern | Conversion 4: 4.4 * 1000 |
Engine | Displacement (CI) | 268.5044740168220494 | 1895504 | 25151 | 13ET0 | 12 | 268.5044740168220494 | | 2020 | DisplacementCI | decimal | Pattern | Conversion 7: 4.4 / 0.016387064 |
Engine | Displacement (L) | 4.4 | 1895504 | 25151 | 13ET0 | 13 | 4.4 | 2023-03-09 15:43:52.26 | 2020 | DisplacementL | decimal | Pattern | Pattern | f
Engine | Fuel Type - Primary | Gasoline | 1895506 | 25151 | 13ET0 | 24 | 4 | 2023-03-09 15:43:52.297 | 2020 | FuelTypePrimary | lookup | Pattern | Pattern | f
Engine | Engine Brake (hp) From | 617 | 1895513 | 25151 | 13ET0 | 71 | 617 | 2023-03-09 15:43:52.537 | 2020 | EngineHP | decimal | Pattern | Pattern | f
Passive Safety System | Seat Belt Type | Manual | 1895515 | 25151 | 13ET0 | 79 | 1 | 2023-03-09 15:43:52.57 | 2020 | SeatBeltsAll | lookup | Pattern | Pattern | f
Passive Safety System | Other Restraint System Info | Seat Belt: All positions / Pretensioners, Side Airbags, Head Inflatable Restraint: Front Row & Rear Outboard Driver-side, Rear Outboard Passenger-side | 1895520 | 25151 | 13ET0 | 121 | Seat Belt: All positions / Pretensioners, Side Airbags, Head Inflatable Restraint: Front Row & Rear Outboard Driver-side, Rear Outboard Passenger-side | 2023-03-09 15:43:52.77 | 2020 | OtherRestraintSystemInfo | string | Pattern | Pattern | f
Passive Safety System | Pretensioner | Yes | 1895514 | 25151 | 13ET0 | 78 | 1 | 2023-03-09 15:43:52.553 | 2020 | Pretensioner | lookup | Pattern | Pattern | f
Passive Safety System / Air Bag Location | Knee Air Bag Locations | 1st Row (Driver and Passenger) | 1895512 | 25151 | 13ET0 | 69 | 3 | 2023-03-09 15:43:52.517 | 2020 | AirBagLocKnee | lookup | Pattern | Pattern | f
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st and 2nd Rows | 1895519 | 25151 | 13ET0 | 107 | 4 | 2023-03-09 15:43:52.633 | 2020 | AirBagLocSide | lookup | Pattern | Pattern | f
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | 1st and 2nd Rows | 1895510 | 25151 | 13ET0 | 55 | 4 | 2023-03-09 15:43:52.483 | 2020 | AirBagLocCurtain | lookup | Pattern | Pattern | f
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 1895511 | 25151 | 13ET0 | 65 | 3 | 2023-03-09 15:43:52.5 | 2020 | AirBagLocFront | lookup | Pattern | Pattern | f
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Direct | 14937 | 4656 | | 168 | 1 | 2024-03-12 13:41:17.253 | | TPMS | lookup | Pattern | Vehicle Specs |
Active Safety System | Anti-lock Braking System (ABS) | Standard | 14937 | 4656 | | 86 | 1 | 2024-03-12 13:41:17.01 | | ABS | lookup | Pattern | Vehicle Specs |
Active Safety System | Keyless Ignition | Standard | 14937 | 4656 | | 176 | 1 | 2024-03-12 13:41:17.317 | | KeylessIgnition | lookup | Pattern | Vehicle Specs |
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Standard | 14937 | 4656 | | 172 | 1 | 2024-03-12 13:41:17.3 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs |
Active Safety System | Electronic Stability Control (ESC) | Standard | 14937 | 4656 | | 99 | 1 | 2024-03-12 13:41:17.05 | | ESC | lookup | Pattern | Vehicle Specs |
Active Safety System | Traction Control | Standard | 14937 | 4656 | | 100 | 1 | 2024-03-12 13:41:17.183 | | TractionControl | lookup | Pattern | Vehicle Specs |
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Optional | 14937 | 4656 | | 81 | 3 | 2024-03-12 13:41:16.993 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 14937 | 4656 | | 170 | 1 | 2024-03-12 13:41:17.267 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 14937 | 4656 | | 101 | 1 | 2024-03-12 13:41:17.2 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 14937 | 4656 | | 87 | 1 | 2024-03-12 13:41:17.02 | | CIB | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 14937 | 4656 | | 171 | 1 | 2024-03-12 13:41:17.28 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Centering Assistance | Optional | 14937 | 4656 | | 194 | 3 | 2024-03-12 13:41:17.543 | | LaneCenteringAssistance | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Blind Spot Warning (BSW) | Standard | 14937 | 4656 | | 88 | 1 | 2024-03-12 13:41:17.037 | | BlindSpotMon | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Optional | 14937 | 4656 | | 103 | 5 | 2024-03-12 13:41:17.227 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Blind Spot Intervention (BSI) | Optional | 14937 | 4656 | | 193 | 3 | 2024-03-12 13:41:17.53 | | BlindSpotIntervention | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 14937 | 4656 | | 102 | 1 | 2024-03-12 13:41:17.213 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Standard | 14937 | 4656 | | 192 | 1 | 2024-03-12 13:41:17.517 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Optional | 14937 | 4656 | | 183 | 2 | 2024-03-12 13:41:17.503 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 14937 | 4656 | | 104 | 1 | 2024-03-12 13:41:17.24 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 14937 | 4656 | | 179 | 1 | 2024-03-12 13:41:17.49 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 14937 | 4656 | | 177 | 1 | 2024-03-12 13:41:17.33 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 14937 | 4656 | | 178 | 3 | 2024-03-12 13:41:17.477 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs |
(60 rows)

68
data/vehicle-etl/GMC.txt
View File

@@ -1,68 +0,0 @@
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
-----------------------------------------------------+-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+----------------+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+----------------------------------+--------------
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
| Additional Error Text | The Model Year decoded for this VIN may be incorrect. If you know the Model year, please enter it and decode again to get more accurate information. | | | | 156 | The Model Year decoded for this VIN may be incorrect. If you know the Model year, please enter it and decode again to get more accurate information. | | | AdditionalErrorText | string | Decoding | Corrections |
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
| Vehicle Descriptor | 3GTUUFEL*PG | | | | 196 | 3GTUUFEL*PG | | | VehicleDescriptor | string | Decoding | Corrections |
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
General | Manufacturer Name | GENERAL MOTORS LLC | | | 3GT | 27 | 984 | | 2066 | Manufacturer | lookup | WMI | Manu. Name |
General | Model Year | 2023 | | | ***X*|Y | 29 | 2023 | | | ModelYear | int | WMI, Input | ModelYear |
General | Make | GMC | 1947758 | 25646 | [NPRUV][HU] | 26 | 472 | | | Make | lookup | WMI, Pattern | pattern - model |
General | Plant City | SILAO | 1947761 | 25646 | *****|*G | 31 | SILAO | 2023-07-11 14:39:30.503 | 2066 | PlantCity | string | Pattern | Pattern |
General | Series | 1500 | 1947762 | 25646 | [NPRUV][HU] | 34 | 1500 | 2023-07-11 14:39:30.503 | 2066 | Series | string | Pattern | Pattern |
General | Trim | AT4X | 1947769 | 25646 | *UF | 38 | AT4X | 2023-07-11 14:39:30.503 | 2066 | Trim | string | Pattern | Pattern |
General | Plant Country | MEXICO | 1947783 | 25646 | *****|*G | 75 | 12 | 2023-07-11 14:39:30.503 | 2066 | PlantCountry | lookup | Pattern | Pattern |
General | Plant State | GUANAJUATO | 1947786 | 25646 | *****|*G | 77 | GUANAJUATO | 2023-07-11 14:39:30.503 | 2066 | PlantState | string | Pattern | Pattern |
General | Vehicle Type | TRUCK | | | 3GT | 39 | 3 | 2015-03-24 15:16:38.563 | 2066 | VehicleType | lookup | WMI | VehType |
General | Model | Sierra | 1947758 | 25646 | [NPRUV][HU] | 28 | 1857 | 2023-07-11 14:46:06.22 | 2066 | Model | lookup | Pattern | Pattern |
Exterior / Body | Body Class | Pickup | 1947732 | 25646 | [NPRUV][HU] | 5 | 60 | 2023-07-11 14:46:06.22 | 2066 | BodyClass | lookup | Pattern | Pattern |
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 2F: 7,001 - 8,000 lb (3,175 - 3,629 kg) | 1947756 | 25646 | [UV][HU] | 25 | 15 | 2023-07-11 14:46:18.457 | 2066 | GVWR | lookup | Pattern | Pattern |
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
Mechanical / Drivetrain | Drive Type | 4WD/4-Wheel Drive/4x4 | 1947745 | 25646 | [NPRUV]U[A-J9] | 15 | 2 | 2023-07-11 14:39:30.503 | 2066 | DriveType | lookup | Pattern | Pattern |
Mechanical / Brake | Brake System Type | Hydraulic | 1947772 | 25646 | [NPRUV][HU] | 42 | 2 | 2023-07-11 14:46:06.22 | 2066 | BrakeSystemType | lookup | Pattern | Pattern |
Engine | Other Engine Info | DI DFM, ALUM, GEN 5 | 1947794 | 25646 | *[HU]**L | 129 | DI DFM, ALUM, GEN 5 | 2023-07-11 14:39:30.503 | 2066 | OtherEngineInfo | string | Pattern | Pattern |
Engine | Displacement (CI) | 378.3472133873401605 | 1947739 | 25646 | *[HU]**L | 12 | 378.3472133873401605 | | 2066 | DisplacementCI | decimal | Pattern | Conversion 7: 6.2 / 0.016387064 |
Engine | Engine Configuration | V-Shaped | 1947780 | 25646 | *[HU]**L | 64 | 2 | 2023-07-11 14:39:30.503 | 2066 | EngineConfiguration | lookup | Pattern | Pattern |
Engine | Fuel Type - Primary | Gasoline | 1947755 | 25646 | *[HU]**L | 24 | 4 | 2023-07-11 14:39:30.503 | 2066 | FuelTypePrimary | lookup | Pattern | Pattern |
Engine | Engine Model | L87 | 1947746 | 25646 | *[HU]**L | 18 | L87 | 2023-07-11 14:39:30.503 | 2066 | EngineModel | string | Pattern | Pattern |
Engine | Displacement (L) | 6.2 | 1947739 | 25646 | *[HU]**L | 13 | 6.2 | 2023-07-11 14:39:30.503 | 2066 | DisplacementL | decimal | Pattern | Pattern |
Engine | Engine Number of Cylinders | 8 | 1947733 | 25646 | *[HU]**L | 9 | 8 | 2023-07-11 14:39:30.503 | 2066 | EngineCylinders | int | Pattern | Pattern |
Engine | Displacement (CC) | 6200.0 | 1947739 | 25646 | *[HU]**L | 11 | 6200.0 | | 2066 | DisplacementCC | decimal | Pattern | Conversion 4: 6.2 * 1000 |
Passive Safety System | Seat Belt Type | Manual | 1947788 | 25646 | *[HU]*E | 79 | 1 | 2023-07-11 14:39:30.503 | 2066 | SeatBeltsAll | lookup | Pattern | Pattern |
Passive Safety System | Other Restraint System Info | AY0 - Active Manual Belts | 1947793 | 25646 | *[HU]*E | 121 | AY0 - Active Manual Belts | 2023-07-11 14:39:30.503 | 2066 | OtherRestraintSystemInfo | string | Pattern | Pattern |
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | All Rows | 1947773 | 25646 | *[HU]*E | 55 | 6 | 2023-07-11 14:39:30.503 | 2066 | AirBagLocCurtain | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 1947781 | 25646 | *[HU]*E | 65 | 3 | 2023-07-11 14:39:30.503 | 2066 | AirBagLocFront | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st Row (Driver and Passenger) | 1947792 | 25646 | *[HU]*E | 107 | 3 | 2023-07-11 14:39:30.503 | 2066 | AirBagLocSide | lookup | Pattern | Pattern |
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Optional | 13609 | 4140 | | 172 | 3 | 2024-01-31 16:04:17.207 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs | f
Active Safety System | Electronic Stability Control (ESC) | Standard | 13609 | 4140 | | 99 | 1 | 2024-01-31 16:04:16.897 | | ESC | lookup | Pattern | Vehicle Specs | f
Active Safety System | Anti-lock Braking System (ABS) | Standard | 13609 | 4140 | | 86 | 1 | 2024-01-31 16:04:16.85 | | ABS | lookup | Pattern | Vehicle Specs | f
Active Safety System | Traction Control | Standard | 13609 | 4140 | | 100 | 1 | 2024-01-31 16:04:16.91 | | TractionControl | lookup | Pattern | Vehicle Specs | f
Active Safety System | Keyless Ignition | Standard | 13609 | 4140 | | 176 | 1 | 2024-01-31 16:04:17.257 | | KeylessIgnition | lookup | Pattern | Vehicle Specs | f
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Direct | 13609 | 4140 | | 168 | 1 | 2024-01-31 16:04:17.157 | | TPMS | lookup | Pattern | Vehicle Specs | f
Active Safety System | Event Data Recorder (EDR) | Standard | 13609 | 4140 | | 175 | 1 | 2024-01-31 16:04:17.24 | | EDR | lookup | Pattern | Vehicle Specs | f
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Optional | 13609 | 4140 | | 81 | 3 | 2024-01-31 16:04:16.837 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs | f
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 13609 | 4140 | | 170 | 1 | 2024-01-31 16:04:17.173 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs | f
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 13609 | 4140 | | 87 | 1 | 2024-01-31 16:04:16.863 | | CIB | lookup | Pattern | Vehicle Specs | f
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 13609 | 4140 | | 171 | 1 | 2024-01-31 16:04:17.19 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs | f
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 13609 | 4140 | | 101 | 1 | 2024-01-31 16:04:16.923 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Standard | 13609 | 4140 | | 103 | 1 | 2024-01-31 16:04:17.103 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 13609 | 4140 | | 102 | 1 | 2024-01-31 16:04:16.94 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lane and Side Assist | Blind Spot Warning (BSW) | Optional | 13609 | 4140 | | 88 | 3 | 2024-01-31 16:04:16.88 | | BlindSpotMon | lookup | Pattern | Vehicle Specs | f
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Optional | 13609 | 4140 | | 192 | 3 | 2024-01-31 16:04:17.487 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs | f
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 13609 | 4140 | | 104 | 1 | 2024-01-31 16:04:17.12 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs | f
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Optional | 13609 | 4140 | | 183 | 2 | 2024-01-31 16:04:17.47 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs | f
Active Safety System / Backing Up and Parking | Parking Assist | Optional | 13609 | 4140 | | 105 | 3 | 2024-01-31 16:04:17.14 | | ParkAssist | lookup | Pattern | Vehicle Specs | f
Active Safety System / 911 Notification | Automatic Crash Notification (ACN) / Advanced Automatic Crash Notification (AACN) | Standard | 13609 | 4140 | | 174 | 1 | 2024-01-31 16:04:17.22 | | CAN_AACN | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 13609 | 4140 | | 178 | 3 | 2024-01-31 16:04:17.427 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lighting Technologies | Adaptive Driving Beam (ADB) | Standard | 13609 | 4140 | | 180 | 1 | 2024-01-31 16:04:17.453 | | AdaptiveDrivingBeam | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 13609 | 4140 | | 177 | 1 | 2024-01-31 16:04:17.273 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs | f
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 13609 | 4140 | | 179 | 1 | 2024-01-31 16:04:17.44 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs | f
(64 rows)

79
data/vehicle-etl/HONDA.txt
View File

@@ -1,79 +0,0 @@
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
-----------------------------------------------------+--------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+----------+---------------+----------------------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+--------------------------------+--------------
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
| Additional Error Text | | | | | 156 | | | | AdditionalErrorText | string | Decoding | Corrections |
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
| Vehicle Descriptor | 2HGFE4F8*SH | | | | 196 | 2HGFE4F8*SH | | | VehicleDescriptor | string | Decoding | Corrections |
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
General | Plant City | ALLISTON | 2123543 | 27387 | *****|*H | 31 | ALLISTON | 2024-09-16 08:37:51.523 | 2096 | PlantCity | string | Pattern | Pattern |
General | Trim | Sport Hybrid / Sport Touring Hybrid | 2086003 | 27007 | FE4F8 | 38 | Sport Hybrid / Sport Touring Hybrid | 2024-06-17 10:36:46.323 | 2096 | Trim | string | Pattern | Pattern |
General | Plant Country | CANADA | 2123544 | 27387 | *****|*H | 75 | 1 | 2024-09-16 08:37:51.54 | 2096 | PlantCountry | lookup | Pattern | Pattern |
General | Plant State | ONTARIO | 2123545 | 27387 | *****|*H | 77 | ONTARIO | 2024-09-16 08:37:51.553 | 2096 | PlantState | string | Pattern | Pattern |
General | Model Year | 2025 | | | ***X*|Y | 29 | 2025 | | | ModelYear | int | WMI, Input | ModelYear |
General | Manufacturer Name | HONDA OF CANADA MFG., A DIVISION OF HONDA CANADA INC. | | | 2HG | 27 | 990 | | 2096 | Manufacturer | lookup | WMI | Manu. Name |
General | Make | HONDA | 2086001 | 27007 | FE4F8 | 26 | 474 | | | Make | lookup | WMI, Pattern | pattern - model |
General | Vehicle Type | PASSENGER CAR | | | 2HG | 39 | 2 | 2015-03-26 16:57:39.147 | 2096 | VehicleType | lookup | WMI | VehType |
General | Model | Civic | 2086001 | 27007 | FE4F8 | 28 | 1863 | 2024-06-17 10:36:46.287 | 2096 | Model | lookup | Pattern | Pattern |
Exterior / Body | Doors | 4 | 2085995 | 27007 | FE4F8 | 14 | 4 | 2024-06-17 10:36:46.163 | 2096 | Doors | int | Pattern | Pattern |
Exterior / Body | Body Class | Sedan/Saloon | 2085992 | 27007 | FE4F8 | 5 | 13 | 2024-06-17 10:36:45.88 | 2096 | BodyClass | lookup | Pattern | Pattern |
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 1C: 4,001 - 5,000 lb (1,814 - 2,268 kg) | 2086000 | 27007 | FE4F8 | 25 | 12 | 2024-06-17 10:36:46.267 | 2096 | GVWR | lookup | Pattern | Pattern |
Exterior / Dimension | Gross Vehicle Weight Rating To | Class 1C: 4,001 - 5,000 lb (1,814 - 2,268 kg) | 2086015 | 27007 | FE4F8 | 190 | 12 | 2024-06-17 10:36:46.713 | 2096 | GVWR_to | lookup | Pattern | Pattern |
Exterior / Truck | Bed Type | Not Applicable | | | | 3 | 0 | 2019-12-21 20:26:30.583 | | BedType | lookup | Pattern | Default |
Exterior / Truck | Cab Type | Not Applicable | | | | 4 | 0 | 2019-12-21 20:26:30.583 | | BodyCabType | lookup | Pattern | Default |
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
Mechanical / Transmission | Transmission Style | Electronic Continuously Variable (e-CVT) | 2086002 | 27007 | FE4F8 | 37 | 4 | 2024-06-17 10:36:46.307 | 2096 | TransmissionStyle | lookup | Pattern | Pattern |
Mechanical / Drivetrain | Drive Type | 4x2 | 2085996 | 27007 | FE4F8 | 15 | 7 | 2024-06-17 10:36:46.18 | 2096 | DriveType | lookup | Pattern | Pattern |
Mechanical / Battery | EV Drive Unit | Single Motor | 2086065 | 27007 | FE4F8 | 72 | 2 | 2024-06-17 10:59:49.297 | 2096 | EVDriveUnit | lookup | Pattern | Pattern |
Engine | Engine Manufacturer | Honda | 2086014 | 27007 | FE4F8 | 146 | Honda | 2024-06-17 10:36:46.69 | 2096 | EngineManufacturer | string | Pattern | Pattern |
Engine | Displacement (CC) | 2000 | 2085994 | 27007 | FE4F8 | 11 | 2000 | | 2096 | DisplacementCC | decimal | Pattern | Conversion 4: 2 * 1000 |
Engine | Displacement (CI) | 122.0474881894645679 | 2085994 | 27007 | FE4F8 | 12 | 122.0474881894645679 | | 2096 | DisplacementCI | decimal | Pattern | Conversion 7: 2 / 0.016387064 |
Engine | Engine Number of Cylinders | 4 | 2085993 | 27007 | FE4F8 | 9 | 4 | 2024-06-17 10:36:45.96 | 2096 | EngineCylinders | int | Pattern | Pattern |
Engine | Displacement (L) | 2 | 2085994 | 27007 | FE4F8 | 13 | 2 | 2024-06-17 10:36:46.023 | 2096 | DisplacementL | decimal | Pattern | Pattern |
Engine | Engine Stroke Cycles | 4 | 2085997 | 27007 | FE4F8 | 17 | 4 | 2024-06-17 10:36:46.203 | 2096 | EngineCycles | int | Pattern | Pattern |
Engine | Engine Model | LFC3 | 2085998 | 27007 | FE4F8 | 18 | LFC3 | 2024-06-17 10:36:46.223 | 2096 | EngineModel | string | Pattern | Pattern |
Engine | Fuel Type - Primary | Gasoline | 2085999 | 27007 | FE4F8 | 24 | 4 | 2024-06-17 10:36:46.247 | 2096 | FuelTypePrimary | lookup | Pattern | Pattern |
Engine | Valve Train Design | Dual Overhead Cam (DOHC) | 2086004 | 27007 | FE4F8 | 62 | 2 | 2024-06-17 10:36:46.343 | 2096 | ValveTrainDesign | lookup | Pattern | Pattern |
Engine | Engine Configuration | In-Line | 2086005 | 27007 | FE4F8 | 64 | 1 | 2024-06-17 10:36:46.363 | 2096 | EngineConfiguration | lookup | Pattern | Pattern |
Engine | Fuel Type - Secondary | Electric | 2086006 | 27007 | FE4F8 | 66 | 2 | 2024-06-17 10:36:46.527 | 2096 | FuelTypeSecondary | lookup | Pattern | Pattern |
Engine | Engine Brake (hp) From | 141 | 2086007 | 27007 | FE4F8 | 71 | 141 | 2024-06-17 10:36:46.543 | 2096 | EngineHP | decimal | Pattern | Pattern |
Engine | Cooling Type | Water | 2086011 | 27007 | FE4F8 | 122 | 2 | 2024-06-17 10:36:46.63 | 2096 | CoolingType | lookup | Pattern | Pattern |
Engine | Electrification Level | Strong HEV (Hybrid Electric Vehicle) | 2086012 | 27007 | FE4F8 | 126 | 2 | 2024-06-17 10:36:46.65 | 2096 | ElectrificationLevel | lookup | Pattern | Pattern |
Engine | Other Engine Info | Direct Fuel Injection / Motor: 135kW | 2086013 | 27007 | FE4F8 | 129 | Direct Fuel Injection / Motor: 135kW | 2024-06-17 10:36:46.67 | 2096 | OtherEngineInfo | string | Pattern | Pattern |
Passive Safety System | Other Restraint System Info | Front: Seat Belt / Rear: Seat Belt , Side Air Bag and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2083833 | 27007 | FE[24]F | 121 | Front: Seat Belt / Rear: Seat Belt , Side Air Bag and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2024-06-17 10:20:27.853 | 2096 | OtherRestraintSystemInfo | string | Pattern | Pattern |
Passive Safety System | Seat Belt Type | Manual | 2083828 | 27007 | FE[24]F | 79 | 1 | 2024-06-17 10:20:27.853 | 2096 | SeatBeltsAll | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st and 2nd Rows | 2083832 | 27007 | FE[24]F | 107 | 4 | 2024-06-17 10:20:27.853 | 2096 | AirBagLocSide | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Knee Air Bag Locations | 1st Row (Driver and Passenger) | 2083826 | 27007 | FE[24]F | 69 | 3 | 2024-06-17 10:05:30.837 | 2096 | AirBagLocKnee | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 2083825 | 27007 | FE[24]F | 65 | 3 | 2024-06-17 10:05:30.837 | 2096 | AirBagLocFront | lookup | Pattern | Pattern |
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | 1st and 2nd Rows | 2083822 | 27007 | FE[24]F | 55 | 4 | 2024-06-17 10:05:30.837 | 2096 | AirBagLocCurtain | lookup | Pattern | Pattern |
Active Safety System | Anti-lock Braking System (ABS) | Standard | 16871 | 5597 | | 86 | 1 | 2025-04-03 14:40:02.47 | | ABS | lookup | Pattern | Vehicle Specs |
Active Safety System | Traction Control | Standard | 16871 | 5597 | | 100 | 1 | 2025-04-03 14:40:02.52 | | TractionControl | lookup | Pattern | Vehicle Specs |
Active Safety System | Automatic Pedestrian Alerting Sound (for Hybrid and EV only) | Standard | 17451 | 5597 | | 173 | 1 | 2025-04-01 10:29:03.217 | | AutomaticPedestrianAlertingSound | lookup | Pattern | Vehicle Specs |
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Indirect | 16871 | 5597 | | 168 | 2 | 2025-04-03 14:40:02.6 | | TPMS | lookup | Pattern | Vehicle Specs |
Active Safety System | Keyless Ignition | Standard | 16871 | 5597 | | 176 | 1 | 2025-04-03 14:40:02.793 | | KeylessIgnition | lookup | Pattern | Vehicle Specs |
Active Safety System | Electronic Stability Control (ESC) | Standard | 16871 | 5597 | | 99 | 1 | 2025-04-03 14:40:02.503 | | ESC | lookup | Pattern | Vehicle Specs |
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Standard | 16871 | 5597 | | 172 | 1 | 2025-04-03 14:40:02.763 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs |
Active Safety System | Event Data Recorder (EDR) | Standard | 16871 | 5597 | | 175 | 1 | 2025-04-03 14:40:02.78 | | EDR | lookup | Pattern | Vehicle Specs |
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Standard | 16871 | 5597 | | 81 | 1 | 2025-04-03 14:40:02.337 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 16871 | 5597 | | 101 | 1 | 2025-04-03 14:40:02.537 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 16871 | 5597 | | 170 | 1 | 2025-04-03 14:40:02.61 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 16871 | 5597 | | 87 | 1 | 2025-04-03 14:40:02.487 | | CIB | lookup | Pattern | Vehicle Specs |
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 16871 | 5597 | | 171 | 1 | 2025-04-03 14:40:02.627 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Standard | 16871 | 5597 | | 103 | 1 | 2025-04-03 14:40:02.567 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Blind Spot Intervention (BSI) | Standard | 17451 | 5597 | | 193 | 1 | 2025-04-01 10:29:03.243 | | BlindSpotIntervention | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Blind Spot Warning (BSW) | Standard | 17451 | 5597 | | 88 | 1 | 2025-04-01 10:29:03.203 | | BlindSpotMon | lookup | Pattern | Vehicle Specs |
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 16871 | 5597 | | 102 | 1 | 2025-04-03 14:40:02.55 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 16871 | 5597 | | 104 | 1 | 2025-04-03 14:40:02.583 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Standard | 16871 | 5597 | | 192 | 1 | 2025-04-03 14:40:02.877 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Standard | 17451 | 5597 | | 183 | 1 | 2025-04-01 10:29:03.23 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 16871 | 5597 | | 179 | 1 | 2025-04-03 14:40:02.843 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 16871 | 5597 | | 177 | 1 | 2025-04-03 14:40:02.81 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs |
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 16871 | 5597 | | 178 | 3 | 2025-04-03 14:40:02.83 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs |
(75 rows)

BIN
data/vehicle-etl/__pycache__/etl_generate_sql.cpython-314.pyc
View File

Binary file not shown.

BIN
data/vehicle-etl/__pycache__/nhtsa_fetch.cpython-314.pyc
View File

Binary file not shown.

BIN
data/vehicle-etl/__pycache__/vehapi_fetch_snapshot.cpython-314.pyc
View File

Binary file not shown.

241
data/vehicle-etl/etl_generate_sql.py
View File

@@ -1,241 +0,0 @@
#!/usr/bin/env python3
"""
Generate SQL import files from a VehAPI snapshot SQLite database.
Reads observed compatibility pairs from the snapshot (trim-filtered engine<->transmission pairs)
and produces:
- output/01_engines.sql
- output/02_transmissions.sql
- output/03_vehicle_options.sql
No legacy JSON or network calls are used. The snapshot path is provided via CLI flag.
"""
import argparse
import os
import sqlite3
from pathlib import Path
from typing import Dict, Iterable, List, Sequence
BATCH_SIZE = 1000
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Generate SQL files from a VehAPI snapshot (SQLite).",
)
parser.add_argument(
"--snapshot-path",
type=Path,
default=os.environ.get("SNAPSHOT_PATH"),
help="Path to snapshots/<date>/snapshot.sqlite produced by vehapi_fetch_snapshot.py (or env SNAPSHOT_PATH)",
)
parser.add_argument(
"--output-dir",
type=Path,
default=Path("output"),
help="Directory to write SQL output files (default: output)",
)
return parser.parse_args()
def load_pairs(snapshot_path: Path) -> List[sqlite3.Row]:
if not snapshot_path.exists():
raise FileNotFoundError(f"Snapshot not found: {snapshot_path}")
# Open in immutable mode to prevent any write attempts on read-only filesystems
absolute_path = snapshot_path.resolve()
uri = f"file:{absolute_path}?immutable=1"
conn = sqlite3.connect(uri, uri=True)
conn.row_factory = sqlite3.Row
try:
cursor = conn.execute(
"""
SELECT
year,
make,
model,
trim,
engine_display,
engine_canon,
engine_bucket,
trans_display,
trans_canon,
trans_bucket
FROM pairs
ORDER BY year, make, model, trim, engine_canon, trans_canon
"""
)
rows = cursor.fetchall()
except sqlite3.Error as exc:
raise RuntimeError(f"Failed to read pairs from snapshot: {exc}") from exc
finally:
conn.close()
if not rows:
raise ValueError("Snapshot contains no rows in pairs table.")
return rows
def choose_engine_label(engine_display: str, engine_bucket: str, engine_canon: str) -> str:
"""
Use VehAPI display string when present, otherwise fall back to the bucket label,
and finally to the canonical key to avoid empty names.
"""
if engine_display:
return engine_display
if engine_bucket:
return engine_bucket
return engine_canon
def choose_trans_label(trans_display: str, trans_bucket: str, trans_canon: str) -> str:
if trans_display:
return trans_display
if trans_bucket:
return trans_bucket
return trans_canon
def build_engine_dimension(rows: Sequence[sqlite3.Row]) -> Dict[str, Dict]:
engines: Dict[str, Dict] = {}
for row in rows:
canon = row["engine_canon"]
if canon is None or canon == "":
raise ValueError(f"Missing engine_canon for row: {dict(row)}")
if canon in engines:
continue
engines[canon] = {
"id": len(engines) + 1,
"name": choose_engine_label(row["engine_display"], row["engine_bucket"], canon),
"fuel_type": row["engine_bucket"] or None,
}
return engines
def build_transmission_dimension(rows: Sequence[sqlite3.Row]) -> Dict[str, Dict]:
transmissions: Dict[str, Dict] = {}
for row in rows:
canon = row["trans_canon"]
if canon is None or canon == "":
raise ValueError(f"Missing trans_canon for row: {dict(row)}")
if canon in transmissions:
continue
transmissions[canon] = {
"id": len(transmissions) + 1,
"type": choose_trans_label(row["trans_display"], row["trans_bucket"], canon),
}
return transmissions
def build_vehicle_options(
rows: Sequence[sqlite3.Row],
engine_map: Dict[str, Dict],
trans_map: Dict[str, Dict],
) -> List[Dict]:
options: List[Dict] = []
for row in rows:
engine_canon = row["engine_canon"]
trans_canon = row["trans_canon"]
options.append(
{
"year": int(row["year"]),
"make": row["make"],
"model": row["model"],
"trim": row["trim"],
"engine_id": engine_map[engine_canon]["id"],
"transmission_id": trans_map[trans_canon]["id"],
}
)
return options
def sql_value(value):
if value is None:
return "NULL"
if isinstance(value, str):
return "'" + value.replace("'", "''") + "'"
return str(value)
def chunked(seq: Iterable[Dict], size: int) -> Iterable[List[Dict]]:
chunk: List[Dict] = []
for item in seq:
chunk.append(item)
if len(chunk) >= size:
yield chunk
chunk = []
if chunk:
yield chunk
def write_insert_file(
path: Path,
table: str,
columns: Sequence[str],
rows: Sequence[Dict],
):
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
f.write(f"-- Auto-generated by etl_generate_sql.py\n")
if not rows:
f.write(f"-- No rows for {table}\n")
return
for batch in chunked(rows, BATCH_SIZE):
values_sql = ",\n".join(
"(" + ",".join(sql_value(row[col]) for col in columns) + ")"
for row in batch
)
f.write(f"INSERT INTO {table} ({', '.join(columns)}) VALUES\n{values_sql};\n\n")
def main():
args = parse_args()
snapshot_path: Path = args.snapshot_path
output_dir: Path = args.output_dir
if snapshot_path is None:
raise SystemExit("Snapshot path is required. Pass --snapshot-path or set SNAPSHOT_PATH.")
print(f"Reading snapshot: {snapshot_path}")
rows = load_pairs(snapshot_path)
years = sorted({int(row["year"]) for row in rows})
print(f" Loaded {len(rows):,} observed engine<->transmission pairs across {len(years)} years")
engines = build_engine_dimension(rows)
transmissions = build_transmission_dimension(rows)
vehicle_options = build_vehicle_options(rows, engines, transmissions)
print(f"Engines: {len(engines):,}")
print(f"Transmissions: {len(transmissions):,}")
print(f"Vehicle options (observed pairs): {len(vehicle_options):,}")
write_insert_file(
output_dir / "01_engines.sql",
"engines",
["id", "name", "fuel_type"],
engines.values(),
)
write_insert_file(
output_dir / "02_transmissions.sql",
"transmissions",
["id", "type"],
transmissions.values(),
)
write_insert_file(
output_dir / "03_vehicle_options.sql",
"vehicle_options",
["year", "make", "model", "trim", "engine_id", "transmission_id"],
vehicle_options,
)
print("\nSQL files generated:")
print(f" - {output_dir / '01_engines.sql'}")
print(f" - {output_dir / '02_transmissions.sql'}")
print(f" - {output_dir / '03_vehicle_options.sql'}")
print(f"\nYear coverage: {years[0]}-{years[-1]}")
if __name__ == "__main__":
main()

0
data/vehicle-etl/export_from_postgres.py → data/vehicle-etl/export_database.py
View File

238374
data/vehicle-etl/import/vehicle-catalog-master.csv Normal file
View File

File diff suppressed because it is too large Load Diff

117
data/vehicle-etl/import_catalog.sh Executable file
View File

@@ -0,0 +1,117 @@
#!/bin/bash
#
# Vehicle Catalog CSV Bulk Import Wrapper
#
# Copies CSV file into mvp-backend container and executes bulk import script.
# Handles large CSV files (250k+ rows) that fail in web import.
#
# Usage:
# ./import_catalog.sh <path_to_csv_file>
#
# Example:
# ./import_catalog.sh data/vehicle-etl/import/vehicle-catalog-master.csv
#
# Requirements:
# - mvp-backend container must be running
# - CSV file must have headers: year, make, model, trim
# - Optional headers: engine_name, transmission_type
#
set -euo pipefail
CONTAINER="mvp-backend"
TEMP_CSV_PATH="/tmp/catalog-import.csv"
SCRIPT_PATH="dist/features/admin/scripts/bulk-import-catalog.js"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Print error and exit
error() {
echo -e "${RED}Error: $1${NC}" >&2
exit 1
}
# Print success message
success() {
echo -e "${GREEN}$1${NC}"
}
# Print warning message
warn() {
echo -e "${YELLOW}$1${NC}"
}
# Check if CSV file argument provided
if [ $# -eq 0 ]; then
error "No CSV file specified.
Usage: $0 <path_to_csv_file>
Example:
$0 data/vehicle-etl/import/vehicle-catalog-master.csv"
fi
CSV_FILE="$1"
# Validate CSV file exists
if [ ! -f "$CSV_FILE" ]; then
error "CSV file not found: $CSV_FILE"
fi
# Get absolute path to CSV file
CSV_FILE_ABS=$(cd "$(dirname "$CSV_FILE")" && pwd)/$(basename "$CSV_FILE")
# Check if container is running
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER}$"; then
error "Container '${CONTAINER}' is not running. Start it with: make start"
fi
echo "=========================================="
echo "Vehicle Catalog Bulk Import"
echo "=========================================="
echo "CSV File: $CSV_FILE_ABS"
echo "Container: $CONTAINER"
echo ""
# Copy CSV file into container
echo "Step 1: Copying CSV file into container..."
if ! docker cp "$CSV_FILE_ABS" "${CONTAINER}:${TEMP_CSV_PATH}"; then
error "Failed to copy CSV file into container"
fi
success "CSV file copied successfully"
echo ""
# Execute import script inside container
echo "Step 2: Running import script..."
echo ""
if docker exec -it "$CONTAINER" node "$SCRIPT_PATH"; then
success "Import completed successfully!"
IMPORT_SUCCESS=true
else
error "Import failed. Check the logs above for details."
IMPORT_SUCCESS=false
fi
# Cleanup: Remove temp CSV file from container
echo ""
echo "Step 3: Cleaning up..."
if docker exec "$CONTAINER" rm -f "$TEMP_CSV_PATH" 2>/dev/null; then
success "Temporary files cleaned up"
else
warn "Warning: Failed to cleanup temp CSV file in container"
fi
echo ""
if [ "$IMPORT_SUCCESS" = true ]; then
echo "=========================================="
success "Import process completed successfully!"
echo "=========================================="
exit 0
else
exit 1
fi

0
data/vehicle-etl/import_data.sh → data/vehicle-etl/import_database.sh
View File

514
data/vehicle-etl/logical-plotting-hartmanis.md
View File

@@ -1,514 +0,0 @@
# vPIC ETL Implementation Plan v2
## Overview
Extract vehicle dropdown data from NHTSA vPIC database for MY2022+ to supplement existing VehAPI data. This revised plan uses a make-specific extraction approach with proper VIN schema parsing.
## Key Changes from v1
1. **Limit to VehAPI makes only** - Only extract the 48 makes that exist in VehAPI data
2. **VIN schema-based extraction** - Extract directly from VIN patterns, not defs_model
3. **Proper field formatting** - Match VehAPI display string formats
4. **Make-specific logic** - Handle different manufacturers' data patterns
## Critical Discovery: WMI Linkage
**Must use `wmi_make` junction table (many-to-many), NOT `wmi.makeid` (one-to-many):**
```sql
-- CORRECT: via wmi_make (finds all makes including Toyota, Hyundai, etc.)
FROM vpic.make m
JOIN vpic.wmi_make wm ON wm.makeid = m.id
JOIN vpic.wmi w ON w.id = wm.wmiid
-- WRONG: via wmi.makeid (misses many major brands)
FROM vpic.make m
JOIN vpic.wmi w ON w.makeid = m.id
```
---
## Make Availability Summary
| Status | Count | Makes |
|--------|-------|-------|
| **Available (2022+ schemas)** | 46 | See table below |
| **No 2022+ data** | 2 | Hummer (discontinued 2010), Scion (discontinued 2016) |
---
## Per-Make Analysis
### Group 1: Japanese Manufacturers (Honda/Acura, Toyota/Lexus, Nissan/Infiniti)
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Acura | Acura | Acura | 48 | Ready |
| Honda | Honda | Honda | 238 | Ready |
| Lexus | Lexus | Lexus | 90 | Ready |
| Toyota | Toyota | Toyota | 152 | Ready |
| Infiniti | INFINITI | Infiniti | 76 | Ready |
| Nissan | Nissan | Nissan | 85 | Ready |
| Mazda | Mazda | Mazda | 37 | Ready |
| Mitsubishi | Mitsubishi | Mitsubishi | 11 | Ready |
| Subaru | Subaru | Subaru | 75 | Ready |
| Isuzu | Isuzu | Isuzu | 11 | Ready |
### Group 2: Korean Manufacturers (Hyundai/Kia/Genesis)
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Genesis | Genesis | Genesis | 74 | Ready |
| Hyundai | Hyundai | Hyundai | 177 | Ready |
| Kia | Kia | Kia | 72 | Ready |
### Group 3: American - GM (Chevrolet, GMC, Buick, Cadillac)
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Buick | Buick | Buick | 20 | Ready |
| Cadillac | Cadillac | Cadillac | 50 | Ready |
| Chevrolet | Chevrolet | Chevrolet | 185 | Ready |
| GMC | GMC | GMC | 107 | Ready |
| Oldsmobile | Oldsmobile | Oldsmobile | 1 | Limited |
| Pontiac | Pontiac | Pontiac | 5 | Limited (2022-2024) |
### Group 4: American - Stellantis (Chrysler, Dodge, Jeep, Ram, Fiat)
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Chrysler | Chrysler | Chrysler | 81 | Ready |
| Dodge | Dodge | Dodge | 86 | Ready |
| FIAT | FIAT | Fiat | 91 | Ready (case diff) |
| Jeep | Jeep | Jeep | 81 | Ready |
| RAM | RAM | Ram | 81 | Ready (case diff) |
| Plymouth | Plymouth | Plymouth | 4 | Limited |
### Group 5: American - Ford
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Ford | Ford | Ford | 108 | Ready |
| Lincoln | Lincoln | Lincoln | 21 | Ready |
| Mercury | Mercury | Mercury | 0 | No data (discontinued 2011) |
### Group 6: American - EV Startups
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Polestar | Polestar | Polestar | 12 | Ready |
| Rivian | Rivian | RIVIAN | 10 | Ready (case diff) |
| Tesla | Tesla | Tesla | 14 | Ready |
### Group 7: German Manufacturers
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Audi | Audi | Audi | 55 | Ready |
| BMW | BMW | BMW | 61 | Ready |
| Mercedes-Benz | Mercedes-Benz | Mercedes-Benz | 39 | Ready |
| MINI | MINI | MINI | 10 | Ready |
| Porsche | Porsche | Porsche | 23 | Ready |
| smart | smart | smart | 5 | Ready |
| Volkswagen | Volkswagen | Volkswagen | 134 | Ready |
### Group 8: European Luxury
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|------|-------------|-----------|-----------------|--------|
| Bentley | Bentley | Bentley | 48 | Ready |
| Ferrari | Ferrari | Ferrari | 9 | Ready |
| Jaguar | Jaguar | Jaguar | 17 | Ready |
| Lamborghini | Lamborghini | Lamborghini | 10 | Ready |
| Lotus | Lotus | Lotus | 5 | Ready |
| Maserati | Maserati | Maserati | 19 | Ready |
| McLaren | McLaren | McLaren | 4 | Ready |
| Volvo | Volvo | Volvo | 80 | Ready |
### Group 9: Discontinued (No 2022+ Data)
| Make | VehAPI Name | Reason | Action |
|------|-------------|--------|--------|
| Hummer | Hummer | Discontinued 2010 (new EV under GMC) | Skip - use existing VehAPI |
| Scion | Scion | Discontinued 2016 | Skip - use existing VehAPI |
| Saab | Saab | Discontinued 2012 | Limited schemas (9) |
| Mercury | Mercury | Discontinued 2011 | No schemas |
---
## Extraction Architecture
### Data Flow
```
vPIC VIN Schemas → Pattern Extraction → Format Transformation → SQLite Pairs
↓
Filter by:
- 48 VehAPI makes only
- Year >= 2022
- Vehicle types (exclude motorcycles, trailers, buses)
```
### Core Query Strategy
For each allowed make:
1. Find WMIs linked to that make
2. Get VIN schemas for years 2022+
3. Extract from patterns:
- Model (from schema name or pattern)
- Trim (Element: Trim)
- Displacement (Element: DisplacementL)
- Horsepower (Element: EngineHP)
- Cylinders (Element: EngineCylinders)
- Engine Config (Element: EngineConfiguration)
- Transmission Style (Element: TransmissionStyle)
- Transmission Speeds (Element: TransmissionSpeeds)
---
## Acura Extraction Template
This pattern applies to Honda/Acura and similar well-structured manufacturers.
### Sample VIN Schema: Acura MDX 2025 (schema_id: 26929)
| Element | Code | Values |
|---------|------|--------|
| Trim | Trim | MDX, Technology, SH-AWD, SH-AWD Technology, SH-AWD A-Spec, SH-AWD Advance, SH-AWD A-Spec Advance, SH-AWD TYPE S ADVANCE |
| Displacement | DisplacementL | 3.5, 3.0 |
| Horsepower | EngineHP | 290, 355 |
| Cylinders | EngineCylinders | 6 |
| Engine Config | EngineConfiguration | V-Shaped |
| Trans Style | TransmissionStyle | Automatic |
| Trans Speeds | TransmissionSpeeds | 10 |
### Output Format
**Engine Display** (match VehAPI):
```
{DisplacementL}L {EngineHP} hp V{EngineCylinders}
→ "3.5L 290 hp V6"
```
**Transmission Display** (match VehAPI):
```
{TransmissionSpeeds}-Speed {TransmissionStyle}
→ "10-Speed Automatic"
```
### Extraction SQL Template
```sql
WITH schema_data AS (
SELECT DISTINCT
vs.id AS schema_id,
vs.name AS schema_name,
wvs.yearfrom,
COALESCE(wvs.yearto, 2027) AS yearto,
m.name AS make_name
FROM vpic.wmi w
JOIN vpic.make m ON w.makeid = m.id
JOIN vpic.wmi_vinschema wvs ON w.id = wvs.wmiid
JOIN vpic.vinschema vs ON wvs.vinschemaid = vs.id
WHERE LOWER(m.name) IN ('acura', 'honda', ...) -- VehAPI makes
AND wvs.yearfrom >= 2022 OR (wvs.yearto >= 2022)
),
trim_data AS (
SELECT DISTINCT sd.schema_id, p.attributeid AS trim
FROM schema_data sd
JOIN vpic.pattern p ON p.vinschemaid = sd.schema_id
JOIN vpic.element e ON p.elementid = e.id
WHERE e.code = 'Trim'
),
engine_data AS (
SELECT DISTINCT
sd.schema_id,
MAX(CASE WHEN e.code = 'DisplacementL' THEN p.attributeid END) AS displacement,
MAX(CASE WHEN e.code = 'EngineHP' THEN p.attributeid END) AS hp,
MAX(CASE WHEN e.code = 'EngineCylinders' THEN p.attributeid END) AS cylinders,
MAX(CASE WHEN e.code = 'EngineConfiguration' THEN ec.name END) AS config
FROM schema_data sd
JOIN vpic.pattern p ON p.vinschemaid = sd.schema_id
JOIN vpic.element e ON p.elementid = e.id
LEFT JOIN vpic.engineconfiguration ec ON e.code = 'EngineConfiguration'
AND p.attributeid ~ '^[0-9]+$' AND ec.id = CAST(p.attributeid AS INT)
WHERE e.code IN ('DisplacementL', 'EngineHP', 'EngineCylinders', 'EngineConfiguration')
GROUP BY sd.schema_id, p.keys -- Group by VIN pattern position
),
trans_data AS (
SELECT DISTINCT
sd.schema_id,
t.name AS style,
MAX(CASE WHEN e.code = 'TransmissionSpeeds' THEN p.attributeid END) AS speeds
FROM schema_data sd
JOIN vpic.pattern p ON p.vinschemaid = sd.schema_id
JOIN vpic.element e ON p.elementid = e.id
LEFT JOIN vpic.transmission t ON e.code = 'TransmissionStyle'
AND p.attributeid ~ '^[0-9]+$' AND t.id = CAST(p.attributeid AS INT)
WHERE e.code IN ('TransmissionStyle', 'TransmissionSpeeds')
GROUP BY sd.schema_id, t.name
)
SELECT ...
```
---
## Allowed Makes (48 from VehAPI)
```python
ALLOWED_MAKES = [
'Acura', 'Audi', 'Bentley', 'BMW', 'Buick', 'Cadillac', 'Chevrolet',
'Chrysler', 'Dodge', 'Ferrari', 'FIAT', 'Ford', 'Genesis', 'GMC',
'Honda', 'Hummer', 'Hyundai', 'INFINITI', 'Isuzu', 'Jaguar', 'Jeep',
'Kia', 'Lamborghini', 'Lexus', 'Lincoln', 'Lotus', 'Maserati', 'Mazda',
'McLaren', 'Mercedes-Benz', 'Mercury', 'MINI', 'Mitsubishi', 'Nissan',
'Oldsmobile', 'Plymouth', 'Polestar', 'Pontiac', 'Porsche', 'RAM',
'Rivian', 'Saab', 'Scion', 'smart', 'Subaru', 'Tesla', 'Toyota',
'Volkswagen', 'Volvo'
]
```
Note: Some makes may have different names in vPIC (case variations, abbreviations).
---
## Implementation Steps
### Phase 1: Rewrite vpic_extract.py
**File:** `vpic_extract.py`
Core extraction query (uses wmi_make junction table):
```sql
WITH base AS (
SELECT DISTINCT
m.name AS make_name,
vs.id AS schema_id,
vs.name AS schema_name,
generate_series(
GREATEST(wvs.yearfrom, 2022),
COALESCE(wvs.yearto, EXTRACT(YEAR FROM NOW()) + 2)
)::INT AS year
FROM vpic.make m
JOIN vpic.wmi_make wm ON wm.makeid = m.id
JOIN vpic.wmi w ON w.id = wm.wmiid
JOIN vpic.wmi_vinschema wvs ON w.id = wvs.wmiid
JOIN vpic.vinschema vs ON wvs.vinschemaid = vs.id
WHERE LOWER(m.name) IN ({allowed_makes})
AND (wvs.yearfrom >= 2022 OR wvs.yearto >= 2022)
)
SELECT ...
```
**Key functions to implement:**
1. `extract_model_from_schema_name(schema_name)` - Parse "Acura MDX Schema..." → "MDX"
2. `get_schema_patterns(schema_id)` - Get all pattern data for a schema
3. `format_engine_display(disp, hp, cyl, config)` - Format as "3.5L 290 hp V6"
4. `format_trans_display(style, speeds)` - Format as "10-Speed Automatic"
5. `generate_trans_records(has_data, style, speeds)` - Return 1 or 2 records
**Make name normalization:**
```python
MAKE_MAPPING = {
'INFINITI': 'INFINITI', # VehAPI uses all-caps
'FIAT': 'FIAT',
'RAM': 'RAM',
'RIVIAN': 'Rivian', # vPIC uses all-caps, normalize
# ... etc
}
```
### Phase 2: Test Extraction
Test with validated VINs:
```bash
source .venv/bin/activate
python3 vpic_extract.py --test-vin 5J8YE1H05SL018611 # Acura MDX
python3 vpic_extract.py --test-vin 5TFJA5DB4SX327537 # Toyota Tundra
python3 vpic_extract.py --test-vin 3GTUUFEL6PG140748 # GMC Sierra
```
### Phase 3: Full Extraction
```bash
python3 vpic_extract.py --min-year 2022 --output-dir snapshots/vpic-2025-12
```
### Phase 4: Merge & Import
```bash
# Merge vPIC with existing VehAPI data
sqlite3 snapshots/merged/snapshot.sqlite "
CREATE TABLE pairs(...);
ATTACH 'snapshots/vehicle-drop-down.sqlite' AS db1;
ATTACH 'snapshots/vpic-2025-12/snapshot.sqlite' AS db2;
INSERT OR IGNORE INTO pairs SELECT * FROM db1.pairs WHERE year < 2022;
INSERT OR IGNORE INTO pairs SELECT * FROM db2.pairs;
"
# Generate SQL and import
python3 etl_generate_sql.py --snapshot-path snapshots/merged/snapshot.sqlite
./import_data.sh
```
---
## Files to Modify
| File | Changes |
|------|---------|
| `vpic_extract.py` | Complete rewrite: VIN schema extraction, dual-record trans logic |
| `README.md` | Already updated with workflow |
---
## Success Criteria
1. Extract all 41 makes with 2022+ VIN schemas
2. ~2,500-5,000 unique vehicle configurations (Year/Make/Model/Trim/Engine)
3. Transmission: Use vPIC data where available (7 makes), dual-record elsewhere
4. Output format matches VehAPI: "3.5L 290 hp V6" / "10-Speed Automatic"
5. Merge preserves 2015-2021 VehAPI data
6. QA validation passes after import
---
## Make Analysis Status (All Families Validated)
| Family | Makes | Status | Trans Data | Strategy |
|--------|-------|--------|------------|----------|
| Honda/Acura | Acura, Honda | VALIDATED | YES (93-97%) | Use vPIC trans data |
| Toyota/Lexus | Toyota, Lexus | VALIDATED | PARTIAL (Toyota 23%, Lexus 0%) | Dual-record for Lexus |
| Nissan/Infiniti | Nissan, Infiniti, Mitsubishi | VALIDATED | LOW (5%) | Dual-record |
| GM | Chevrolet, GMC, Buick, Cadillac | VALIDATED | LOW (0-7%) | Dual-record |
| Stellantis | Chrysler, Dodge, Jeep, Ram, Fiat | VALIDATED | NONE (0%) | Dual-record |
| Ford | Ford, Lincoln | VALIDATED | NONE (0%) | Dual-record |
| VW Group | Volkswagen, Audi, Porsche, Bentley, Lamborghini | VALIDATED | MIXED (0-84%) | VW/Audi use vPIC; others dual-record |
| BMW | BMW, MINI | VALIDATED | NONE (0%) | Dual-record |
| Mercedes | Mercedes-Benz, smart | VALIDATED | YES (52%) | Use vPIC trans data |
| Hyundai/Kia/Genesis | Hyundai, Kia, Genesis | VALIDATED | NONE (0%) | Dual-record |
| Subaru | Subaru | VALIDATED | YES (64%) | Use vPIC trans data |
| Mazda | Mazda | VALIDATED | LOW (11%) | Dual-record |
| Volvo | Volvo, Polestar | VALIDATED | LOW (3%/0%) | Dual-record |
| Exotics | Ferrari, Maserati, Jaguar, Lotus, McLaren | VALIDATED | MIXED | Per-make handling |
| EV | Tesla, Rivian | VALIDATED | NONE (0%) | Dual-record (though EVs don't have "manual") |
### Special Cases
1. **Electric Vehicles** (Tesla, Rivian, Polestar): Don't have manual transmissions
- Still create dual-record for consistency with dropdown
- User can select "Automatic" (single-speed EV)
2. **Luxury Exotics** (Ferrari, Lamborghini, etc.): Mix of automated manual/DCT
- Dual-record covers all options
---
## CRITICAL FINDING: Transmission Data Availability
**Most manufacturers do NOT encode transmission info in VINs.**
### VIN Decode Validation Results (12 Families)
| Family | VIN | Make | Model | Year | Trim | Engine | Trans |
|--------|-----|------|-------|------|------|--------|-------|
| Honda/Acura | 5J8YE1H05SL018611 | ACURA | MDX | 2025 | SH-AWD A-Spec | 3.5L V6 290hp | 10-Spd Auto |
| Honda/Acura | 2HGFE4F88SH315466 | HONDA | Civic | 2025 | Sport Hybrid | 2.0L I4 141hp | e-CVT |
| Toyota/Lexus | 5TFJA5DB4SX327537 | TOYOTA | Tundra | 2025 | Limited | 3.4L V6 389hp | 10-Spd Auto |
| Nissan/Infiniti | 5N1AL1FW9TC332353 | INFINITI | QX60 | 2026 | Luxe | 2.0L (no cyl/hp) | **MISSING** |
| GM | 3GTUUFEL6PG140748 | GMC | Sierra | 2023 | AT4X | 6.2L V8 (no hp) | **MISSING** |
| Stellantis | 1C4HJXEG7PW506480 | JEEP | Wrangler | 2023 | Sahara | 3.6L V6 285hp | **MISSING** |
| Ford | 1FTFW4L59SFC03038 | FORD | F-150 | 2025 | Tremor | 5.0L V8 (no hp) | **MISSING** |
| VW Group | WVWEB7CD9RW229116 | VOLKSWAGEN | Golf R | 2024 | **MISSING** | 2.0L 4cyl 315hp | Auto (no spd) |
| BMW | 5YM13ET06R9S31554 | BMW | X5 | 2024 | X5 M Competition | 4.4L 8cyl 617hp | **MISSING** |
| Mercedes | W1KAF4HB1SR287126 | MERCEDES-BENZ | C-Class | 2025 | C300 4MATIC | 2.0L I4 255hp | 9-Spd Auto |
| Hyundai/Kia | 5XYRLDJC0SG336002 | KIA | Sorento | 2025 | S | 2.5L 4cyl 191hp | **MISSING** |
| Subaru | JF1VBAF67P9806852 | SUBARU | WRX | 2023 | Premium | 2.4L 4cyl 271hp | 6-Spd Manual |
| Mazda | JM3KFBCL3R0522361 | MAZDA | CX-5 | 2024 | Preferred Pkg | 2.5L I4 187hp | 6-Spd Auto |
| Volvo | YV4M12RJ9S1094167 | VOLVO | XC60 | 2025 | Core | 2.0L 4cyl 247hp | 8-Spd Auto |
### Transmission Data Coverage in vPIC Schemas
| Coverage | Makes | Trans Schemas / Total |
|----------|-------|----------------------|
| **HIGH (>40%)** | Honda, Acura, Subaru, Audi, VW, Mercedes, Jaguar | 225/233, 42/45, 47/74, 46/55, 47/132, 13/25, 17/17 |
| **LOW (<10%)** | Chevrolet, Cadillac, Nissan, Infiniti, Mazda, Volvo | 4/164, 7/43, 4/82, 4/74, 4/36, 2/72 |
| **NONE (0%)** | GMC, Buick, Ford, Lincoln, Jeep, Dodge, Chrysler, Ram, Fiat, BMW, MINI, Porsche, Hyundai, Kia, Genesis, Lexus, Tesla, Rivian, Polestar | 0% |
### Makes WITHOUT Transmission Data (22 of 41 makes = 54%)
- **ALL Stellantis**: Chrysler, Dodge, Jeep, Ram, Fiat
- **ALL Ford**: Ford, Lincoln
- **ALL Korean**: Hyundai, Kia, Genesis
- **ALL BMW Group**: BMW, MINI
- **GM (partial)**: GMC, Buick (Chevy/Cadillac have minimal)
- **Others**: Lexus, Porsche, Bentley, Lamborghini, Tesla, Rivian, Polestar
---
## Extraction Strategy (SELECTED)
### Dual-Record Strategy for Missing Transmission Data
When transmission data is NOT available from vPIC:
- **Create TWO records** for each vehicle configuration
- One with `trans_display = "Automatic"`, `trans_canon = "automatic"`
- One with `trans_display = "Manual"`, `trans_canon = "manual"`
This ensures:
- All transmission options available in dropdown for user selection
- User can select the correct transmission type
- No false "Unknown" values that break filtering
### Implementation Logic
```python
def generate_trans_records(has_trans_data: bool, trans_style: str, trans_speeds: str):
if has_trans_data:
# Use actual vPIC data
return [(format_trans_display(trans_style, trans_speeds),
canonicalize_trans(trans_style))]
else:
# Generate both options
return [
("Automatic", "automatic"),
("Manual", "manual")
]
```
### Expected Output Growth
For makes without trans data, record count approximately doubles:
- GMC Sierra AT4X + 6.2L V8 → 2 records (Auto + Manual)
- Ford F-150 Tremor + 5.0L V8 → 2 records (Auto + Manual)
This is acceptable as it provides complete dropdown coverage.
---
## Validated Extraction Examples
### Acura MDX 2025 (VIN: 5J8YE1H05SL018611)
- **vPIC**: Make=ACURA, Model=MDX, Trim=SH-AWD A-Spec, Engine=3.5L V6 290hp, Trans=10-Speed Automatic
- **Output**: `3.5L 290 hp V6` | `10-Speed Automatic`
### Honda Civic 2025 (VIN: 2HGFE4F88SH315466)
- **vPIC**: Make=HONDA, Model=Civic, Trim=Sport Hybrid / Sport Touring Hybrid, Engine=2L I4 141hp, Trans=e-CVT
- **Output**: `2.0L 141 hp I4` | `Electronic Continuously Variable (e-CVT)`
### Toyota Tundra 2025 (VIN: 5TFJA5DB4SX327537)
- **vPIC**: Make=TOYOTA, Model=Tundra, Trim=Limited, Engine=3.4L V6 389hp, Trans=10-Speed Automatic
- **Output**: `3.4L 389 hp V6` | `10-Speed Automatic`
### Mercedes C-Class 2025 (VIN: W1KAF4HB1SR287126)
- **vPIC**: Make=MERCEDES-BENZ, Model=C-Class, Trim=C300 4MATIC, Engine=2.0L I4 255hp, Trans=9-Speed Automatic
- **Output**: `2.0L 255 hp I4` | `9-Speed Automatic`
### Subaru WRX 2023 (VIN: JF1VBAF67P9806852)
- **vPIC**: Make=SUBARU, Model=WRX, Trim=Premium, Engine=2.4L 4cyl 271hp, Trans=6-Speed Manual
- **Output**: `2.4L 271 hp 4cyl` | `6-Speed Manual`
### Mazda CX-5 2024 (VIN: JM3KFBCL3R0522361)
- **vPIC**: Make=MAZDA, Model=CX-5, Trim=Preferred Package, Engine=2.5L I4 187hp, Trans=6-Speed Automatic
- **Output**: `2.5L 187 hp I4` | `6-Speed Automatic`
### Volvo XC60 2025 (VIN: YV4M12RJ9S1094167)
- **vPIC**: Make=VOLVO, Model=XC60, Trim=Core, Engine=2.0L 4cyl 247hp, Trans=8-Speed Automatic
- **Output**: `2.0L 247 hp 4cyl` | `8-Speed Automatic`

131
data/vehicle-etl/merge_snapshots.py
View File

@@ -1,131 +0,0 @@
#!/usr/bin/env python3
"""
Merges two VehAPI snapshot databases into a single consolidated database.
Handles deduplication via PRIMARY KEY constraint.
"""
from __future__ import annotations
import sqlite3
import sys
from pathlib import Path
def merge_databases(db1_path: Path, db2_path: Path, output_path: Path) -> dict:
"""Merge two snapshot databases into one, deduplicating by PRIMARY KEY."""
if output_path.exists():
output_path.unlink()
print(f"[info] Removed existing output file: {output_path}")
conn = sqlite3.connect(output_path)
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA synchronous=NORMAL;")
# Create target schema (pairs table only)
conn.execute("""
CREATE TABLE pairs(
year INT,
make TEXT,
model TEXT,
trim TEXT,
engine_display TEXT,
engine_canon TEXT,
engine_bucket TEXT,
trans_display TEXT,
trans_canon TEXT,
trans_bucket TEXT,
PRIMARY KEY(year, make, model, trim, engine_canon, trans_canon)
)
""")
conn.commit()
# Attach source databases
conn.execute(f"ATTACH DATABASE '{db1_path}' AS db1")
conn.execute(f"ATTACH DATABASE '{db2_path}' AS db2")
# Insert from first database
print(f"[info] Inserting records from {db1_path.name}...")
cursor = conn.execute("""
INSERT OR IGNORE INTO pairs
SELECT year, make, model, trim, engine_display, engine_canon, engine_bucket,
trans_display, trans_canon, trans_bucket
FROM db1.pairs
""")
db1_inserted = cursor.rowcount
conn.commit()
print(f"[info] Inserted {db1_inserted:,} records from {db1_path.name}")
# Insert from second database (duplicates ignored)
print(f"[info] Inserting records from {db2_path.name}...")
cursor = conn.execute("""
INSERT OR IGNORE INTO pairs
SELECT year, make, model, trim, engine_display, engine_canon, engine_bucket,
trans_display, trans_canon, trans_bucket
FROM db2.pairs
""")
db2_inserted = cursor.rowcount
conn.commit()
print(f"[info] Inserted {db2_inserted:,} new records from {db2_path.name}")
# Detach source databases
conn.execute("DETACH DATABASE db1")
conn.execute("DETACH DATABASE db2")
# Get final stats
total_count = conn.execute("SELECT COUNT(*) FROM pairs").fetchone()[0]
min_year = conn.execute("SELECT MIN(year) FROM pairs").fetchone()[0]
max_year = conn.execute("SELECT MAX(year) FROM pairs").fetchone()[0]
# Optimize the database
print("[info] Running VACUUM to optimize database...")
conn.execute("VACUUM")
conn.close()
stats = {
"db1_inserted": db1_inserted,
"db2_inserted": db2_inserted,
"total_records": total_count,
"min_year": min_year,
"max_year": max_year,
"output_path": str(output_path),
}
return stats
def main() -> int:
base_dir = Path(__file__).resolve().parent
snapshots_dir = base_dir / "snapshots"
db1_path = snapshots_dir / "1980-2007.sqlite"
db2_path = snapshots_dir / "2007-2022.sqlite"
output_path = snapshots_dir / "1980-2022-vehicles.sqlite"
# Validate source files exist
if not db1_path.exists():
print(f"[error] Source database not found: {db1_path}", file=sys.stderr)
return 1
if not db2_path.exists():
print(f"[error] Source database not found: {db2_path}", file=sys.stderr)
return 1
print(f"[info] Merging databases...")
print(f" Source 1: {db1_path}")
print(f" Source 2: {db2_path}")
print(f" Output: {output_path}")
print()
stats = merge_databases(db1_path, db2_path, output_path)
print()
print(f"[done] Merge complete!")
print(f" Total records: {stats['total_records']:,}")
print(f" Year range: {stats['min_year']} - {stats['max_year']}")
print(f" Output: {stats['output_path']}")
return 0
if __name__ == "__main__":
sys.exit(main())

513
data/vehicle-etl/vehapi_fetch_snapshot.py
View File

@@ -1,513 +0,0 @@
#!/usr/bin/env python3
"""
Fetches VehAPI data into an offline snapshot (SQLite + meta.json).
Workflow:
1. Walks Year -> Make -> Model -> Trim -> Transmission -> Engine using VehAPI.
2. Persists observed compatibility pairs to snapshot.sqlite (no Cartesian products).
3. Stores request/response cache for resume; obeys rate limits and 429 retry-after.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import random
import sqlite3
import sys
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence
from urllib.parse import quote
try:
import requests
except ImportError: # pragma: no cover - env guard
print("[error] Missing dependency 'requests'. Install with `pip install requests`.", file=sys.stderr)
sys.exit(1)
SCRIPT_VERSION = "vehapi_fetch_snapshot.py@1.1.0"
DEFAULT_MIN_YEAR = 2015
DEFAULT_MAX_YEAR = 2022
DEFAULT_RATE_PER_MIN = 55 # stays under the 60 req/min ceiling
MAX_ATTEMPTS = 5
FALLBACK_TRIMS = ["Base"]
FALLBACK_TRANSMISSIONS = ["Manual", "Automatic"]
DEFAULT_BASE_URL = "https://vehapi.com/api/v1/car-lists/get/car"
def canonicalize(value: str) -> str:
"""Lowercase, trim, collapse spaces, and normalize hyphens for dedupe keys."""
import re
cleaned = (value or "").strip()
cleaned = re.sub(r"[\s\u00A0]+", " ", cleaned)
cleaned = re.sub(r"[-\u2010-\u2015]+", "-", cleaned)
return cleaned.lower()
def infer_trans_bucket(trans_str: str) -> str:
lowered = (trans_str or "").lower()
if "manual" in lowered or "mt" in lowered or "m/t" in lowered:
return "Manual"
return "Automatic"
def infer_fuel_bucket(engine_str: str, trans_str: str, trim_str: str) -> str:
target = " ".join([engine_str or "", trans_str or "", trim_str or ""]).lower()
if any(token in target for token in ["electric", "ev", "battery", "motor", "kwh"]):
return "Electric"
if any(token in target for token in ["hybrid", "phev", "plug-in", "hev", "e-hybrid"]):
return "Hybrid"
if any(token in target for token in ["diesel", "tdi", "dci", "duramax", "power stroke", "cummins"]):
return "Diesel"
return "Gas"
def read_text_file(path: Path) -> str:
with path.open("r", encoding="utf-8") as fh:
return fh.read()
def read_lines(path: Path) -> List[str]:
return [line.strip() for line in read_text_file(path).splitlines() if line.strip()]
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as fh:
for chunk in iter(lambda: fh.read(8192), b""):
h.update(chunk)
return h.hexdigest()
def ensure_snapshot_dir(root: Path, custom_dir: Optional[str]) -> Path:
if custom_dir:
snapshot_dir = Path(custom_dir)
else:
today = datetime.now(timezone.utc).date().isoformat()
snapshot_dir = root / today
snapshot_dir.mkdir(parents=True, exist_ok=True)
return snapshot_dir
class RateLimiter:
"""Fixed delay limiter to stay below the VehAPI threshold (60 req/min)."""
def __init__(self, max_per_min: int) -> None:
self.delay = 60.0 / max_per_min # ~1.09 sec for 55 rpm
self._last_request = 0.0
def acquire(self) -> None:
now = time.monotonic()
elapsed = now - self._last_request
if elapsed < self.delay:
time.sleep(self.delay - elapsed)
self._last_request = time.monotonic()
@dataclass
class FetchCounts:
pairs_inserted: int = 0
cache_hits: int = 0
fallback_transmissions: int = 0
fallback_engines: int = 0
class VehapiFetcher:
def __init__(
self,
session: requests.Session,
base_url: str,
token: str,
min_year: int,
max_year: int,
allowed_makes: Sequence[str],
snapshot_path: Path,
responses_cache: bool = True,
rate_per_min: int = DEFAULT_RATE_PER_MIN,
) -> None:
self.session = session
self.base_url = base_url.rstrip("/")
self.token = token
self.min_year = min_year
self.max_year = max_year
self.allowed_makes = {canonicalize(m): m for m in allowed_makes}
self.snapshot_path = snapshot_path
self.conn = sqlite3.connect(self.snapshot_path)
self.conn.execute("PRAGMA journal_mode=WAL;")
self.conn.execute("PRAGMA synchronous=NORMAL;")
self._init_schema()
self.responses_cache = responses_cache
self.rate_limiter = RateLimiter(rate_per_min)
self.counts = FetchCounts()
def _init_schema(self) -> None:
self.conn.execute(
"""
CREATE TABLE IF NOT EXISTS pairs(
year INT,
make TEXT,
model TEXT,
trim TEXT,
engine_display TEXT,
engine_canon TEXT,
engine_bucket TEXT,
trans_display TEXT,
trans_canon TEXT,
trans_bucket TEXT,
PRIMARY KEY(year, make, model, trim, engine_canon, trans_canon)
)
"""
)
self.conn.execute(
"""
CREATE TABLE IF NOT EXISTS meta(
key TEXT PRIMARY KEY,
value TEXT
)
"""
)
self.conn.execute(
"""
CREATE TABLE IF NOT EXISTS responses(
request_key TEXT PRIMARY KEY,
url TEXT,
status INT,
headers_json TEXT,
body_json TEXT,
fetched_at TEXT
)
"""
)
self.conn.commit()
def _store_meta(self, meta: Dict[str, Any]) -> None:
rows = [(k, str(v)) for k, v in meta.items()]
self.conn.executemany("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)", rows)
self.conn.commit()
def _load_cached_response(self, request_key: str) -> Optional[Any]:
if not self.responses_cache:
return None
cur = self.conn.execute("SELECT body_json FROM responses WHERE request_key = ?", (request_key,))
row = cur.fetchone()
if not row:
return None
self.counts.cache_hits += 1
try:
return json.loads(row[0])
except Exception:
return None
def _save_response(self, request_key: str, url: str, status: int, headers: Dict[str, Any], body: Any) -> None:
self.conn.execute(
"""
INSERT OR REPLACE INTO responses(request_key, url, status, headers_json, body_json, fetched_at)
VALUES (?, ?, ?, ?, ?, ?)
""",
(
request_key,
url,
status,
json.dumps(dict(headers), default=str),
json.dumps(body, default=str),
datetime.now(timezone.utc).isoformat(),
),
)
self.conn.commit()
def _request_json(self, path_parts: Sequence[str], label: str) -> Any:
path_parts = [str(p) for p in path_parts]
request_key = "/".join(path_parts)
cached = self._load_cached_response(request_key)
if cached is not None:
return cached
url = f"{self.base_url}/" + "/".join(quote(p, safe="") for p in path_parts)
attempts = 0
backoff = 1.0
while attempts < MAX_ATTEMPTS:
attempts += 1
self.rate_limiter.acquire()
try:
resp = self.session.get(url, headers={"Authorization": f"Bearer {self.token}"}, timeout=30)
except requests.RequestException as exc:
print(f"[warn] {label}: request error {exc}; retrying...", file=sys.stderr)
time.sleep(backoff + random.uniform(0, 0.5))
backoff = min(backoff * 2, 30)
continue
if resp.status_code == 429:
retry_after = resp.headers.get("retry-after") or resp.headers.get("Retry-After")
try:
retry_seconds = float(retry_after)
except (TypeError, ValueError):
retry_seconds = 30.0
sleep_for = retry_seconds + random.uniform(0, 0.5)
print(f"[info] {label}: hit 429, sleeping {sleep_for:.1f}s before retry", file=sys.stderr)
time.sleep(sleep_for)
backoff = min(backoff * 2, 30)
continue
if resp.status_code >= 500:
print(f"[warn] {label}: server {resp.status_code}, retrying...", file=sys.stderr)
time.sleep(backoff + random.uniform(0, 0.5))
backoff = min(backoff * 2, 30)
continue
if not resp.ok:
print(f"[warn] {label}: HTTP {resp.status_code}, skipping", file=sys.stderr)
return []
try:
body = resp.json()
except ValueError:
print(f"[warn] {label}: non-JSON response, skipping", file=sys.stderr)
return []
self._save_response(request_key, url, resp.status_code, resp.headers, body)
return body
print(f"[error] {label}: exhausted retries", file=sys.stderr)
return []
@staticmethod
def _extract_values(payload: Any, keys: Sequence[str]) -> List[str]:
values: List[str] = []
if isinstance(payload, dict):
payload = payload.get("data") or payload.get("results") or payload.get("items") or payload
if not payload:
return values
if isinstance(payload, list):
for item in payload:
if isinstance(item, str):
if item.strip():
values.append(item.strip())
continue
if isinstance(item, dict):
for key in keys:
val = item.get(key)
if val:
values.append(str(val).strip())
break
return values
def _record_pair(
self,
year: int,
make: str,
model: str,
trim: str,
engine_display: str,
engine_bucket: str,
trans_display: str,
trans_bucket: str,
) -> None:
engine_canon = canonicalize(engine_display)
trans_canon = canonicalize(trans_display)
cur = self.conn.execute(
"""
INSERT OR IGNORE INTO pairs(
year, make, model, trim,
engine_display, engine_canon, engine_bucket,
trans_display, trans_canon, trans_bucket
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
year,
make,
model,
trim,
engine_display.strip(),
engine_canon,
engine_bucket,
trans_display.strip(),
trans_canon,
trans_bucket,
),
)
if cur.rowcount:
self.counts.pairs_inserted += 1
def _fetch_engines_for_transmission(
self, year: int, make: str, model: str, trim: str, transmission: str, trans_bucket: str
) -> None:
path = ["engines", str(year), make, model, trim, transmission]
label = f"engines:{year}/{make}/{model}/{trim}/{transmission}"
engines_payload = self._request_json(path, label)
engines = self._extract_values(engines_payload, ["engine"])
if not engines:
engine_bucket = infer_fuel_bucket("", transmission, trim)
fallback_engine = engine_bucket
self._record_pair(year, make, model, trim, fallback_engine, engine_bucket, transmission, trans_bucket)
self.counts.fallback_engines += 1
return
for engine in engines:
engine_bucket = infer_fuel_bucket(engine, transmission, trim)
self._record_pair(year, make, model, trim, engine, engine_bucket, transmission, trans_bucket)
def _fetch_transmissions_for_trim(self, year: int, make: str, model: str, trim: str) -> None:
path = ["transmissions", str(year), make, model, trim]
label = f"transmissions:{year}/{make}/{model}/{trim}"
transmissions_payload = self._request_json(path, label)
transmissions = self._extract_values(transmissions_payload, ["transmission"])
if not transmissions:
for fallback in FALLBACK_TRANSMISSIONS:
trans_bucket = infer_trans_bucket(fallback)
engine_bucket = infer_fuel_bucket("", fallback, trim)
self._record_pair(year, make, model, trim, engine_bucket, engine_bucket, fallback, trans_bucket)
self.counts.fallback_transmissions += 1
self.counts.fallback_engines += 1
return
for trans in transmissions:
trans_bucket = infer_trans_bucket(trans)
self._fetch_engines_for_transmission(year, make, model, trim, trans, trans_bucket)
def _fetch_trims_for_model(self, year: int, make: str, model: str) -> None:
print(f" -> {year} {make} {model}", file=sys.stderr)
path = ["trims", str(year), make, model]
label = f"trims:{year}/{make}/{model}"
trims_payload = self._request_json(path, label)
trims = self._extract_values(trims_payload, ["trim"])
if not trims:
trims = FALLBACK_TRIMS
for trim in trims:
self._fetch_transmissions_for_trim(year, make, model, trim)
self.conn.commit()
def _fetch_models_for_make(self, year: int, make: str) -> None:
path = ["models", str(year), make]
label = f"models:{year}/{make}"
models_payload = self._request_json(path, label)
models = self._extract_values(models_payload, ["model"])
if not models:
print(f"[warn] {label}: no models returned", file=sys.stderr)
return
for model in models:
self._fetch_trims_for_model(year, make, model)
def _fetch_makes_for_year(self, year: int) -> List[str]:
path = ["makes", str(year)]
label = f"makes:{year}"
makes_payload = self._request_json(path, label)
makes = self._extract_values(makes_payload, ["make"])
filtered = []
for make in makes:
canon = canonicalize(make)
if canon in self.allowed_makes:
filtered.append(make)
return filtered
def run(self) -> FetchCounts:
for year in range(self.min_year, self.max_year + 1):
makes = self._fetch_makes_for_year(year)
if not makes:
print(f"[info] {year}: no allowed makes found, skipping", file=sys.stderr)
continue
print(f"[info] {year}: {len(makes)} makes", file=sys.stderr)
for idx, make in enumerate(makes, 1):
print(f"[{year}] ({idx}/{len(makes)}) {make}", file=sys.stderr)
self._fetch_models_for_make(year, make)
print(f" [{self.counts.pairs_inserted} pairs so far]", file=sys.stderr)
self.conn.commit()
return self.counts
def build_arg_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Fetch VehAPI snapshot into SQLite.")
parser.add_argument("--min-year", type=int, default=int(read_env("MIN_YEAR", DEFAULT_MIN_YEAR)), help="Inclusive min year (default env MIN_YEAR or 2017)")
parser.add_argument("--max-year", type=int, default=int(read_env("MAX_YEAR", DEFAULT_MAX_YEAR)), help="Inclusive max year (default env MAX_YEAR or 2026)")
parser.add_argument("--snapshot-dir", type=str, help="Target snapshot directory (default snapshots/<today>)")
parser.add_argument("--base-url", type=str, default=read_env("VEHAPI_BASE_URL", DEFAULT_BASE_URL), help="VehAPI base URL (e.g. https://vehapi.com/api/v1/car-lists/get/car)")
parser.add_argument("--rate-per-min", type=int, default=int(read_env("VEHAPI_MAX_RPM", DEFAULT_RATE_PER_MIN)), help="Max requests per minute (<=60)")
parser.add_argument("--makes-file", type=str, default="source-makes.txt", help="Path to source-makes.txt")
parser.add_argument("--api-key-file", type=str, default="vehapi.key", help="Path to VehAPI bearer token file")
parser.add_argument("--no-response-cache", action="store_true", help="Disable request cache stored in snapshot.sqlite")
return parser
def read_env(key: str, default: Any) -> Any:
import os
return os.environ.get(key, default)
def main(argv: Sequence[str]) -> int:
parser = build_arg_parser()
args = parser.parse_args(argv)
base_dir = Path(__file__).resolve().parent
snapshot_root = base_dir / "snapshots"
snapshot_dir = ensure_snapshot_dir(snapshot_root, args.snapshot_dir)
snapshot_path = snapshot_dir / "snapshot.sqlite"
meta_path = snapshot_dir / "meta.json"
makes_file = (base_dir / args.makes_file).resolve()
api_key_file = (base_dir / args.api_key_file).resolve()
if not makes_file.exists():
print(f"[error] makes file not found: {makes_file}", file=sys.stderr)
return 1
if not api_key_file.exists():
print(f"[error] api key file not found: {api_key_file}", file=sys.stderr)
return 1
allowed_makes = read_lines(makes_file)
token = read_text_file(api_key_file).strip()
if not token:
print("[error] vehapi.key is empty", file=sys.stderr)
return 1
session = requests.Session()
fetcher = VehapiFetcher(
session=session,
base_url=args.base_url,
token=token,
min_year=args.min_year,
max_year=args.max_year,
allowed_makes=allowed_makes,
snapshot_path=snapshot_path,
responses_cache=not args.no_response_cache,
rate_per_min=args.rate_per_min,
)
started_at = datetime.now(timezone.utc)
counts = fetcher.run()
finished_at = datetime.now(timezone.utc)
meta = {
"generated_at": finished_at.isoformat(),
"started_at": started_at.isoformat(),
"min_year": args.min_year,
"max_year": args.max_year,
"script_version": SCRIPT_VERSION,
"makes_file": str(makes_file),
"makes_hash": sha256_file(makes_file),
"api_base_url": args.base_url,
"snapshot_path": str(snapshot_path),
"pairs_inserted": counts.pairs_inserted,
"fallback_transmissions": counts.fallback_transmissions,
"fallback_engines": counts.fallback_engines,
"response_cache_hits": counts.cache_hits,
}
fetcher._store_meta(meta)
with meta_path.open("w", encoding="utf-8") as fh:
json.dump(meta, fh, indent=2)
print(
f"[done] wrote snapshot to {snapshot_path} with {counts.pairs_inserted} pairs "
f"(fallback trans={counts.fallback_transmissions}, fallback engines={counts.fallback_engines}, cache hits={counts.cache_hits})",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

2489
data/vehicle-etl/vehicle-api.postman_collection.json
View File

File diff suppressed because it is too large Load Diff

10
docs/PROMPTS.md
View File

@@ -22,17 +22,17 @@ You are a senior software engineer specializsing in NodeJS, Typescript, front en
- Make no assumptions.
- Ask clarifying questions.
- Ultrathink
- You will be fixing a bug with the vehicle catalog import.
- You will be fixing a bug with the application backup function.
*** CONTEXT ***
- This is a modern web app for managing a vehicle fleet. It has both a desktop and mobile versions of the site that both need to maintain feature parity. It's currently deployed via docker compose but in the future will be deployed via k8s.
- Read README.md CLAUDE.md and AI-INDEX.md and follow relevant instructions to understand this code repository in the context of this change.
- There is an error when you try and upload a CSV file that is 15.1MB.
- The error says the file is too large
- We need to increase the max size to 25MB
- There is an error when you try and create a backup.
- Start with this file. /Users/egullickson/Documents/Technology/coding/motovaultpro/backend/src/features/admin/backup/api/backup.controller.ts
*** CHANGES TO IMPLEMENT ***
- Research this code base and ask iterative questions to compile a complete plan.
- Research this code base and ask iterative questions to compile a complete plan.
- We will pair troubleshoot this. Tell me what logs and things to run and I will

2
frontend/src/features/onboarding/components/AddVehicleStep.tsx
View File

@@ -71,7 +71,7 @@ export const AddVehicleStep: React.FC<AddVehicleStepProps> = ({
</Button>
</div>
<div className="pt-4 border-t border-gray-200 dark:border-gray-700">
<div className="pt-4 border-t border-gray-200 dark:border-border">
<Button
variant="secondary"
onClick={onBack}

4
frontend/src/features/onboarding/components/PreferencesStep.tsx
View File

@@ -60,7 +60,7 @@ export const PreferencesStep: React.FC<PreferencesStepProps> = ({ onNext, loadin
className={`min-h-[44px] py-3 px-4 rounded-lg border-2 font-medium transition-all ${
unitSystem === 'imperial'
? 'border-primary-600 bg-primary-50 text-primary-700 dark:border-primary-500 dark:bg-primary-900 dark:text-primary-300'
: 'border-gray-300 bg-white text-gray-700 hover:border-gray-400 dark:border-gray-600 dark:bg-gray-800 dark:text-gray-200 dark:hover:border-gray-500'
: 'border-gray-300 bg-white text-gray-700 hover:border-gray-400 dark:border-border dark:bg-card dark:text-gray-200 dark:hover:border-inactive'
}`}
>
<div className="text-sm font-semibold">Imperial</div>
@@ -72,7 +72,7 @@ export const PreferencesStep: React.FC<PreferencesStepProps> = ({ onNext, loadin
className={`min-h-[44px] py-3 px-4 rounded-lg border-2 font-medium transition-all ${
unitSystem === 'metric'
? 'border-primary-600 bg-primary-50 text-primary-700 dark:border-primary-500 dark:bg-primary-900 dark:text-primary-300'
: 'border-gray-300 bg-white text-gray-700 hover:border-gray-400 dark:border-gray-600 dark:bg-gray-800 dark:text-gray-200 dark:hover:border-gray-500'
: 'border-gray-300 bg-white text-gray-700 hover:border-gray-400 dark:border-border dark:bg-card dark:text-gray-200 dark:hover:border-inactive'
}`}
>
<div className="text-sm font-semibold">Metric</div>

4
frontend/src/features/onboarding/mobile/OnboardingMobileScreen.tsx
View File

@@ -87,7 +87,7 @@ export const OnboardingMobileScreen: React.FC = () => {
className={`w-10 h-10 rounded-full flex items-center justify-center font-semibold text-sm transition-all ${
stepNumbers[currentStep] >= stepNumbers[step]
? 'bg-primary-600 text-white dark:bg-primary-700 dark:text-white'
: 'bg-gray-200 text-gray-500 dark:bg-gray-700 dark:text-gray-400'
: 'bg-gray-200 text-gray-500 dark:bg-inactive dark:text-gray-400'
}`}
>
{stepNumbers[step]}
@@ -109,7 +109,7 @@ export const OnboardingMobileScreen: React.FC = () => {
className={`flex-1 h-1 mx-2 rounded transition-all ${
stepNumbers[currentStep] > stepNumbers[step]
? 'bg-primary-600 dark:bg-primary-700'
: 'bg-gray-200 dark:bg-gray-700'
: 'bg-gray-200 dark:bg-inactive'
}`}
/>
)}

6
frontend/src/features/onboarding/pages/OnboardingPage.tsx
View File

@@ -80,7 +80,7 @@ export const OnboardingPage: React.FC = () => {
className={`w-10 h-10 rounded-full flex items-center justify-center font-semibold transition-all ${
stepNumbers[currentStep] >= stepNumbers[step]
? 'bg-primary-600 text-white dark:bg-primary-700 dark:text-white'
: 'bg-gray-200 text-gray-500 dark:bg-gray-700 dark:text-gray-400'
: 'bg-gray-200 text-gray-500 dark:bg-inactive dark:text-gray-400'
}`}
>
{stepNumbers[step]}
@@ -102,7 +102,7 @@ export const OnboardingPage: React.FC = () => {
className={`flex-1 h-1 mx-2 rounded transition-all ${
stepNumbers[currentStep] > stepNumbers[step]
? 'bg-primary-600 dark:bg-primary-700'
: 'bg-gray-200 dark:bg-gray-700'
: 'bg-gray-200 dark:bg-inactive'
}`}
/>
)}
@@ -115,7 +115,7 @@ export const OnboardingPage: React.FC = () => {
</div>
{/* Step Content */}
<div className="bg-white dark:bg-gray-800 rounded-2xl shadow-xl border border-slate-200 dark:border-gray-700 p-6 md:p-8">
<div className="bg-white dark:bg-card rounded-2xl shadow-xl border border-slate-200 dark:border-border p-6 md:p-8">
{currentStep === 'preferences' && (
<PreferencesStep
onNext={handleSavePreferences}

2
frontend/src/shared-minimal/components/Button.tsx
View File

@@ -24,7 +24,7 @@ export const Button: React.FC<ButtonProps> = ({
const variants = {
primary: 'bg-primary-500 text-white hover:bg-primary-600 focus:ring-primary-500 dark:bg-primary-600 dark:hover:bg-primary-700',
secondary: 'bg-gray-200 text-gray-900 hover:bg-gray-300 focus:ring-gray-500 dark:bg-gray-700 dark:text-gray-100 dark:hover:bg-gray-600',
secondary: 'bg-gray-200 text-gray-900 hover:bg-gray-300 focus:ring-gray-500 dark:bg-inactive dark:text-gray-100 dark:hover:bg-border',
danger: 'bg-red-600 text-white hover:bg-red-700 focus:ring-red-500 dark:bg-red-700 dark:hover:bg-red-800',
};

3
frontend/tailwind.config.js
View File

@@ -27,6 +27,9 @@ export default {
// Dark theme tokens (Ferrari-inspired palette)
paper: '#1D1A18', // MUI background.paper - DARKEST brownish-black
nero: '#231F1C', // Nero Daytona - page base
card: '#2A2725', // Card backgrounds - lighter than nero
border: '#3A3735', // Borders in dark mode
inactive: '#4A4845', // Inactive UI elements
avus: '#F2F3F6', // Bianco Avus - primary text on dark
titanio: '#A8B8C0', // Grigio Titanio - secondary text on dark
canna: '#7E8792', // Canna Di Fucile - placeholder text

129
scripts/download-make-logos.sh
View File

@@ -1,129 +0,0 @@
#!/bin/bash
# Download automotive make logos from vehapi.com
# Usage: ./scripts/download-make-logos.sh
set -e
# Output directory
OUTPUT_DIR="frontend/public/images/makes"
mkdir -p "$OUTPUT_DIR"
# Base URL
BASE_URL="https://vehapi.com/img/car-logos"
# List of common automotive makes
# Format: "output_filename:url_name" (if same, just use the name)
# vehapi uses underscores for multi-word makes: alfa_romeo, aston_martin, etc.
MAKES=(
"acura"
"alfa-romeo:alfa_romeo"
"aston-martin:aston_martin"
"audi"
"bentley"
"bmw"
"buick"
"cadillac"
"chevrolet"
"chrysler"
"dodge"
"ferrari"
"fiat"
"ford"
"genesis"
"gmc"
"honda"
"hummer"
"hyundai"
"infiniti"
"isuzu"
"jaguar"
"jeep"
"kia"
"lamborghini"
"land-rover:land_rover"
"lexus"
"lincoln"
"lotus"
"maserati"
"mazda"
"mercedes-benz:mercedes_benz"
"mercury"
"mini"
"mitsubishi"
"nissan"
"oldsmobile"
"plymouth"
"polestar"
"pontiac"
"porsche"
"ram"
"rivian"
"rolls-royce:rolls_royce"
"saab"
"scion"
"smart"
"subaru"
"tesla"
"toyota"
"volkswagen"
"volvo"
)
echo "Downloading make logos to $OUTPUT_DIR..."
echo ""
SUCCESS_COUNT=0
FAIL_COUNT=0
FAILED_MAKES=()
for entry in "${MAKES[@]}"; do
# Parse entry - format is "output_name:url_name" or just "name"
if [[ "$entry" == *":"* ]]; then
OUTPUT_NAME="${entry%%:*}"
URL_NAME="${entry##*:}"
else
OUTPUT_NAME="$entry"
URL_NAME="$entry"
fi
OUTPUT_FILE="$OUTPUT_DIR/$OUTPUT_NAME.png"
URL="$BASE_URL/$URL_NAME.png"
echo -n "Downloading $OUTPUT_NAME... "
# Use curl with -f to fail on HTTP errors, -s for silent, -o for output
if curl -fsSL "$URL" -o "$OUTPUT_FILE" 2>/dev/null; then
# Verify it's actually an image (check for PNG magic bytes)
if file "$OUTPUT_FILE" | grep -q "PNG image"; then
echo "OK"
((SUCCESS_COUNT++))
else
echo "FAILED (not a valid PNG)"
rm -f "$OUTPUT_FILE"
((FAIL_COUNT++))
FAILED_MAKES+=("$OUTPUT_NAME")
fi
else
echo "FAILED (HTTP error)"
rm -f "$OUTPUT_FILE"
((FAIL_COUNT++))
FAILED_MAKES+=("$OUTPUT_NAME")
fi
done
echo ""
echo "================================"
echo "Download complete!"
echo "Success: $SUCCESS_COUNT"
echo "Failed: $FAIL_COUNT"
if [ ${#FAILED_MAKES[@]} -gt 0 ]; then
echo ""
echo "Failed makes:"
for make in "${FAILED_MAKES[@]}"; do
echo " - $make"
done
fi
echo ""
echo "Logos saved to: $OUTPUT_DIR"