Vehicle ETL Process fixed. Admin settings fixed.

This commit is contained in:
Eric Gullickson
2025-12-15 20:51:52 -06:00
parent 1a9ead9d9d
commit b84d4c7fef
23 changed files with 4553 additions and 2450 deletions

View File

@@ -0,0 +1,476 @@
/**
* @ai-summary Catalog CSV import/export service
* @ai-context Handles bulk import with preview and export of vehicle catalog data
*/
import { Pool } from 'pg';
import { logger } from '../../../core/logging/logger';
export interface ImportRow {
action: 'add' | 'update' | 'delete';
year: number;
make: string;
model: string;
trim: string;
engineName: string | null;
transmissionType: string | null;
}
export interface ImportError {
row: number;
error: string;
}
export interface ImportPreviewResult {
previewId: string;
toCreate: ImportRow[];
toUpdate: ImportRow[];
toDelete: ImportRow[];
errors: ImportError[];
valid: boolean;
}
export interface ImportApplyResult {
created: number;
updated: number;
deleted: number;
errors: ImportError[];
}
export interface ExportRow {
year: number;
make: string;
model: string;
trim: string;
engineName: string | null;
transmissionType: string | null;
}
// In-memory preview cache (expires after 15 minutes)
const previewCache = new Map<string, { data: ImportPreviewResult; expiresAt: number }>();
export class CatalogImportService {
constructor(private pool: Pool) {}
/**
* Parse CSV content and validate without applying changes
*/
async previewImport(csvContent: string): Promise<ImportPreviewResult> {
const previewId = uuidv4();
const toCreate: ImportRow[] = [];
const toUpdate: ImportRow[] = [];
const toDelete: ImportRow[] = [];
const errors: ImportError[] = [];
const lines = csvContent.trim().split('\n');
if (lines.length < 2) {
return {
previewId,
toCreate,
toUpdate,
toDelete,
errors: [{ row: 0, error: 'CSV must have a header row and at least one data row' }],
valid: false,
};
}
// Parse header row
const header = this.parseCSVLine(lines[0]);
const headerLower = header.map(h => h.toLowerCase().trim());
// Validate required headers
const requiredHeaders = ['action', 'year', 'make', 'model', 'trim'];
for (const required of requiredHeaders) {
if (!headerLower.includes(required)) {
return {
previewId,
toCreate,
toUpdate,
toDelete,
errors: [{ row: 1, error: `Missing required header: ${required}` }],
valid: false,
};
}
}
// Find column indices
const colIndices = {
action: headerLower.indexOf('action'),
year: headerLower.indexOf('year'),
make: headerLower.indexOf('make'),
model: headerLower.indexOf('model'),
trim: headerLower.indexOf('trim'),
engineName: headerLower.indexOf('engine_name'),
transmissionType: headerLower.indexOf('transmission_type'),
};
// Parse data rows
for (let i = 1; i < lines.length; i++) {
const line = lines[i].trim();
if (!line) continue;
const values = this.parseCSVLine(line);
const rowNum = i + 1;
try {
const action = values[colIndices.action]?.toLowerCase().trim();
const year = parseInt(values[colIndices.year], 10);
const make = values[colIndices.make]?.trim();
const model = values[colIndices.model]?.trim();
const trim = values[colIndices.trim]?.trim();
const engineName = colIndices.engineName >= 0 ? values[colIndices.engineName]?.trim() || null : null;
const transmissionType = colIndices.transmissionType >= 0 ? values[colIndices.transmissionType]?.trim() || null : null;
// Validate action
if (!['add', 'update', 'delete'].includes(action)) {
errors.push({ row: rowNum, error: `Invalid action: ${action}. Must be add, update, or delete` });
continue;
}
// Validate year
if (isNaN(year) || year < 1900 || year > 2100) {
errors.push({ row: rowNum, error: `Invalid year: ${values[colIndices.year]}` });
continue;
}
// Validate required fields
if (!make) {
errors.push({ row: rowNum, error: 'Make is required' });
continue;
}
if (!model) {
errors.push({ row: rowNum, error: 'Model is required' });
continue;
}
if (!trim) {
errors.push({ row: rowNum, error: 'Trim is required' });
continue;
}
const row: ImportRow = {
action: action as 'add' | 'update' | 'delete',
year,
make,
model,
trim,
engineName,
transmissionType,
};
// Check if record exists for validation
const existsResult = await this.pool.query(
`SELECT id FROM vehicle_options
WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4
LIMIT 1`,
[year, make, model, trim]
);
const exists = (existsResult.rowCount || 0) > 0;
if (action === 'add' && exists) {
errors.push({ row: rowNum, error: `Record already exists: ${year} ${make} ${model} ${trim}` });
continue;
}
if ((action === 'update' || action === 'delete') && !exists) {
errors.push({ row: rowNum, error: `Record not found: ${year} ${make} ${model} ${trim}` });
continue;
}
// Sort into appropriate bucket
switch (action) {
case 'add':
toCreate.push(row);
break;
case 'update':
toUpdate.push(row);
break;
case 'delete':
toDelete.push(row);
break;
}
} catch (error: any) {
errors.push({ row: rowNum, error: error.message || 'Parse error' });
}
}
const result: ImportPreviewResult = {
previewId,
toCreate,
toUpdate,
toDelete,
errors,
valid: errors.length === 0,
};
// Cache preview for 15 minutes
previewCache.set(previewId, {
data: result,
expiresAt: Date.now() + 15 * 60 * 1000,
});
// Clean up expired previews
this.cleanupExpiredPreviews();
return result;
}
/**
* Apply a previously previewed import
*/
async applyImport(previewId: string, changedBy: string): Promise<ImportApplyResult> {
const cached = previewCache.get(previewId);
if (!cached || cached.expiresAt < Date.now()) {
throw new Error('Preview expired or not found. Please upload the file again.');
}
const preview = cached.data;
if (!preview.valid) {
throw new Error('Cannot apply import with validation errors');
}
const result: ImportApplyResult = {
created: 0,
updated: 0,
deleted: 0,
errors: [],
};
const client = await this.pool.connect();
try {
await client.query('BEGIN');
// Process creates
for (const row of preview.toCreate) {
try {
// Get or create engine
let engineId: number | null = null;
if (row.engineName) {
const engineResult = await client.query(
`INSERT INTO engines (name, fuel_type)
VALUES ($1, 'Gas')
ON CONFLICT (LOWER(name)) DO UPDATE SET name = EXCLUDED.name
RETURNING id`,
[row.engineName]
);
engineId = engineResult.rows[0].id;
}
// Get or create transmission
let transmissionId: number | null = null;
if (row.transmissionType) {
const transResult = await client.query(
`INSERT INTO transmissions (type)
VALUES ($1)
ON CONFLICT (LOWER(type)) DO UPDATE SET type = EXCLUDED.type
RETURNING id`,
[row.transmissionType]
);
transmissionId = transResult.rows[0].id;
}
// Insert vehicle option
await client.query(
`INSERT INTO vehicle_options (year, make, model, trim, engine_id, transmission_id)
VALUES ($1, $2, $3, $4, $5, $6)`,
[row.year, row.make, row.model, row.trim, engineId, transmissionId]
);
result.created++;
} catch (error: any) {
result.errors.push({ row: 0, error: `Failed to create ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
}
}
// Process updates
for (const row of preview.toUpdate) {
try {
// Get or create engine
let engineId: number | null = null;
if (row.engineName) {
const engineResult = await client.query(
`INSERT INTO engines (name, fuel_type)
VALUES ($1, 'Gas')
ON CONFLICT (LOWER(name)) DO UPDATE SET name = EXCLUDED.name
RETURNING id`,
[row.engineName]
);
engineId = engineResult.rows[0].id;
}
// Get or create transmission
let transmissionId: number | null = null;
if (row.transmissionType) {
const transResult = await client.query(
`INSERT INTO transmissions (type)
VALUES ($1)
ON CONFLICT (LOWER(type)) DO UPDATE SET type = EXCLUDED.type
RETURNING id`,
[row.transmissionType]
);
transmissionId = transResult.rows[0].id;
}
// Update vehicle option
await client.query(
`UPDATE vehicle_options
SET engine_id = $5, transmission_id = $6, updated_at = NOW()
WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4`,
[row.year, row.make, row.model, row.trim, engineId, transmissionId]
);
result.updated++;
} catch (error: any) {
result.errors.push({ row: 0, error: `Failed to update ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
}
}
// Process deletes
for (const row of preview.toDelete) {
try {
await client.query(
`DELETE FROM vehicle_options
WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4`,
[row.year, row.make, row.model, row.trim]
);
result.deleted++;
} catch (error: any) {
result.errors.push({ row: 0, error: `Failed to delete ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
}
}
await client.query('COMMIT');
// Log the import action
await this.pool.query(
`INSERT INTO platform_change_log (change_type, resource_type, resource_id, old_value, new_value, changed_by)
VALUES ('CREATE', 'import', $1, NULL, $2, $3)`,
[
previewId,
JSON.stringify({ created: result.created, updated: result.updated, deleted: result.deleted }),
changedBy,
]
);
// Remove preview from cache
previewCache.delete(previewId);
logger.info('Catalog import completed', {
previewId,
created: result.created,
updated: result.updated,
deleted: result.deleted,
errors: result.errors.length,
changedBy,
});
return result;
} catch (error) {
await client.query('ROLLBACK');
throw error;
} finally {
client.release();
}
}
/**
* Export all vehicle options as CSV
*/
async exportCatalog(): Promise<string> {
const result = await this.pool.query(`
SELECT
vo.year,
vo.make,
vo.model,
vo.trim,
e.name AS engine_name,
t.type AS transmission_type
FROM vehicle_options vo
LEFT JOIN engines e ON vo.engine_id = e.id
LEFT JOIN transmissions t ON vo.transmission_id = t.id
ORDER BY vo.year DESC, vo.make ASC, vo.model ASC, vo.trim ASC
`);
// Build CSV
const header = 'year,make,model,trim,engine_name,transmission_type';
const rows = result.rows.map(row => {
return [
row.year,
this.escapeCSVField(row.make),
this.escapeCSVField(row.model),
this.escapeCSVField(row.trim),
this.escapeCSVField(row.engine_name || ''),
this.escapeCSVField(row.transmission_type || ''),
].join(',');
});
return [header, ...rows].join('\n');
}
/**
* Parse a single CSV line, handling quoted fields
*/
private parseCSVLine(line: string): string[] {
const result: string[] = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
const nextChar = line[i + 1];
if (inQuotes) {
if (char === '"' && nextChar === '"') {
current += '"';
i++; // Skip next quote
} else if (char === '"') {
inQuotes = false;
} else {
current += char;
}
} else {
if (char === '"') {
inQuotes = true;
} else if (char === ',') {
result.push(current);
current = '';
} else {
current += char;
}
}
}
result.push(current);
return result;
}
/**
* Escape a field for CSV output
*/
private escapeCSVField(value: string): string {
if (!value) return '';
if (value.includes(',') || value.includes('"') || value.includes('\n')) {
return `"${value.replace(/"/g, '""')}"`;
}
return value;
}
/**
* Clean up expired preview cache entries
*/
private cleanupExpiredPreviews(): void {
const now = Date.now();
for (const [id, entry] of previewCache.entries()) {
if (entry.expiresAt < now) {
previewCache.delete(id);
}
}
}
}
// Simple UUID generation without external dependency
function uuidv4(): string {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
const r = Math.random() * 16 | 0;
const v = c === 'x' ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
}

View File

@@ -60,6 +60,34 @@ export interface PlatformChangeLog {
createdAt: Date;
}
export interface CatalogSearchResult {
id: number;
year: number;
make: string;
model: string;
trim: string;
engineId: number | null;
engineName: string | null;
transmissionId: number | null;
transmissionType: string | null;
}
export interface CatalogSearchResponse {
items: CatalogSearchResult[];
total: number;
page: number;
pageSize: number;
}
export interface CascadeDeleteResult {
deletedMakes: number;
deletedModels: number;
deletedYears: number;
deletedTrims: number;
deletedEngines: number;
totalDeleted: number;
}
const VEHICLE_SCHEMA = 'vehicles';
export class VehicleCatalogService {
@@ -611,6 +639,338 @@ export class VehicleCatalogService {
}
}
// SEARCH -----------------------------------------------------------------
async searchCatalog(
query: string,
page: number = 1,
pageSize: number = 50
): Promise<CatalogSearchResponse> {
const offset = (page - 1) * pageSize;
const sanitizedQuery = query.trim();
if (!sanitizedQuery) {
return { items: [], total: 0, page, pageSize };
}
// Convert query to tsquery format - split words and join with &
const tsQueryTerms = sanitizedQuery
.split(/\s+/)
.filter(term => term.length > 0)
.map(term => term.replace(/[^\w]/g, ''))
.filter(term => term.length > 0)
.map(term => `${term}:*`)
.join(' & ');
if (!tsQueryTerms) {
return { items: [], total: 0, page, pageSize };
}
try {
// Count total matching records
const countQuery = `
SELECT COUNT(*) as total
FROM vehicle_options vo
WHERE to_tsvector('english', vo.year::text || ' ' || vo.make || ' ' || vo.model || ' ' || vo.trim)
@@ to_tsquery('english', $1)
`;
const countResult = await this.pool.query(countQuery, [tsQueryTerms]);
const total = parseInt(countResult.rows[0].total, 10);
// Fetch paginated results with engine and transmission details
const searchQuery = `
SELECT
vo.id,
vo.year,
vo.make,
vo.model,
vo.trim,
vo.engine_id,
e.name AS engine_name,
vo.transmission_id,
t.type AS transmission_type
FROM vehicle_options vo
LEFT JOIN engines e ON vo.engine_id = e.id
LEFT JOIN transmissions t ON vo.transmission_id = t.id
WHERE to_tsvector('english', vo.year::text || ' ' || vo.make || ' ' || vo.model || ' ' || vo.trim)
@@ to_tsquery('english', $1)
ORDER BY vo.year DESC, vo.make ASC, vo.model ASC, vo.trim ASC
LIMIT $2 OFFSET $3
`;
const result = await this.pool.query(searchQuery, [tsQueryTerms, pageSize, offset]);
const items: CatalogSearchResult[] = result.rows.map((row) => ({
id: Number(row.id),
year: Number(row.year),
make: row.make,
model: row.model,
trim: row.trim,
engineId: row.engine_id ? Number(row.engine_id) : null,
engineName: row.engine_name || null,
transmissionId: row.transmission_id ? Number(row.transmission_id) : null,
transmissionType: row.transmission_type || null,
}));
return { items, total, page, pageSize };
} catch (error) {
logger.error('Error searching catalog', { error, query: sanitizedQuery });
throw error;
}
}
// CASCADE DELETE METHODS -------------------------------------------------
async deleteMakeCascade(makeId: number, changedBy: string): Promise<CascadeDeleteResult> {
const result: CascadeDeleteResult = {
deletedMakes: 0,
deletedModels: 0,
deletedYears: 0,
deletedTrims: 0,
deletedEngines: 0,
totalDeleted: 0,
};
await this.runInTransaction(async (client) => {
// Verify make exists
const makeResult = await client.query(
`SELECT id, name, created_at, updated_at FROM ${VEHICLE_SCHEMA}.make WHERE id = $1`,
[makeId]
);
if (makeResult.rowCount === 0) {
throw new Error(`Make ${makeId} not found`);
}
const makeData = this.mapMakeRow(makeResult.rows[0]);
// Get all models for this make
const modelsResult = await client.query(
`SELECT id FROM ${VEHICLE_SCHEMA}.model WHERE make_id = $1`,
[makeId]
);
// Cascade delete all models and their children
for (const modelRow of modelsResult.rows) {
const modelDeletes = await this.deleteModelCascadeInTransaction(client, modelRow.id, changedBy);
result.deletedModels += modelDeletes.deletedModels;
result.deletedYears += modelDeletes.deletedYears;
result.deletedTrims += modelDeletes.deletedTrims;
result.deletedEngines += modelDeletes.deletedEngines;
}
// Delete the make itself
await client.query(`DELETE FROM ${VEHICLE_SCHEMA}.make WHERE id = $1`, [makeId]);
await this.logChange(client, 'DELETE', 'makes', makeId.toString(), makeData, null, changedBy);
result.deletedMakes = 1;
});
result.totalDeleted = result.deletedMakes + result.deletedModels + result.deletedYears + result.deletedTrims + result.deletedEngines;
await this.cacheService.invalidateVehicleData();
return result;
}
async deleteModelCascade(modelId: number, changedBy: string): Promise<CascadeDeleteResult> {
const result: CascadeDeleteResult = {
deletedMakes: 0,
deletedModels: 0,
deletedYears: 0,
deletedTrims: 0,
deletedEngines: 0,
totalDeleted: 0,
};
await this.runInTransaction(async (client) => {
const deletes = await this.deleteModelCascadeInTransaction(client, modelId, changedBy);
result.deletedModels = deletes.deletedModels;
result.deletedYears = deletes.deletedYears;
result.deletedTrims = deletes.deletedTrims;
result.deletedEngines = deletes.deletedEngines;
});
result.totalDeleted = result.deletedModels + result.deletedYears + result.deletedTrims + result.deletedEngines;
await this.cacheService.invalidateVehicleData();
return result;
}
async deleteYearCascade(yearId: number, changedBy: string): Promise<CascadeDeleteResult> {
const result: CascadeDeleteResult = {
deletedMakes: 0,
deletedModels: 0,
deletedYears: 0,
deletedTrims: 0,
deletedEngines: 0,
totalDeleted: 0,
};
await this.runInTransaction(async (client) => {
const deletes = await this.deleteYearCascadeInTransaction(client, yearId, changedBy);
result.deletedYears = deletes.deletedYears;
result.deletedTrims = deletes.deletedTrims;
result.deletedEngines = deletes.deletedEngines;
});
result.totalDeleted = result.deletedYears + result.deletedTrims + result.deletedEngines;
await this.cacheService.invalidateVehicleData();
return result;
}
async deleteTrimCascade(trimId: number, changedBy: string): Promise<CascadeDeleteResult> {
const result: CascadeDeleteResult = {
deletedMakes: 0,
deletedModels: 0,
deletedYears: 0,
deletedTrims: 0,
deletedEngines: 0,
totalDeleted: 0,
};
await this.runInTransaction(async (client) => {
const deletes = await this.deleteTrimCascadeInTransaction(client, trimId, changedBy);
result.deletedTrims = deletes.deletedTrims;
result.deletedEngines = deletes.deletedEngines;
});
result.totalDeleted = result.deletedTrims + result.deletedEngines;
await this.cacheService.invalidateVehicleData();
return result;
}
// Private cascade helpers (run within existing transaction)
private async deleteModelCascadeInTransaction(
client: PoolClient,
modelId: number,
changedBy: string
): Promise<{ deletedModels: number; deletedYears: number; deletedTrims: number; deletedEngines: number }> {
const result = { deletedModels: 0, deletedYears: 0, deletedTrims: 0, deletedEngines: 0 };
// Verify model exists
const modelResult = await client.query(
`SELECT id, make_id, name, created_at, updated_at FROM ${VEHICLE_SCHEMA}.model WHERE id = $1`,
[modelId]
);
if (modelResult.rowCount === 0) {
throw new Error(`Model ${modelId} not found`);
}
const modelData = this.mapModelRow(modelResult.rows[0]);
// Get all years for this model
const yearsResult = await client.query(
`SELECT id FROM ${VEHICLE_SCHEMA}.model_year WHERE model_id = $1`,
[modelId]
);
// Cascade delete all years and their children
for (const yearRow of yearsResult.rows) {
const yearDeletes = await this.deleteYearCascadeInTransaction(client, yearRow.id, changedBy);
result.deletedYears += yearDeletes.deletedYears;
result.deletedTrims += yearDeletes.deletedTrims;
result.deletedEngines += yearDeletes.deletedEngines;
}
// Delete the model itself
await client.query(`DELETE FROM ${VEHICLE_SCHEMA}.model WHERE id = $1`, [modelId]);
await this.logChange(client, 'DELETE', 'models', modelId.toString(), modelData, null, changedBy);
result.deletedModels = 1;
return result;
}
private async deleteYearCascadeInTransaction(
client: PoolClient,
yearId: number,
changedBy: string
): Promise<{ deletedYears: number; deletedTrims: number; deletedEngines: number }> {
const result = { deletedYears: 0, deletedTrims: 0, deletedEngines: 0 };
// Verify year exists
const yearResult = await client.query(
`SELECT id, model_id, year, created_at, updated_at FROM ${VEHICLE_SCHEMA}.model_year WHERE id = $1`,
[yearId]
);
if (yearResult.rowCount === 0) {
throw new Error(`Year ${yearId} not found`);
}
const yearData = this.mapYearRow(yearResult.rows[0]);
// Get all trims for this year
const trimsResult = await client.query(
`SELECT id FROM ${VEHICLE_SCHEMA}.trim WHERE model_year_id = $1`,
[yearId]
);
// Cascade delete all trims and their engines
for (const trimRow of trimsResult.rows) {
const trimDeletes = await this.deleteTrimCascadeInTransaction(client, trimRow.id, changedBy);
result.deletedTrims += trimDeletes.deletedTrims;
result.deletedEngines += trimDeletes.deletedEngines;
}
// Delete the year itself
await client.query(`DELETE FROM ${VEHICLE_SCHEMA}.model_year WHERE id = $1`, [yearId]);
await this.logChange(client, 'DELETE', 'years', yearId.toString(), yearData, null, changedBy);
result.deletedYears = 1;
return result;
}
private async deleteTrimCascadeInTransaction(
client: PoolClient,
trimId: number,
changedBy: string
): Promise<{ deletedTrims: number; deletedEngines: number }> {
const result = { deletedTrims: 0, deletedEngines: 0 };
// Verify trim exists
const trimResult = await client.query(
`SELECT id, model_year_id, name, created_at, updated_at FROM ${VEHICLE_SCHEMA}.trim WHERE id = $1`,
[trimId]
);
if (trimResult.rowCount === 0) {
throw new Error(`Trim ${trimId} not found`);
}
const trimData = this.mapTrimRow(trimResult.rows[0]);
// Get all engines linked to this trim
const enginesResult = await client.query(
`SELECT e.id, e.name, e.displacement_l, e.cylinders, e.fuel_type, e.created_at, e.updated_at, te.trim_id
FROM ${VEHICLE_SCHEMA}.engine e
JOIN ${VEHICLE_SCHEMA}.trim_engine te ON te.engine_id = e.id
WHERE te.trim_id = $1`,
[trimId]
);
// Delete engine links and potentially orphaned engines
for (const engineRow of enginesResult.rows) {
const engineData = this.mapEngineRow(engineRow);
// Remove the link first
await client.query(
`DELETE FROM ${VEHICLE_SCHEMA}.trim_engine WHERE trim_id = $1 AND engine_id = $2`,
[trimId, engineRow.id]
);
// Check if this engine is used by other trims
const otherLinksResult = await client.query(
`SELECT 1 FROM ${VEHICLE_SCHEMA}.trim_engine WHERE engine_id = $1 LIMIT 1`,
[engineRow.id]
);
// If no other trims use this engine, delete the engine itself
if ((otherLinksResult.rowCount || 0) === 0) {
await client.query(`DELETE FROM ${VEHICLE_SCHEMA}.engine WHERE id = $1`, [engineRow.id]);
await this.logChange(client, 'DELETE', 'engines', engineRow.id.toString(), engineData, null, changedBy);
result.deletedEngines += 1;
}
}
// Delete the trim itself
await client.query(`DELETE FROM ${VEHICLE_SCHEMA}.trim WHERE id = $1`, [trimId]);
await this.logChange(client, 'DELETE', 'trims', trimId.toString(), trimData, null, changedBy);
result.deletedTrims = 1;
return result;
}
// HELPERS ----------------------------------------------------------------
private async runInTransaction<T>(handler: (client: PoolClient) => Promise<T>): Promise<T> {