Vehicle ETL Process fixed. Admin settings fixed.
This commit is contained in:
Eric Gullickson
476
backend/src/features/admin/domain/catalog-import.service.ts
Normal file
476
backend/src/features/admin/domain/catalog-import.service.ts
Normal file
@@ -0,0 +1,476 @@
|
||||
/**
|
||||
* @ai-summary Catalog CSV import/export service
|
||||
* @ai-context Handles bulk import with preview and export of vehicle catalog data
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
import { logger } from '../../../core/logging/logger';
|
||||
|
||||
export interface ImportRow {
|
||||
action: 'add' | 'update' | 'delete';
|
||||
year: number;
|
||||
make: string;
|
||||
model: string;
|
||||
trim: string;
|
||||
engineName: string | null;
|
||||
transmissionType: string | null;
|
||||
}
|
||||
|
||||
export interface ImportError {
|
||||
row: number;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export interface ImportPreviewResult {
|
||||
previewId: string;
|
||||
toCreate: ImportRow[];
|
||||
toUpdate: ImportRow[];
|
||||
toDelete: ImportRow[];
|
||||
errors: ImportError[];
|
||||
valid: boolean;
|
||||
}
|
||||
|
||||
export interface ImportApplyResult {
|
||||
created: number;
|
||||
updated: number;
|
||||
deleted: number;
|
||||
errors: ImportError[];
|
||||
}
|
||||
|
||||
export interface ExportRow {
|
||||
year: number;
|
||||
make: string;
|
||||
model: string;
|
||||
trim: string;
|
||||
engineName: string | null;
|
||||
transmissionType: string | null;
|
||||
}
|
||||
|
||||
// In-memory preview cache (expires after 15 minutes)
|
||||
const previewCache = new Map<string, { data: ImportPreviewResult; expiresAt: number }>();
|
||||
|
||||
export class CatalogImportService {
|
||||
constructor(private pool: Pool) {}
|
||||
|
||||
/**
|
||||
* Parse CSV content and validate without applying changes
|
||||
*/
|
||||
async previewImport(csvContent: string): Promise<ImportPreviewResult> {
|
||||
const previewId = uuidv4();
|
||||
const toCreate: ImportRow[] = [];
|
||||
const toUpdate: ImportRow[] = [];
|
||||
const toDelete: ImportRow[] = [];
|
||||
const errors: ImportError[] = [];
|
||||
|
||||
const lines = csvContent.trim().split('\n');
|
||||
if (lines.length < 2) {
|
||||
return {
|
||||
previewId,
|
||||
toCreate,
|
||||
toUpdate,
|
||||
toDelete,
|
||||
errors: [{ row: 0, error: 'CSV must have a header row and at least one data row' }],
|
||||
valid: false,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse header row
|
||||
const header = this.parseCSVLine(lines[0]);
|
||||
const headerLower = header.map(h => h.toLowerCase().trim());
|
||||
|
||||
// Validate required headers
|
||||
const requiredHeaders = ['action', 'year', 'make', 'model', 'trim'];
|
||||
for (const required of requiredHeaders) {
|
||||
if (!headerLower.includes(required)) {
|
||||
return {
|
||||
previewId,
|
||||
toCreate,
|
||||
toUpdate,
|
||||
toDelete,
|
||||
errors: [{ row: 1, error: `Missing required header: ${required}` }],
|
||||
valid: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Find column indices
|
||||
const colIndices = {
|
||||
action: headerLower.indexOf('action'),
|
||||
year: headerLower.indexOf('year'),
|
||||
make: headerLower.indexOf('make'),
|
||||
model: headerLower.indexOf('model'),
|
||||
trim: headerLower.indexOf('trim'),
|
||||
engineName: headerLower.indexOf('engine_name'),
|
||||
transmissionType: headerLower.indexOf('transmission_type'),
|
||||
};
|
||||
|
||||
// Parse data rows
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
if (!line) continue;
|
||||
|
||||
const values = this.parseCSVLine(line);
|
||||
const rowNum = i + 1;
|
||||
|
||||
try {
|
||||
const action = values[colIndices.action]?.toLowerCase().trim();
|
||||
const year = parseInt(values[colIndices.year], 10);
|
||||
const make = values[colIndices.make]?.trim();
|
||||
const model = values[colIndices.model]?.trim();
|
||||
const trim = values[colIndices.trim]?.trim();
|
||||
const engineName = colIndices.engineName >= 0 ? values[colIndices.engineName]?.trim() || null : null;
|
||||
const transmissionType = colIndices.transmissionType >= 0 ? values[colIndices.transmissionType]?.trim() || null : null;
|
||||
|
||||
// Validate action
|
||||
if (!['add', 'update', 'delete'].includes(action)) {
|
||||
errors.push({ row: rowNum, error: `Invalid action: ${action}. Must be add, update, or delete` });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate year
|
||||
if (isNaN(year) || year < 1900 || year > 2100) {
|
||||
errors.push({ row: rowNum, error: `Invalid year: ${values[colIndices.year]}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate required fields
|
||||
if (!make) {
|
||||
errors.push({ row: rowNum, error: 'Make is required' });
|
||||
continue;
|
||||
}
|
||||
if (!model) {
|
||||
errors.push({ row: rowNum, error: 'Model is required' });
|
||||
continue;
|
||||
}
|
||||
if (!trim) {
|
||||
errors.push({ row: rowNum, error: 'Trim is required' });
|
||||
continue;
|
||||
}
|
||||
|
||||
const row: ImportRow = {
|
||||
action: action as 'add' | 'update' | 'delete',
|
||||
year,
|
||||
make,
|
||||
model,
|
||||
trim,
|
||||
engineName,
|
||||
transmissionType,
|
||||
};
|
||||
|
||||
// Check if record exists for validation
|
||||
const existsResult = await this.pool.query(
|
||||
`SELECT id FROM vehicle_options
|
||||
WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4
|
||||
LIMIT 1`,
|
||||
[year, make, model, trim]
|
||||
);
|
||||
const exists = (existsResult.rowCount || 0) > 0;
|
||||
|
||||
if (action === 'add' && exists) {
|
||||
errors.push({ row: rowNum, error: `Record already exists: ${year} ${make} ${model} ${trim}` });
|
||||
continue;
|
||||
}
|
||||
if ((action === 'update' || action === 'delete') && !exists) {
|
||||
errors.push({ row: rowNum, error: `Record not found: ${year} ${make} ${model} ${trim}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sort into appropriate bucket
|
||||
switch (action) {
|
||||
case 'add':
|
||||
toCreate.push(row);
|
||||
break;
|
||||
case 'update':
|
||||
toUpdate.push(row);
|
||||
break;
|
||||
case 'delete':
|
||||
toDelete.push(row);
|
||||
break;
|
||||
}
|
||||
} catch (error: any) {
|
||||
errors.push({ row: rowNum, error: error.message || 'Parse error' });
|
||||
}
|
||||
}
|
||||
|
||||
const result: ImportPreviewResult = {
|
||||
previewId,
|
||||
toCreate,
|
||||
toUpdate,
|
||||
toDelete,
|
||||
errors,
|
||||
valid: errors.length === 0,
|
||||
};
|
||||
|
||||
// Cache preview for 15 minutes
|
||||
previewCache.set(previewId, {
|
||||
data: result,
|
||||
expiresAt: Date.now() + 15 * 60 * 1000,
|
||||
});
|
||||
|
||||
// Clean up expired previews
|
||||
this.cleanupExpiredPreviews();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a previously previewed import
|
||||
*/
|
||||
async applyImport(previewId: string, changedBy: string): Promise<ImportApplyResult> {
|
||||
const cached = previewCache.get(previewId);
|
||||
if (!cached || cached.expiresAt < Date.now()) {
|
||||
throw new Error('Preview expired or not found. Please upload the file again.');
|
||||
}
|
||||
|
||||
const preview = cached.data;
|
||||
if (!preview.valid) {
|
||||
throw new Error('Cannot apply import with validation errors');
|
||||
}
|
||||
|
||||
const result: ImportApplyResult = {
|
||||
created: 0,
|
||||
updated: 0,
|
||||
deleted: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
// Process creates
|
||||
for (const row of preview.toCreate) {
|
||||
try {
|
||||
// Get or create engine
|
||||
let engineId: number | null = null;
|
||||
if (row.engineName) {
|
||||
const engineResult = await client.query(
|
||||
`INSERT INTO engines (name, fuel_type)
|
||||
VALUES ($1, 'Gas')
|
||||
ON CONFLICT (LOWER(name)) DO UPDATE SET name = EXCLUDED.name
|
||||
RETURNING id`,
|
||||
[row.engineName]
|
||||
);
|
||||
engineId = engineResult.rows[0].id;
|
||||
}
|
||||
|
||||
// Get or create transmission
|
||||
let transmissionId: number | null = null;
|
||||
if (row.transmissionType) {
|
||||
const transResult = await client.query(
|
||||
`INSERT INTO transmissions (type)
|
||||
VALUES ($1)
|
||||
ON CONFLICT (LOWER(type)) DO UPDATE SET type = EXCLUDED.type
|
||||
RETURNING id`,
|
||||
[row.transmissionType]
|
||||
);
|
||||
transmissionId = transResult.rows[0].id;
|
||||
}
|
||||
|
||||
// Insert vehicle option
|
||||
await client.query(
|
||||
`INSERT INTO vehicle_options (year, make, model, trim, engine_id, transmission_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||
[row.year, row.make, row.model, row.trim, engineId, transmissionId]
|
||||
);
|
||||
|
||||
result.created++;
|
||||
} catch (error: any) {
|
||||
result.errors.push({ row: 0, error: `Failed to create ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
|
||||
}
|
||||
}
|
||||
|
||||
// Process updates
|
||||
for (const row of preview.toUpdate) {
|
||||
try {
|
||||
// Get or create engine
|
||||
let engineId: number | null = null;
|
||||
if (row.engineName) {
|
||||
const engineResult = await client.query(
|
||||
`INSERT INTO engines (name, fuel_type)
|
||||
VALUES ($1, 'Gas')
|
||||
ON CONFLICT (LOWER(name)) DO UPDATE SET name = EXCLUDED.name
|
||||
RETURNING id`,
|
||||
[row.engineName]
|
||||
);
|
||||
engineId = engineResult.rows[0].id;
|
||||
}
|
||||
|
||||
// Get or create transmission
|
||||
let transmissionId: number | null = null;
|
||||
if (row.transmissionType) {
|
||||
const transResult = await client.query(
|
||||
`INSERT INTO transmissions (type)
|
||||
VALUES ($1)
|
||||
ON CONFLICT (LOWER(type)) DO UPDATE SET type = EXCLUDED.type
|
||||
RETURNING id`,
|
||||
[row.transmissionType]
|
||||
);
|
||||
transmissionId = transResult.rows[0].id;
|
||||
}
|
||||
|
||||
// Update vehicle option
|
||||
await client.query(
|
||||
`UPDATE vehicle_options
|
||||
SET engine_id = $5, transmission_id = $6, updated_at = NOW()
|
||||
WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4`,
|
||||
[row.year, row.make, row.model, row.trim, engineId, transmissionId]
|
||||
);
|
||||
|
||||
result.updated++;
|
||||
} catch (error: any) {
|
||||
result.errors.push({ row: 0, error: `Failed to update ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
|
||||
}
|
||||
}
|
||||
|
||||
// Process deletes
|
||||
for (const row of preview.toDelete) {
|
||||
try {
|
||||
await client.query(
|
||||
`DELETE FROM vehicle_options
|
||||
WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4`,
|
||||
[row.year, row.make, row.model, row.trim]
|
||||
);
|
||||
result.deleted++;
|
||||
} catch (error: any) {
|
||||
result.errors.push({ row: 0, error: `Failed to delete ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
|
||||
}
|
||||
}
|
||||
|
||||
await client.query('COMMIT');
|
||||
|
||||
// Log the import action
|
||||
await this.pool.query(
|
||||
`INSERT INTO platform_change_log (change_type, resource_type, resource_id, old_value, new_value, changed_by)
|
||||
VALUES ('CREATE', 'import', $1, NULL, $2, $3)`,
|
||||
[
|
||||
previewId,
|
||||
JSON.stringify({ created: result.created, updated: result.updated, deleted: result.deleted }),
|
||||
changedBy,
|
||||
]
|
||||
);
|
||||
|
||||
// Remove preview from cache
|
||||
previewCache.delete(previewId);
|
||||
|
||||
logger.info('Catalog import completed', {
|
||||
previewId,
|
||||
created: result.created,
|
||||
updated: result.updated,
|
||||
deleted: result.deleted,
|
||||
errors: result.errors.length,
|
||||
changedBy,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export all vehicle options as CSV
|
||||
*/
|
||||
async exportCatalog(): Promise<string> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
vo.year,
|
||||
vo.make,
|
||||
vo.model,
|
||||
vo.trim,
|
||||
e.name AS engine_name,
|
||||
t.type AS transmission_type
|
||||
FROM vehicle_options vo
|
||||
LEFT JOIN engines e ON vo.engine_id = e.id
|
||||
LEFT JOIN transmissions t ON vo.transmission_id = t.id
|
||||
ORDER BY vo.year DESC, vo.make ASC, vo.model ASC, vo.trim ASC
|
||||
`);
|
||||
|
||||
// Build CSV
|
||||
const header = 'year,make,model,trim,engine_name,transmission_type';
|
||||
const rows = result.rows.map(row => {
|
||||
return [
|
||||
row.year,
|
||||
this.escapeCSVField(row.make),
|
||||
this.escapeCSVField(row.model),
|
||||
this.escapeCSVField(row.trim),
|
||||
this.escapeCSVField(row.engine_name || ''),
|
||||
this.escapeCSVField(row.transmission_type || ''),
|
||||
].join(',');
|
||||
});
|
||||
|
||||
return [header, ...rows].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single CSV line, handling quoted fields
|
||||
*/
|
||||
private parseCSVLine(line: string): string[] {
|
||||
const result: string[] = [];
|
||||
let current = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const char = line[i];
|
||||
const nextChar = line[i + 1];
|
||||
|
||||
if (inQuotes) {
|
||||
if (char === '"' && nextChar === '"') {
|
||||
current += '"';
|
||||
i++; // Skip next quote
|
||||
} else if (char === '"') {
|
||||
inQuotes = false;
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
} else {
|
||||
if (char === '"') {
|
||||
inQuotes = true;
|
||||
} else if (char === ',') {
|
||||
result.push(current);
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.push(current);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape a field for CSV output
|
||||
*/
|
||||
private escapeCSVField(value: string): string {
|
||||
if (!value) return '';
|
||||
if (value.includes(',') || value.includes('"') || value.includes('\n')) {
|
||||
return `"${value.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up expired preview cache entries
|
||||
*/
|
||||
private cleanupExpiredPreviews(): void {
|
||||
const now = Date.now();
|
||||
for (const [id, entry] of previewCache.entries()) {
|
||||
if (entry.expiresAt < now) {
|
||||
previewCache.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Simple UUID generation without external dependency
|
||||
function uuidv4(): string {
|
||||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
||||
const r = Math.random() * 16 | 0;
|
||||
const v = c === 'x' ? r : (r & 0x3 | 0x8);
|
||||
return v.toString(16);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user