fix: Database schema fixes. CI/CD improvements.

2025-12-27 16:23:22 -06:00
parent 344df5184c
commit dc2c731119
26 changed files with 242360 additions and 481192 deletions
--- a/backend/src/features/admin/domain/catalog-import.service.ts
+++ b/backend/src/features/admin/domain/catalog-import.service.ts
@@ -59,7 +59,7 @@ export class CatalogImportService {
  async previewImport(csvContent: string): Promise<ImportPreviewResult> {
    const previewId = uuidv4();
    const toCreate: ImportRow[] = [];
-    const toUpdate: ImportRow[] = [];
+    const toUpdate: ImportRow[] = []; // Kept for interface compatibility (will be empty)
    const errors: ImportError[] = [];

    const lines = csvContent.trim().split('\n');
@@ -146,21 +146,8 @@ export class CatalogImportService {
          transmissionType,
        };

-        // Check if record exists to determine create vs update (upsert logic)
-        const existsResult = await this.pool.query(
-          `SELECT id FROM vehicle_options
-           WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4
-           LIMIT 1`,
-          [year, make, model, trim]
-        );
-        const exists = (existsResult.rowCount || 0) > 0;
-
-        // Auto-detect: if exists -> update, else -> create
-        if (exists) {
-          toUpdate.push(row);
-        } else {
-          toCreate.push(row);
-        }
+        // All rows will be inserted with ON CONFLICT handling (proper upsert)
+        toCreate.push(row);
      } catch (error: any) {
        errors.push({ row: rowNum, error: error.message || 'Parse error' });
      }
@@ -239,61 +226,29 @@ export class CatalogImportService {
            transmissionId = transResult.rows[0].id;
          }

-          // Insert vehicle option
-          await client.query(
+          // Upsert vehicle option (insert or update if exists)
+          const upsertResult = await client.query(
            `INSERT INTO vehicle_options (year, make, model, trim, engine_id, transmission_id)
-             VALUES ($1, $2, $3, $4, $5, $6)`,
+             VALUES ($1, $2, $3, $4, $5, $6)
+             ON CONFLICT (year, make, model, trim, engine_id, transmission_id)
+             DO UPDATE SET updated_at = NOW()
+             RETURNING (xmax = 0) AS inserted`,
            [row.year, row.make, row.model, row.trim, engineId, transmissionId]
          );

-          result.created++;
+          // Check if this was an insert (xmax=0) or update (xmax!=0)
+          const wasInserted = upsertResult.rows[0].inserted;
+          if (wasInserted) {
+            result.created++;
+          } else {
+            result.updated++;
+          }
        } catch (error: any) {
-          result.errors.push({ row: 0, error: `Failed to create ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
+          result.errors.push({ row: 0, error: `Failed to upsert ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
        }
      }

-      // Process updates
-      for (const row of preview.toUpdate) {
-        try {
-          // Get or create engine
-          let engineId: number | null = null;
-          if (row.engineName) {
-            const engineResult = await client.query(
-              `INSERT INTO engines (name, fuel_type)
-               VALUES ($1, 'Gas')
-               ON CONFLICT ((lower(name))) DO UPDATE SET name = EXCLUDED.name
-               RETURNING id`,
-              [row.engineName]
-            );
-            engineId = engineResult.rows[0].id;
-          }
-
-          // Get or create transmission
-          let transmissionId: number | null = null;
-          if (row.transmissionType) {
-            const transResult = await client.query(
-              `INSERT INTO transmissions (type)
-               VALUES ($1)
-               ON CONFLICT ((lower(type))) DO UPDATE SET type = EXCLUDED.type
-               RETURNING id`,
-              [row.transmissionType]
-            );
-            transmissionId = transResult.rows[0].id;
-          }
-
-          // Update vehicle option
-          await client.query(
-            `UPDATE vehicle_options
-             SET engine_id = $5, transmission_id = $6, updated_at = NOW()
-             WHERE year = $1 AND make = $2 AND model = $3 AND trim = $4`,
-            [row.year, row.make, row.model, row.trim, engineId, transmissionId]
-          );
-
-          result.updated++;
-        } catch (error: any) {
-          result.errors.push({ row: 0, error: `Failed to update ${row.year} ${row.make} ${row.model} ${row.trim}: ${error.message}` });
-        }
-      }
+      // Note: Separate "Process updates" loop removed - ON CONFLICT handles both INSERT and UPDATE

      await client.query('COMMIT');

@@ -306,13 +261,23 @@ export class CatalogImportService {
        logger.debug('Vehicle data cache invalidated after import');
      }

-      logger.info('Catalog import completed', {
-        previewId,
-        created: result.created,
-        updated: result.updated,
-        errors: result.errors.length,
-        changedBy,
-      });
+      // Log completion with appropriate level
+      if (result.errors.length > 0) {
+        logger.warn('Catalog import completed with errors', {
+          previewId,
+          created: result.created,
+          updated: result.updated,
+          errors: result.errors.length,
+          changedBy,
+        });
+      } else {
+        logger.info('Catalog import completed successfully', {
+          previewId,
+          created: result.created,
+          updated: result.updated,
+          changedBy,
+        });
+      }

      return result;
    } catch (error) {
--- a/backend/src/features/backup/api/backup.controller.ts
+++ b/backend/src/features/backup/api/backup.controller.ts
@@ -180,8 +180,14 @@ export class BackupController {
      const preview = await this.restoreService.previewRestore(request.params.id);
      reply.send(preview);
    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Failed to preview restore';
+      logger.error('Preview restore failed', {
+        backupId: request.params.id,
+        error: errorMessage,
+        stack: error instanceof Error ? error.stack : undefined,
+      });
      reply.status(400).send({
-        error: error instanceof Error ? error.message : 'Failed to preview restore',
+        error: errorMessage,
      });
    }
  }
--- a/backend/src/features/platform/data/engines.sql
+++ b/backend/src/features/platform/data/engines.sql
--- a/backend/src/features/platform/data/transmissions.sql
+++ b/backend/src/features/platform/data/transmissions.sql
@@ -0,0 +1,47 @@
+INSERT INTO public.transmissions VALUES (3393, '8-Speed Dual-Clutch', NULL, NULL, '2025-12-27 20:24:19.358069', '2025-12-27 20:24:19.358069');
+INSERT INTO public.transmissions VALUES (11, 'Continuously Variable Transmission', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (3404, 'Single-Speed Direct Drive', NULL, NULL, '2025-12-27 20:24:19.358069', '2025-12-27 20:24:19.358069');
+INSERT INTO public.transmissions VALUES (15, '5-Speed Automatic Overdrive', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (3413, '2-Speed Direct Drive', NULL, NULL, '2025-12-27 20:24:19.358069', '2025-12-27 20:24:19.358069');
+INSERT INTO public.transmissions VALUES (32, '4-Speed CVT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (3072, 'Single-Speed Transmission', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (24, '5-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (9, '4-Speed Automatic Overdrive', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (5304, 'ISR Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (5081, 'Electric', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (10, '5-Speed Manual Overdrive', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (36, '10-Speed Automatic Transmission', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (13, '6-Speed Manual Overdrive', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (22, '1-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (18, '6-Speed CVT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (29, '8-Speed CVT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (4, '5-Speed Manual', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (5, '4-Speed Manual', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (3, '3-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (6, '3-Speed Manual', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (35, '2-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (1184, '9-Speed DCT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (23, '7-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (33, '10-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (34, '10-Speed CVT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (1159, '8-Speed DCT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (1172, '7-Speed DCT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (17, '6-Speed Automatic Overdrive', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (7, '4-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (14, '1-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (25, '7-Speed CVT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (30, '9-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (12, '5-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (20, '6-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (19, '7-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (28, '8-Speed Dual Clutch', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (26, '7-Speed Manual', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (8, '6-Speed Manual', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (115, 'CVT', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (2, 'Manual', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (1, 'Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (27, '9-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (21, '8-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (119, '1-Speed Direct Drive', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (16, '6-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
+INSERT INTO public.transmissions VALUES (31, '10-Speed Automatic', NULL, NULL, '2025-12-27 17:00:28.222415', '2025-12-27 17:00:28.222415');
--- a/backend/src/features/platform/data/vehicle_options.sql
+++ b/backend/src/features/platform/data/vehicle_options.sql
--- a/backend/src/features/platform/domain/catalog-seed.service.ts
+++ b/backend/src/features/platform/domain/catalog-seed.service.ts
@@ -0,0 +1,75 @@
+/**
+ * @ai-summary Vehicle catalog data seeding service
+ * @ai-context Loads vehicle catalog data from exported SQL files after migrations
+ */
+
+import { Pool } from 'pg';
+import { logger } from '../../../core/logging/logger';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export class CatalogSeedService {
+  private readonly dataDir = '/app/migrations/features/platform/data';
+
+  constructor(private pool: Pool) {}
+
+  /**
+   * Seed vehicle catalog data if tables are empty
+   */
+  async seedIfEmpty(): Promise<void> {
+    try {
+      // Check if data already exists
+      const count = await this.pool.query('SELECT COUNT(*) FROM vehicle_options');
+      const rowCount = parseInt(count.rows[0].count, 10);
+
+      if (rowCount > 0) {
+        logger.info('Vehicle catalog already seeded, skipping', { rowCount });
+        return;
+      }
+
+      logger.info('Seeding vehicle catalog data...');
+
+      // Load data files in order
+      await this.loadDataFile('engines.sql');
+      await this.loadDataFile('transmissions.sql');
+      await this.loadDataFile('vehicle_options.sql');
+
+      // Verify data loaded
+      const finalCount = await this.pool.query('SELECT COUNT(*) FROM vehicle_options');
+      const finalRowCount = parseInt(finalCount.rows[0].count, 10);
+
+      logger.info('Vehicle catalog seeding complete', { rowCount: finalRowCount });
+    } catch (error: any) {
+      logger.error('Failed to seed vehicle catalog', { error: error.message });
+      throw error;
+    }
+  }
+
+  /**
+   * Load and execute a SQL data file
+   */
+  private async loadDataFile(filename: string): Promise<void> {
+    const filePath = path.join(this.dataDir, filename);
+
+    // Check if file exists
+    if (!fs.existsSync(filePath)) {
+      logger.warn('Data file not found, skipping', { filePath });
+      return;
+    }
+
+    logger.info('Loading data file', { filename });
+
+    try {
+      // Read SQL file
+      const sql = fs.readFileSync(filePath, 'utf-8');
+
+      // Execute SQL (pg library handles INSERT statements properly)
+      await this.pool.query(sql);
+
+      logger.info('Data file loaded successfully', { filename });
+    } catch (error: any) {
+      logger.error('Failed to load data file', { filename, error: error.message });
+      throw error;
+    }
+  }
+}