fix: Fix imports and database bugs. Removed legacy ETL code.
This commit is contained in:
Eric Gullickson
Binary file not shown.
BIN
data/backups/motovaultpro_backup_2025-12-27T18-03-54.tar.gz
Normal file
BIN
data/backups/motovaultpro_backup_2025-12-27T18-03-54.tar.gz
Normal file
Binary file not shown.
@@ -1,74 +0,0 @@
|
||||
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
|
||||
-----------------------------------------------------+-----------------------------------------------+-----------------------------------------------------------------------------------------------------------------+---------------+-----------------+--------------+---------------+-----------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+----------------------------------+--------------
|
||||
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
|
||||
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
|
||||
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
|
||||
| Vehicle Descriptor | 5J8YE1H0*SL | | | | 196 | 5J8YE1H0*SL | | | VehicleDescriptor | string | Decoding | Corrections |
|
||||
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
|
||||
| Additional Error Text | | | | | 156 | | | | AdditionalErrorText | string | Decoding | Corrections |
|
||||
General | Vehicle Type | MULTIPURPOSE PASSENGER VEHICLE (MPV) | | | 5J8 | 39 | 7 | 2015-03-27 15:31:28.273 | 2105 | VehicleType | lookup | WMI | VehType |
|
||||
General | Make | ACURA | 2074168 | 26929 | Y[ED]*H | 26 | 475 | | | Make | lookup | WMI, Pattern | pattern - model |
|
||||
General | Manufacturer Name | AMERICAN HONDA MOTOR CO., INC. | | | 5J8 | 27 | 988 | | 2105 | Manufacturer | lookup | WMI | Manu. Name |
|
||||
General | Model Year | 2025 | | | ***X*|Y | 29 | 2025 | | | ModelYear | int | WMI, Input | ModelYear |
|
||||
General | Model | MDX | 2074168 | 26929 | Y[ED]*H | 28 | 2147 | 2024-05-21 10:17:20.287 | 2105 | Model | lookup | Pattern | Pattern |
|
||||
General | Plant City | EAST LIBERTY | 2117059 | 27341 | *****|*L | 31 | EAST LIBERTY | 2024-08-27 07:36:52.53 | 2105 | PlantCity | string | Pattern | Pattern |
|
||||
General | Plant State | OHIO | 2117056 | 27341 | *****|*[ALY] | 77 | OHIO | 2024-08-27 07:35:26.557 | 2105 | PlantState | string | Pattern | Pattern |
|
||||
General | Plant Country | UNITED STATES (USA) | 2117055 | 27341 | *****|*[ALY] | 75 | 6 | 2024-08-27 07:35:26.54 | 2105 | PlantCountry | lookup | Pattern | Pattern |
|
||||
General | Trim | SH-AWD A-Spec | 2074202 | 26929 | YE1H0 | 38 | SH-AWD A-Spec | 2024-05-21 10:24:09.057 | 2105 | Trim | string | Pattern | Pattern |
|
||||
Exterior / Body | Body Class | Sport Utility Vehicle (SUV)/Multi-Purpose Vehicle (MPV) | 2074165 | 26929 | Y[ED]*H | 5 | 7 | 2024-05-21 10:17:20.143 | 2105 | BodyClass | lookup | Pattern | Pattern |
|
||||
Exterior / Body | Doors | 5 | 2074166 | 26929 | Y[ED]*H | 14 | 5 | 2024-05-21 10:17:20.223 | 2105 | Doors | int | Pattern | Pattern |
|
||||
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 1D: 5,001 - 6,000 lb (2,268 - 2,722 kg) | 2074167 | 26929 | Y[ED]*H | 25 | 13 | 2024-05-21 10:17:20.27 | 2105 | GVWR | lookup | Pattern | Pattern |
|
||||
Exterior / Dimension | Gross Vehicle Weight Rating To | Class 1D: 5,001 - 6,000 lb (2,268 - 2,722 kg) | 2074180 | 26929 | Y[ED]*H | 190 | 13 | 2024-05-21 10:17:20.63 | 2105 | GVWR_to | lookup | Pattern | Pattern |
|
||||
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
|
||||
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
|
||||
Mechanical / Transmission | Transmission Speeds | 10 | 2074171 | 26929 | Y[ED]*H | 63 | 10 | 2024-05-21 10:17:20.343 | 2105 | TransmissionSpeeds | int | Pattern | Pattern |
|
||||
Mechanical / Transmission | Transmission Style | Automatic | 2074169 | 26929 | Y[ED]*H | 37 | 2 | 2024-05-21 10:17:20.303 | 2105 | TransmissionStyle | lookup | Pattern | Pattern |
|
||||
Mechanical / Drivetrain | Drive Type | 4WD/4-Wheel Drive/4x4 | 2074181 | 26929 | YE1 | 15 | 2 | 2024-05-21 10:18:30.127 | 2105 | DriveType | lookup | Pattern | Pattern |
|
||||
Engine | Other Engine Info | Direct Fuel Injection | 2074214 | 26929 | Y[ED][19] | 129 | Direct Fuel Injection | 2024-05-21 10:42:05.407 | 2105 | OtherEngineInfo | string | Pattern | Pattern |
|
||||
Engine | Displacement (CC) | 3500.0 | 2074206 | 26929 | Y[ED][19] | 11 | 3500.0 | | 2105 | DisplacementCC | decimal | Pattern | Conversion 4: 3.5 * 1000 |
|
||||
Engine | Displacement (CI) | 213.5831043315629938 | 2074206 | 26929 | Y[ED][19] | 12 | 213.5831043315629938 | | 2105 | DisplacementCI | decimal | Pattern | Conversion 7: 3.5 / 0.016387064 |
|
||||
Engine | Engine Number of Cylinders | 6 | 2074205 | 26929 | Y[ED][19] | 9 | 6 | 2024-05-21 10:42:05.137 | 2105 | EngineCylinders | int | Pattern | Pattern |
|
||||
Engine | Displacement (L) | 3.5 | 2074206 | 26929 | Y[ED][19] | 13 | 3.5 | 2024-05-21 10:42:05.157 | 2105 | DisplacementL | decimal | Pattern | Pattern |
|
||||
Engine | Engine Stroke Cycles | 4 | 2074207 | 26929 | Y[ED][19] | 17 | 4 | 2024-05-21 10:42:05.287 | 2105 | EngineCycles | int | Pattern | Pattern |
|
||||
Engine | Engine Model | J35Y5 | 2074208 | 26929 | Y[ED][19] | 18 | J35Y5 | 2024-05-21 10:42:05.303 | 2105 | EngineModel | string | Pattern | Pattern |
|
||||
Engine | Fuel Type - Primary | Gasoline | 2074209 | 26929 | Y[ED][19] | 24 | 4 | 2024-05-21 10:42:05.32 | 2105 | FuelTypePrimary | lookup | Pattern | Pattern |
|
||||
Engine | Valve Train Design | Single Overhead Cam (SOHC) | 2074210 | 26929 | Y[ED][19] | 62 | 4 | 2024-05-21 10:42:05.337 | 2105 | ValveTrainDesign | lookup | Pattern | Pattern |
|
||||
Engine | Engine Configuration | V-Shaped | 2074211 | 26929 | Y[ED][19] | 64 | 2 | 2024-05-21 10:42:05.353 | 2105 | EngineConfiguration | lookup | Pattern | Pattern |
|
||||
Engine | Engine Brake (hp) From | 290 | 2074212 | 26929 | Y[ED][19] | 71 | 290 | 2024-05-21 10:42:05.37 | 2105 | EngineHP | decimal | Pattern | Pattern |
|
||||
Engine | Cooling Type | Water | 2074213 | 26929 | Y[ED][19] | 122 | 2 | 2024-05-21 10:42:05.39 | 2105 | CoolingType | lookup | Pattern | Pattern |
|
||||
Engine | Engine Manufacturer | Honda | 2074215 | 26929 | Y[ED][19] | 146 | Honda | 2024-05-21 10:42:05.423 | 2105 | EngineManufacturer | string | Pattern | Pattern |
|
||||
Passive Safety System | Other Restraint System Info | Front: Seat Belts, Mid/Rear: Seat Belt and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2074179 | 26929 | Y[ED]*H | 121 | Front: Seat Belts, Mid/Rear: Seat Belt and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2024-05-21 10:17:20.61 | 2105 | OtherRestraintSystemInfo | string | Pattern | Pattern |
|
||||
Passive Safety System | Seat Belt Type | Manual | 2074174 | 26929 | Y[ED]*H | 79 | 1 | 2024-05-21 10:17:20.393 | 2105 | SeatBeltsAll | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st Row (Driver and Passenger) | 2074178 | 26929 | Y[ED]*H | 107 | 3 | 2024-05-21 10:17:20.593 | 2105 | AirBagLocSide | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Knee Air Bag Locations | 1st Row (Driver and Passenger) | 2074173 | 26929 | Y[ED]*H | 69 | 3 | 2024-05-21 10:17:20.377 | 2105 | AirBagLocKnee | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 2074172 | 26929 | Y[ED]*H | 65 | 3 | 2024-05-21 10:17:20.36 | 2105 | AirBagLocFront | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | 1st and 2nd and 3rd Rows | 2074170 | 26929 | Y[ED]*H | 55 | 5 | 2024-05-21 10:17:20.327 | 2105 | AirBagLocCurtain | lookup | Pattern | Pattern |
|
||||
Active Safety System | Keyless Ignition | Standard | 16875 | 5601 | | 176 | 1 | 2025-04-03 14:07:35.83 | | KeylessIgnition | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Electronic Stability Control (ESC) | Standard | 16875 | 5601 | | 99 | 1 | 2025-04-03 14:07:35.587 | | ESC | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Direct | 16875 | 5601 | | 168 | 1 | 2025-04-03 14:07:35.767 | | TPMS | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Event Data Recorder (EDR) | Standard | 16875 | 5601 | | 175 | 1 | 2025-04-03 14:07:35.817 | | EDR | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Standard | 16875 | 5601 | | 172 | 1 | 2025-04-03 14:07:35.803 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Anti-lock Braking System (ABS) | Standard | 16875 | 5601 | | 86 | 1 | 2025-04-03 14:07:35.56 | | ABS | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Traction Control | Standard | 16875 | 5601 | | 100 | 1 | 2025-04-03 14:07:35.6 | | TractionControl | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Standard | 16875 | 5601 | | 81 | 1 | 2025-04-03 14:07:35.55 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 16875 | 5601 | | 101 | 1 | 2025-04-03 14:07:35.613 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 16875 | 5601 | | 170 | 1 | 2025-04-03 14:07:35.78 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 16875 | 5601 | | 171 | 1 | 2025-04-03 14:07:35.79 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 16875 | 5601 | | 87 | 1 | 2025-04-03 14:07:35.573 | | CIB | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Standard | 16875 | 5601 | | 103 | 1 | 2025-04-03 14:07:35.64 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 16875 | 5601 | | 102 | 1 | 2025-04-03 14:07:35.627 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Centering Assistance | Standard | 16875 | 5601 | | 194 | 1 | 2025-04-03 14:07:36.043 | | LaneCenteringAssistance | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Blind Spot Intervention (BSI) | Standard | 16875 | 5601 | | 193 | 1 | 2025-04-03 14:07:36.03 | | BlindSpotIntervention | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Standard | 16875 | 5601 | | 192 | 1 | 2025-04-03 14:07:36.01 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Standard | 16875 | 5601 | | 183 | 1 | 2025-04-03 14:07:35.897 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 16875 | 5601 | | 104 | 1 | 2025-04-03 14:07:35.753 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 16875 | 5601 | | 178 | 3 | 2025-04-03 14:07:35.863 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 16875 | 5601 | | 177 | 1 | 2025-04-03 14:07:35.843 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 16875 | 5601 | | 179 | 1 | 2025-04-03 14:07:35.88 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs |
|
||||
(70 rows)
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
|
||||
-----------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+----------------------------------+--------------
|
||||
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
|
||||
| Additional Error Text | | | | | 156 | | | | AdditionalErrorText | string | Decoding | Corrections |
|
||||
| Vehicle Descriptor | 5YM13ET0*R9 | | | | 196 | 5YM13ET0*R9 | | | VehicleDescriptor | string | Decoding | Corrections |
|
||||
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
|
||||
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
|
||||
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
|
||||
General | Vehicle Type | MULTIPURPOSE PASSENGER VEHICLE (MPV) | | | 5YM | 39 | 7 | 2015-03-11 13:50:19.203 | 2020 | VehicleType | lookup | WMI | VehType |
|
||||
General | Plant State | SOUTH CAROLINA | 1765033 | 23981 | *****|*9 | 77 | SOUTH CAROLINA | 2022-06-22 14:28:01.55 | 2020 | PlantState | string | Pattern | Pattern |
|
||||
General | Plant Country | UNITED STATES (USA) | 1765032 | 23981 | *****|*9 | 75 | 6 | 2022-06-22 14:28:01.55 | 2020 | PlantCountry | lookup | Pattern | Pattern |
|
||||
General | Plant City | GREER | 1765031 | 23981 | *****|*9 | 31 | GREER | 2022-06-22 14:28:01.55 | 2020 | PlantCity | string | Pattern | Pattern |
|
||||
General | Manufacturer Name | BMW MANUFACTURER CORPORATION / BMW NORTH AMERICA | | | 5YM | 27 | 968 | | 2020 | Manufacturer | lookup | WMI | Manu. Name |
|
||||
General | Make | BMW | 1895508 | 25151 | 13ET0 | 26 | 452 | | | Make | lookup | WMI, Pattern | pattern - model |
|
||||
General | Model | X5 | 1895508 | 25151 | 13ET0 | 28 | 1717 | 2023-03-09 15:43:52.33 | 2020 | Model | lookup | Pattern | Pattern | f
|
||||
General | Trim | X5 M Competition | 2031372 | 25151 | 13ET0 | 38 | X5 M Competition | 2024-02-20 12:44:30.22 | 2020 | Trim | string | Pattern | Pattern | f
|
||||
General | Model Year | 2024 | | | ***X*|Y | 29 | 2024 | | | ModelYear | int | WMI, Input | ModelYear |
|
||||
Exterior / Body | Doors | 4 | 1895505 | 25151 | 13ET0 | 14 | 4 | 2023-03-09 15:43:52.28 | 2020 | Doors | int | Pattern | Pattern | f
|
||||
Exterior / Body | Body Class | Sport Utility Vehicle (SUV)/Multi-Purpose Vehicle (MPV) | 1895502 | 25151 | 13ET0 | 5 | 7 | 2023-03-09 15:43:52.227 | 2020 | BodyClass | lookup | Pattern | Pattern | f
|
||||
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 2E: 6,001 - 7,000 lb (2,722 - 3,175 kg) | 1895507 | 25151 | 13ET0 | 25 | 14 | 2023-03-09 15:43:52.313 | 2020 | GVWR | lookup | Pattern | Pattern | f
|
||||
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
|
||||
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
|
||||
Engine | Engine Number of Cylinders | 8 | 1895503 | 25151 | 13ET0 | 9 | 8 | 2023-03-09 15:43:52.243 | 2020 | EngineCylinders | int | Pattern | Pattern | f
|
||||
Engine | Displacement (CC) | 4400.0 | 1895504 | 25151 | 13ET0 | 11 | 4400.0 | | 2020 | DisplacementCC | decimal | Pattern | Conversion 4: 4.4 * 1000 |
|
||||
Engine | Displacement (CI) | 268.5044740168220494 | 1895504 | 25151 | 13ET0 | 12 | 268.5044740168220494 | | 2020 | DisplacementCI | decimal | Pattern | Conversion 7: 4.4 / 0.016387064 |
|
||||
Engine | Displacement (L) | 4.4 | 1895504 | 25151 | 13ET0 | 13 | 4.4 | 2023-03-09 15:43:52.26 | 2020 | DisplacementL | decimal | Pattern | Pattern | f
|
||||
Engine | Fuel Type - Primary | Gasoline | 1895506 | 25151 | 13ET0 | 24 | 4 | 2023-03-09 15:43:52.297 | 2020 | FuelTypePrimary | lookup | Pattern | Pattern | f
|
||||
Engine | Engine Brake (hp) From | 617 | 1895513 | 25151 | 13ET0 | 71 | 617 | 2023-03-09 15:43:52.537 | 2020 | EngineHP | decimal | Pattern | Pattern | f
|
||||
Passive Safety System | Seat Belt Type | Manual | 1895515 | 25151 | 13ET0 | 79 | 1 | 2023-03-09 15:43:52.57 | 2020 | SeatBeltsAll | lookup | Pattern | Pattern | f
|
||||
Passive Safety System | Other Restraint System Info | Seat Belt: All positions / Pretensioners, Side Airbags, Head Inflatable Restraint: Front Row & Rear Outboard Driver-side, Rear Outboard Passenger-side | 1895520 | 25151 | 13ET0 | 121 | Seat Belt: All positions / Pretensioners, Side Airbags, Head Inflatable Restraint: Front Row & Rear Outboard Driver-side, Rear Outboard Passenger-side | 2023-03-09 15:43:52.77 | 2020 | OtherRestraintSystemInfo | string | Pattern | Pattern | f
|
||||
Passive Safety System | Pretensioner | Yes | 1895514 | 25151 | 13ET0 | 78 | 1 | 2023-03-09 15:43:52.553 | 2020 | Pretensioner | lookup | Pattern | Pattern | f
|
||||
Passive Safety System / Air Bag Location | Knee Air Bag Locations | 1st Row (Driver and Passenger) | 1895512 | 25151 | 13ET0 | 69 | 3 | 2023-03-09 15:43:52.517 | 2020 | AirBagLocKnee | lookup | Pattern | Pattern | f
|
||||
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st and 2nd Rows | 1895519 | 25151 | 13ET0 | 107 | 4 | 2023-03-09 15:43:52.633 | 2020 | AirBagLocSide | lookup | Pattern | Pattern | f
|
||||
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | 1st and 2nd Rows | 1895510 | 25151 | 13ET0 | 55 | 4 | 2023-03-09 15:43:52.483 | 2020 | AirBagLocCurtain | lookup | Pattern | Pattern | f
|
||||
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 1895511 | 25151 | 13ET0 | 65 | 3 | 2023-03-09 15:43:52.5 | 2020 | AirBagLocFront | lookup | Pattern | Pattern | f
|
||||
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Direct | 14937 | 4656 | | 168 | 1 | 2024-03-12 13:41:17.253 | | TPMS | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Anti-lock Braking System (ABS) | Standard | 14937 | 4656 | | 86 | 1 | 2024-03-12 13:41:17.01 | | ABS | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Keyless Ignition | Standard | 14937 | 4656 | | 176 | 1 | 2024-03-12 13:41:17.317 | | KeylessIgnition | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Standard | 14937 | 4656 | | 172 | 1 | 2024-03-12 13:41:17.3 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Electronic Stability Control (ESC) | Standard | 14937 | 4656 | | 99 | 1 | 2024-03-12 13:41:17.05 | | ESC | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Traction Control | Standard | 14937 | 4656 | | 100 | 1 | 2024-03-12 13:41:17.183 | | TractionControl | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Optional | 14937 | 4656 | | 81 | 3 | 2024-03-12 13:41:16.993 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 14937 | 4656 | | 170 | 1 | 2024-03-12 13:41:17.267 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 14937 | 4656 | | 101 | 1 | 2024-03-12 13:41:17.2 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 14937 | 4656 | | 87 | 1 | 2024-03-12 13:41:17.02 | | CIB | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 14937 | 4656 | | 171 | 1 | 2024-03-12 13:41:17.28 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Centering Assistance | Optional | 14937 | 4656 | | 194 | 3 | 2024-03-12 13:41:17.543 | | LaneCenteringAssistance | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Blind Spot Warning (BSW) | Standard | 14937 | 4656 | | 88 | 1 | 2024-03-12 13:41:17.037 | | BlindSpotMon | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Optional | 14937 | 4656 | | 103 | 5 | 2024-03-12 13:41:17.227 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Blind Spot Intervention (BSI) | Optional | 14937 | 4656 | | 193 | 3 | 2024-03-12 13:41:17.53 | | BlindSpotIntervention | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 14937 | 4656 | | 102 | 1 | 2024-03-12 13:41:17.213 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Standard | 14937 | 4656 | | 192 | 1 | 2024-03-12 13:41:17.517 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Optional | 14937 | 4656 | | 183 | 2 | 2024-03-12 13:41:17.503 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 14937 | 4656 | | 104 | 1 | 2024-03-12 13:41:17.24 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 14937 | 4656 | | 179 | 1 | 2024-03-12 13:41:17.49 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 14937 | 4656 | | 177 | 1 | 2024-03-12 13:41:17.33 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 14937 | 4656 | | 178 | 3 | 2024-03-12 13:41:17.477 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs |
|
||||
(60 rows)
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
|
||||
-----------------------------------------------------+-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+----------------+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+----------------------------------+--------------
|
||||
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
|
||||
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
|
||||
| Additional Error Text | The Model Year decoded for this VIN may be incorrect. If you know the Model year, please enter it and decode again to get more accurate information. | | | | 156 | The Model Year decoded for this VIN may be incorrect. If you know the Model year, please enter it and decode again to get more accurate information. | | | AdditionalErrorText | string | Decoding | Corrections |
|
||||
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
|
||||
| Vehicle Descriptor | 3GTUUFEL*PG | | | | 196 | 3GTUUFEL*PG | | | VehicleDescriptor | string | Decoding | Corrections |
|
||||
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
|
||||
General | Manufacturer Name | GENERAL MOTORS LLC | | | 3GT | 27 | 984 | | 2066 | Manufacturer | lookup | WMI | Manu. Name |
|
||||
General | Model Year | 2023 | | | ***X*|Y | 29 | 2023 | | | ModelYear | int | WMI, Input | ModelYear |
|
||||
General | Make | GMC | 1947758 | 25646 | [NPRUV][HU] | 26 | 472 | | | Make | lookup | WMI, Pattern | pattern - model |
|
||||
General | Plant City | SILAO | 1947761 | 25646 | *****|*G | 31 | SILAO | 2023-07-11 14:39:30.503 | 2066 | PlantCity | string | Pattern | Pattern |
|
||||
General | Series | 1500 | 1947762 | 25646 | [NPRUV][HU] | 34 | 1500 | 2023-07-11 14:39:30.503 | 2066 | Series | string | Pattern | Pattern |
|
||||
General | Trim | AT4X | 1947769 | 25646 | *UF | 38 | AT4X | 2023-07-11 14:39:30.503 | 2066 | Trim | string | Pattern | Pattern |
|
||||
General | Plant Country | MEXICO | 1947783 | 25646 | *****|*G | 75 | 12 | 2023-07-11 14:39:30.503 | 2066 | PlantCountry | lookup | Pattern | Pattern |
|
||||
General | Plant State | GUANAJUATO | 1947786 | 25646 | *****|*G | 77 | GUANAJUATO | 2023-07-11 14:39:30.503 | 2066 | PlantState | string | Pattern | Pattern |
|
||||
General | Vehicle Type | TRUCK | | | 3GT | 39 | 3 | 2015-03-24 15:16:38.563 | 2066 | VehicleType | lookup | WMI | VehType |
|
||||
General | Model | Sierra | 1947758 | 25646 | [NPRUV][HU] | 28 | 1857 | 2023-07-11 14:46:06.22 | 2066 | Model | lookup | Pattern | Pattern |
|
||||
Exterior / Body | Body Class | Pickup | 1947732 | 25646 | [NPRUV][HU] | 5 | 60 | 2023-07-11 14:46:06.22 | 2066 | BodyClass | lookup | Pattern | Pattern |
|
||||
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 2F: 7,001 - 8,000 lb (3,175 - 3,629 kg) | 1947756 | 25646 | [UV][HU] | 25 | 15 | 2023-07-11 14:46:18.457 | 2066 | GVWR | lookup | Pattern | Pattern |
|
||||
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
|
||||
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
|
||||
Mechanical / Drivetrain | Drive Type | 4WD/4-Wheel Drive/4x4 | 1947745 | 25646 | [NPRUV]U[A-J9] | 15 | 2 | 2023-07-11 14:39:30.503 | 2066 | DriveType | lookup | Pattern | Pattern |
|
||||
Mechanical / Brake | Brake System Type | Hydraulic | 1947772 | 25646 | [NPRUV][HU] | 42 | 2 | 2023-07-11 14:46:06.22 | 2066 | BrakeSystemType | lookup | Pattern | Pattern |
|
||||
Engine | Other Engine Info | DI DFM, ALUM, GEN 5 | 1947794 | 25646 | *[HU]**L | 129 | DI DFM, ALUM, GEN 5 | 2023-07-11 14:39:30.503 | 2066 | OtherEngineInfo | string | Pattern | Pattern |
|
||||
Engine | Displacement (CI) | 378.3472133873401605 | 1947739 | 25646 | *[HU]**L | 12 | 378.3472133873401605 | | 2066 | DisplacementCI | decimal | Pattern | Conversion 7: 6.2 / 0.016387064 |
|
||||
Engine | Engine Configuration | V-Shaped | 1947780 | 25646 | *[HU]**L | 64 | 2 | 2023-07-11 14:39:30.503 | 2066 | EngineConfiguration | lookup | Pattern | Pattern |
|
||||
Engine | Fuel Type - Primary | Gasoline | 1947755 | 25646 | *[HU]**L | 24 | 4 | 2023-07-11 14:39:30.503 | 2066 | FuelTypePrimary | lookup | Pattern | Pattern |
|
||||
Engine | Engine Model | L87 | 1947746 | 25646 | *[HU]**L | 18 | L87 | 2023-07-11 14:39:30.503 | 2066 | EngineModel | string | Pattern | Pattern |
|
||||
Engine | Displacement (L) | 6.2 | 1947739 | 25646 | *[HU]**L | 13 | 6.2 | 2023-07-11 14:39:30.503 | 2066 | DisplacementL | decimal | Pattern | Pattern |
|
||||
Engine | Engine Number of Cylinders | 8 | 1947733 | 25646 | *[HU]**L | 9 | 8 | 2023-07-11 14:39:30.503 | 2066 | EngineCylinders | int | Pattern | Pattern |
|
||||
Engine | Displacement (CC) | 6200.0 | 1947739 | 25646 | *[HU]**L | 11 | 6200.0 | | 2066 | DisplacementCC | decimal | Pattern | Conversion 4: 6.2 * 1000 |
|
||||
Passive Safety System | Seat Belt Type | Manual | 1947788 | 25646 | *[HU]*E | 79 | 1 | 2023-07-11 14:39:30.503 | 2066 | SeatBeltsAll | lookup | Pattern | Pattern |
|
||||
Passive Safety System | Other Restraint System Info | AY0 - Active Manual Belts | 1947793 | 25646 | *[HU]*E | 121 | AY0 - Active Manual Belts | 2023-07-11 14:39:30.503 | 2066 | OtherRestraintSystemInfo | string | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | All Rows | 1947773 | 25646 | *[HU]*E | 55 | 6 | 2023-07-11 14:39:30.503 | 2066 | AirBagLocCurtain | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 1947781 | 25646 | *[HU]*E | 65 | 3 | 2023-07-11 14:39:30.503 | 2066 | AirBagLocFront | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st Row (Driver and Passenger) | 1947792 | 25646 | *[HU]*E | 107 | 3 | 2023-07-11 14:39:30.503 | 2066 | AirBagLocSide | lookup | Pattern | Pattern |
|
||||
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Optional | 13609 | 4140 | | 172 | 3 | 2024-01-31 16:04:17.207 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System | Electronic Stability Control (ESC) | Standard | 13609 | 4140 | | 99 | 1 | 2024-01-31 16:04:16.897 | | ESC | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System | Anti-lock Braking System (ABS) | Standard | 13609 | 4140 | | 86 | 1 | 2024-01-31 16:04:16.85 | | ABS | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System | Traction Control | Standard | 13609 | 4140 | | 100 | 1 | 2024-01-31 16:04:16.91 | | TractionControl | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System | Keyless Ignition | Standard | 13609 | 4140 | | 176 | 1 | 2024-01-31 16:04:17.257 | | KeylessIgnition | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Direct | 13609 | 4140 | | 168 | 1 | 2024-01-31 16:04:17.157 | | TPMS | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System | Event Data Recorder (EDR) | Standard | 13609 | 4140 | | 175 | 1 | 2024-01-31 16:04:17.24 | | EDR | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Optional | 13609 | 4140 | | 81 | 3 | 2024-01-31 16:04:16.837 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 13609 | 4140 | | 170 | 1 | 2024-01-31 16:04:17.173 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 13609 | 4140 | | 87 | 1 | 2024-01-31 16:04:16.863 | | CIB | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 13609 | 4140 | | 171 | 1 | 2024-01-31 16:04:17.19 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 13609 | 4140 | | 101 | 1 | 2024-01-31 16:04:16.923 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Standard | 13609 | 4140 | | 103 | 1 | 2024-01-31 16:04:17.103 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 13609 | 4140 | | 102 | 1 | 2024-01-31 16:04:16.94 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lane and Side Assist | Blind Spot Warning (BSW) | Optional | 13609 | 4140 | | 88 | 3 | 2024-01-31 16:04:16.88 | | BlindSpotMon | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Optional | 13609 | 4140 | | 192 | 3 | 2024-01-31 16:04:17.487 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 13609 | 4140 | | 104 | 1 | 2024-01-31 16:04:17.12 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Optional | 13609 | 4140 | | 183 | 2 | 2024-01-31 16:04:17.47 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Backing Up and Parking | Parking Assist | Optional | 13609 | 4140 | | 105 | 3 | 2024-01-31 16:04:17.14 | | ParkAssist | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / 911 Notification | Automatic Crash Notification (ACN) / Advanced Automatic Crash Notification (AACN) | Standard | 13609 | 4140 | | 174 | 1 | 2024-01-31 16:04:17.22 | | CAN_AACN | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 13609 | 4140 | | 178 | 3 | 2024-01-31 16:04:17.427 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lighting Technologies | Adaptive Driving Beam (ADB) | Standard | 13609 | 4140 | | 180 | 1 | 2024-01-31 16:04:17.453 | | AdaptiveDrivingBeam | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 13609 | 4140 | | 177 | 1 | 2024-01-31 16:04:17.273 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs | f
|
||||
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 13609 | 4140 | | 179 | 1 | 2024-01-31 16:04:17.44 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs | f
|
||||
(64 rows)
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
groupname | variable | value | itempatternid | itemvinschemaid | itemkeys | itemelementid | itemattributeid | itemcreatedon | itemwmiid | code | datatype | decode | itemsource | itemtobeqced
|
||||
-----------------------------------------------------+--------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------+---------------+-----------------+----------+---------------+----------------------------------------------------------------------------------------------------------------------------+-------------------------+-----------+-------------------------------------+----------+--------------+--------------------------------+--------------
|
||||
| Error Text | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | | 191 | 0 - VIN decoded clean. Check Digit (9th position) is correct | | | ErrorText | string | Decoding | Corrections |
|
||||
| Additional Error Text | | | | | 156 | | | | AdditionalErrorText | string | Decoding | Corrections |
|
||||
| Error Code | 0 | | | | 143 | 0 | | | ErrorCode | lookup | Decoding | Corrections |
|
||||
| Possible Values | | | | | 144 | | | | PossibleValues | string | Decoding | Corrections |
|
||||
| Vehicle Descriptor | 2HGFE4F8*SH | | | | 196 | 2HGFE4F8*SH | | | VehicleDescriptor | string | Decoding | Corrections |
|
||||
| Suggested VIN | | | | | 142 | | | | SuggestedVIN | string | Decoding | Corrections |
|
||||
General | Plant City | ALLISTON | 2123543 | 27387 | *****|*H | 31 | ALLISTON | 2024-09-16 08:37:51.523 | 2096 | PlantCity | string | Pattern | Pattern |
|
||||
General | Trim | Sport Hybrid / Sport Touring Hybrid | 2086003 | 27007 | FE4F8 | 38 | Sport Hybrid / Sport Touring Hybrid | 2024-06-17 10:36:46.323 | 2096 | Trim | string | Pattern | Pattern |
|
||||
General | Plant Country | CANADA | 2123544 | 27387 | *****|*H | 75 | 1 | 2024-09-16 08:37:51.54 | 2096 | PlantCountry | lookup | Pattern | Pattern |
|
||||
General | Plant State | ONTARIO | 2123545 | 27387 | *****|*H | 77 | ONTARIO | 2024-09-16 08:37:51.553 | 2096 | PlantState | string | Pattern | Pattern |
|
||||
General | Model Year | 2025 | | | ***X*|Y | 29 | 2025 | | | ModelYear | int | WMI, Input | ModelYear |
|
||||
General | Manufacturer Name | HONDA OF CANADA MFG., A DIVISION OF HONDA CANADA INC. | | | 2HG | 27 | 990 | | 2096 | Manufacturer | lookup | WMI | Manu. Name |
|
||||
General | Make | HONDA | 2086001 | 27007 | FE4F8 | 26 | 474 | | | Make | lookup | WMI, Pattern | pattern - model |
|
||||
General | Vehicle Type | PASSENGER CAR | | | 2HG | 39 | 2 | 2015-03-26 16:57:39.147 | 2096 | VehicleType | lookup | WMI | VehType |
|
||||
General | Model | Civic | 2086001 | 27007 | FE4F8 | 28 | 1863 | 2024-06-17 10:36:46.287 | 2096 | Model | lookup | Pattern | Pattern |
|
||||
Exterior / Body | Doors | 4 | 2085995 | 27007 | FE4F8 | 14 | 4 | 2024-06-17 10:36:46.163 | 2096 | Doors | int | Pattern | Pattern |
|
||||
Exterior / Body | Body Class | Sedan/Saloon | 2085992 | 27007 | FE4F8 | 5 | 13 | 2024-06-17 10:36:45.88 | 2096 | BodyClass | lookup | Pattern | Pattern |
|
||||
Exterior / Dimension | Gross Vehicle Weight Rating From | Class 1C: 4,001 - 5,000 lb (1,814 - 2,268 kg) | 2086000 | 27007 | FE4F8 | 25 | 12 | 2024-06-17 10:36:46.267 | 2096 | GVWR | lookup | Pattern | Pattern |
|
||||
Exterior / Dimension | Gross Vehicle Weight Rating To | Class 1C: 4,001 - 5,000 lb (1,814 - 2,268 kg) | 2086015 | 27007 | FE4F8 | 190 | 12 | 2024-06-17 10:36:46.713 | 2096 | GVWR_to | lookup | Pattern | Pattern |
|
||||
Exterior / Truck | Bed Type | Not Applicable | | | | 3 | 0 | 2019-12-21 20:26:30.583 | | BedType | lookup | Pattern | Default |
|
||||
Exterior / Truck | Cab Type | Not Applicable | | | | 4 | 0 | 2019-12-21 20:26:30.583 | | BodyCabType | lookup | Pattern | Default |
|
||||
Exterior / Trailer | Trailer Body Type | Not Applicable | | | | 117 | 0 | 2019-12-21 20:26:30.583 | | TrailerBodyType | lookup | Pattern | Default |
|
||||
Exterior / Trailer | Trailer Type Connection | Not Applicable | | | | 116 | 0 | 2019-12-21 20:26:30.583 | | TrailerType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Suspension Type | Not Applicable | | | | 152 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleSuspensionType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Custom Motorcycle Type | Not Applicable | | | | 151 | 0 | 2019-12-21 20:26:30.583 | | CustomMotorcycleType | lookup | Pattern | Default |
|
||||
Exterior / Motorcycle | Motorcycle Chassis Type | Not Applicable | | | | 153 | 0 | 2019-12-21 20:26:30.583 | | MotorcycleChassisType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Floor Configuration Type | Not Applicable | | | | 148 | 0 | 2019-12-21 20:26:30.583 | | BusFloorConfigType | lookup | Pattern | Default |
|
||||
Exterior / Bus | Bus Type | Not Applicable | | | | 149 | 0 | 2019-12-21 20:26:30.583 | | BusType | lookup | Pattern | Default |
|
||||
Mechanical / Transmission | Transmission Style | Electronic Continuously Variable (e-CVT) | 2086002 | 27007 | FE4F8 | 37 | 4 | 2024-06-17 10:36:46.307 | 2096 | TransmissionStyle | lookup | Pattern | Pattern |
|
||||
Mechanical / Drivetrain | Drive Type | 4x2 | 2085996 | 27007 | FE4F8 | 15 | 7 | 2024-06-17 10:36:46.18 | 2096 | DriveType | lookup | Pattern | Pattern |
|
||||
Mechanical / Battery | EV Drive Unit | Single Motor | 2086065 | 27007 | FE4F8 | 72 | 2 | 2024-06-17 10:59:49.297 | 2096 | EVDriveUnit | lookup | Pattern | Pattern |
|
||||
Engine | Engine Manufacturer | Honda | 2086014 | 27007 | FE4F8 | 146 | Honda | 2024-06-17 10:36:46.69 | 2096 | EngineManufacturer | string | Pattern | Pattern |
|
||||
Engine | Displacement (CC) | 2000 | 2085994 | 27007 | FE4F8 | 11 | 2000 | | 2096 | DisplacementCC | decimal | Pattern | Conversion 4: 2 * 1000 |
|
||||
Engine | Displacement (CI) | 122.0474881894645679 | 2085994 | 27007 | FE4F8 | 12 | 122.0474881894645679 | | 2096 | DisplacementCI | decimal | Pattern | Conversion 7: 2 / 0.016387064 |
|
||||
Engine | Engine Number of Cylinders | 4 | 2085993 | 27007 | FE4F8 | 9 | 4 | 2024-06-17 10:36:45.96 | 2096 | EngineCylinders | int | Pattern | Pattern |
|
||||
Engine | Displacement (L) | 2 | 2085994 | 27007 | FE4F8 | 13 | 2 | 2024-06-17 10:36:46.023 | 2096 | DisplacementL | decimal | Pattern | Pattern |
|
||||
Engine | Engine Stroke Cycles | 4 | 2085997 | 27007 | FE4F8 | 17 | 4 | 2024-06-17 10:36:46.203 | 2096 | EngineCycles | int | Pattern | Pattern |
|
||||
Engine | Engine Model | LFC3 | 2085998 | 27007 | FE4F8 | 18 | LFC3 | 2024-06-17 10:36:46.223 | 2096 | EngineModel | string | Pattern | Pattern |
|
||||
Engine | Fuel Type - Primary | Gasoline | 2085999 | 27007 | FE4F8 | 24 | 4 | 2024-06-17 10:36:46.247 | 2096 | FuelTypePrimary | lookup | Pattern | Pattern |
|
||||
Engine | Valve Train Design | Dual Overhead Cam (DOHC) | 2086004 | 27007 | FE4F8 | 62 | 2 | 2024-06-17 10:36:46.343 | 2096 | ValveTrainDesign | lookup | Pattern | Pattern |
|
||||
Engine | Engine Configuration | In-Line | 2086005 | 27007 | FE4F8 | 64 | 1 | 2024-06-17 10:36:46.363 | 2096 | EngineConfiguration | lookup | Pattern | Pattern |
|
||||
Engine | Fuel Type - Secondary | Electric | 2086006 | 27007 | FE4F8 | 66 | 2 | 2024-06-17 10:36:46.527 | 2096 | FuelTypeSecondary | lookup | Pattern | Pattern |
|
||||
Engine | Engine Brake (hp) From | 141 | 2086007 | 27007 | FE4F8 | 71 | 141 | 2024-06-17 10:36:46.543 | 2096 | EngineHP | decimal | Pattern | Pattern |
|
||||
Engine | Cooling Type | Water | 2086011 | 27007 | FE4F8 | 122 | 2 | 2024-06-17 10:36:46.63 | 2096 | CoolingType | lookup | Pattern | Pattern |
|
||||
Engine | Electrification Level | Strong HEV (Hybrid Electric Vehicle) | 2086012 | 27007 | FE4F8 | 126 | 2 | 2024-06-17 10:36:46.65 | 2096 | ElectrificationLevel | lookup | Pattern | Pattern |
|
||||
Engine | Other Engine Info | Direct Fuel Injection / Motor: 135kW | 2086013 | 27007 | FE4F8 | 129 | Direct Fuel Injection / Motor: 135kW | 2024-06-17 10:36:46.67 | 2096 | OtherEngineInfo | string | Pattern | Pattern |
|
||||
Passive Safety System | Other Restraint System Info | Front: Seat Belt / Rear: Seat Belt , Side Air Bag and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2083833 | 27007 | FE[24]F | 121 | Front: Seat Belt / Rear: Seat Belt , Side Air Bag and Side Curtain Air Bag (Outer positions) / Seat Belt (Center position) | 2024-06-17 10:20:27.853 | 2096 | OtherRestraintSystemInfo | string | Pattern | Pattern |
|
||||
Passive Safety System | Seat Belt Type | Manual | 2083828 | 27007 | FE[24]F | 79 | 1 | 2024-06-17 10:20:27.853 | 2096 | SeatBeltsAll | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Side Air Bag Locations | 1st and 2nd Rows | 2083832 | 27007 | FE[24]F | 107 | 4 | 2024-06-17 10:20:27.853 | 2096 | AirBagLocSide | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Knee Air Bag Locations | 1st Row (Driver and Passenger) | 2083826 | 27007 | FE[24]F | 69 | 3 | 2024-06-17 10:05:30.837 | 2096 | AirBagLocKnee | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Front Air Bag Locations | 1st Row (Driver and Passenger) | 2083825 | 27007 | FE[24]F | 65 | 3 | 2024-06-17 10:05:30.837 | 2096 | AirBagLocFront | lookup | Pattern | Pattern |
|
||||
Passive Safety System / Air Bag Location | Curtain Air Bag Locations | 1st and 2nd Rows | 2083822 | 27007 | FE[24]F | 55 | 4 | 2024-06-17 10:05:30.837 | 2096 | AirBagLocCurtain | lookup | Pattern | Pattern |
|
||||
Active Safety System | Anti-lock Braking System (ABS) | Standard | 16871 | 5597 | | 86 | 1 | 2025-04-03 14:40:02.47 | | ABS | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Traction Control | Standard | 16871 | 5597 | | 100 | 1 | 2025-04-03 14:40:02.52 | | TractionControl | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Automatic Pedestrian Alerting Sound (for Hybrid and EV only) | Standard | 17451 | 5597 | | 173 | 1 | 2025-04-01 10:29:03.217 | | AutomaticPedestrianAlertingSound | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Tire Pressure Monitoring System (TPMS) Type | Indirect | 16871 | 5597 | | 168 | 2 | 2025-04-03 14:40:02.6 | | TPMS | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Keyless Ignition | Standard | 16871 | 5597 | | 176 | 1 | 2025-04-03 14:40:02.793 | | KeylessIgnition | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Electronic Stability Control (ESC) | Standard | 16871 | 5597 | | 99 | 1 | 2025-04-03 14:40:02.503 | | ESC | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Auto-Reverse System for Windows and Sunroofs | Standard | 16871 | 5597 | | 172 | 1 | 2025-04-03 14:40:02.763 | | AutoReverseSystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System | Event Data Recorder (EDR) | Standard | 16871 | 5597 | | 175 | 1 | 2025-04-03 14:40:02.78 | | EDR | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Maintaining Safe Distance | Adaptive Cruise Control (ACC) | Standard | 16871 | 5597 | | 81 | 1 | 2025-04-03 14:40:02.337 | | AdaptiveCruiseControl | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Forward Collision Warning (FCW) | Standard | 16871 | 5597 | | 101 | 1 | 2025-04-03 14:40:02.537 | | ForwardCollisionWarning | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Dynamic Brake Support (DBS) | Standard | 16871 | 5597 | | 170 | 1 | 2025-04-03 14:40:02.61 | | DynamicBrakeSupport | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Crash Imminent Braking (CIB) | Standard | 16871 | 5597 | | 87 | 1 | 2025-04-03 14:40:02.487 | | CIB | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Forward Collision Prevention | Pedestrian Automatic Emergency Braking (PAEB) | Standard | 16871 | 5597 | | 171 | 1 | 2025-04-03 14:40:02.627 | | PedestrianAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Keeping Assistance (LKA) | Standard | 16871 | 5597 | | 103 | 1 | 2025-04-03 14:40:02.567 | | LaneKeepSystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Blind Spot Intervention (BSI) | Standard | 17451 | 5597 | | 193 | 1 | 2025-04-01 10:29:03.243 | | BlindSpotIntervention | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Blind Spot Warning (BSW) | Standard | 17451 | 5597 | | 88 | 1 | 2025-04-01 10:29:03.203 | | BlindSpotMon | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lane and Side Assist | Lane Departure Warning (LDW) | Standard | 16871 | 5597 | | 102 | 1 | 2025-04-03 14:40:02.55 | | LaneDepartureWarning | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Backup Camera | Standard | 16871 | 5597 | | 104 | 1 | 2025-04-03 14:40:02.583 | | RearVisibilitySystem | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Rear Automatic Emergency Braking | Standard | 16871 | 5597 | | 192 | 1 | 2025-04-03 14:40:02.877 | | RearAutomaticEmergencyBraking | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Backing Up and Parking | Rear Cross Traffic Alert | Standard | 17451 | 5597 | | 183 | 1 | 2025-04-01 10:29:03.23 | | RearCrossTrafficAlert | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Semiautomatic Headlamp Beam Switching | Standard | 16871 | 5597 | | 179 | 1 | 2025-04-03 14:40:02.843 | | SemiautomaticHeadlampBeamSwitching | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Daytime Running Light (DRL) | Standard | 16871 | 5597 | | 177 | 1 | 2025-04-03 14:40:02.81 | | DaytimeRunningLight | lookup | Pattern | Vehicle Specs |
|
||||
Active Safety System / Lighting Technologies | Headlamp Light Source | LED | 16871 | 5597 | | 178 | 3 | 2025-04-03 14:40:02.83 | | LowerBeamHeadlampLightSource | lookup | Pattern | Vehicle Specs |
|
||||
(75 rows)
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,241 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate SQL import files from a VehAPI snapshot SQLite database.
|
||||
|
||||
Reads observed compatibility pairs from the snapshot (trim-filtered engine<->transmission pairs)
|
||||
and produces:
|
||||
- output/01_engines.sql
|
||||
- output/02_transmissions.sql
|
||||
- output/03_vehicle_options.sql
|
||||
|
||||
No legacy JSON or network calls are used. The snapshot path is provided via CLI flag.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Sequence
|
||||
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate SQL files from a VehAPI snapshot (SQLite).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--snapshot-path",
|
||||
type=Path,
|
||||
default=os.environ.get("SNAPSHOT_PATH"),
|
||||
help="Path to snapshots/<date>/snapshot.sqlite produced by vehapi_fetch_snapshot.py (or env SNAPSHOT_PATH)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=Path("output"),
|
||||
help="Directory to write SQL output files (default: output)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_pairs(snapshot_path: Path) -> List[sqlite3.Row]:
|
||||
if not snapshot_path.exists():
|
||||
raise FileNotFoundError(f"Snapshot not found: {snapshot_path}")
|
||||
|
||||
# Open in immutable mode to prevent any write attempts on read-only filesystems
|
||||
absolute_path = snapshot_path.resolve()
|
||||
uri = f"file:{absolute_path}?immutable=1"
|
||||
conn = sqlite3.connect(uri, uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
year,
|
||||
make,
|
||||
model,
|
||||
trim,
|
||||
engine_display,
|
||||
engine_canon,
|
||||
engine_bucket,
|
||||
trans_display,
|
||||
trans_canon,
|
||||
trans_bucket
|
||||
FROM pairs
|
||||
ORDER BY year, make, model, trim, engine_canon, trans_canon
|
||||
"""
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
except sqlite3.Error as exc:
|
||||
raise RuntimeError(f"Failed to read pairs from snapshot: {exc}") from exc
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if not rows:
|
||||
raise ValueError("Snapshot contains no rows in pairs table.")
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def choose_engine_label(engine_display: str, engine_bucket: str, engine_canon: str) -> str:
|
||||
"""
|
||||
Use VehAPI display string when present, otherwise fall back to the bucket label,
|
||||
and finally to the canonical key to avoid empty names.
|
||||
"""
|
||||
if engine_display:
|
||||
return engine_display
|
||||
if engine_bucket:
|
||||
return engine_bucket
|
||||
return engine_canon
|
||||
|
||||
|
||||
def choose_trans_label(trans_display: str, trans_bucket: str, trans_canon: str) -> str:
|
||||
if trans_display:
|
||||
return trans_display
|
||||
if trans_bucket:
|
||||
return trans_bucket
|
||||
return trans_canon
|
||||
|
||||
|
||||
def build_engine_dimension(rows: Sequence[sqlite3.Row]) -> Dict[str, Dict]:
|
||||
engines: Dict[str, Dict] = {}
|
||||
for row in rows:
|
||||
canon = row["engine_canon"]
|
||||
if canon is None or canon == "":
|
||||
raise ValueError(f"Missing engine_canon for row: {dict(row)}")
|
||||
if canon in engines:
|
||||
continue
|
||||
engines[canon] = {
|
||||
"id": len(engines) + 1,
|
||||
"name": choose_engine_label(row["engine_display"], row["engine_bucket"], canon),
|
||||
"fuel_type": row["engine_bucket"] or None,
|
||||
}
|
||||
return engines
|
||||
|
||||
|
||||
def build_transmission_dimension(rows: Sequence[sqlite3.Row]) -> Dict[str, Dict]:
|
||||
transmissions: Dict[str, Dict] = {}
|
||||
for row in rows:
|
||||
canon = row["trans_canon"]
|
||||
if canon is None or canon == "":
|
||||
raise ValueError(f"Missing trans_canon for row: {dict(row)}")
|
||||
if canon in transmissions:
|
||||
continue
|
||||
transmissions[canon] = {
|
||||
"id": len(transmissions) + 1,
|
||||
"type": choose_trans_label(row["trans_display"], row["trans_bucket"], canon),
|
||||
}
|
||||
return transmissions
|
||||
|
||||
|
||||
def build_vehicle_options(
|
||||
rows: Sequence[sqlite3.Row],
|
||||
engine_map: Dict[str, Dict],
|
||||
trans_map: Dict[str, Dict],
|
||||
) -> List[Dict]:
|
||||
options: List[Dict] = []
|
||||
for row in rows:
|
||||
engine_canon = row["engine_canon"]
|
||||
trans_canon = row["trans_canon"]
|
||||
options.append(
|
||||
{
|
||||
"year": int(row["year"]),
|
||||
"make": row["make"],
|
||||
"model": row["model"],
|
||||
"trim": row["trim"],
|
||||
"engine_id": engine_map[engine_canon]["id"],
|
||||
"transmission_id": trans_map[trans_canon]["id"],
|
||||
}
|
||||
)
|
||||
return options
|
||||
|
||||
|
||||
def sql_value(value):
|
||||
if value is None:
|
||||
return "NULL"
|
||||
if isinstance(value, str):
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
return str(value)
|
||||
|
||||
|
||||
def chunked(seq: Iterable[Dict], size: int) -> Iterable[List[Dict]]:
|
||||
chunk: List[Dict] = []
|
||||
for item in seq:
|
||||
chunk.append(item)
|
||||
if len(chunk) >= size:
|
||||
yield chunk
|
||||
chunk = []
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
|
||||
def write_insert_file(
|
||||
path: Path,
|
||||
table: str,
|
||||
columns: Sequence[str],
|
||||
rows: Sequence[Dict],
|
||||
):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(f"-- Auto-generated by etl_generate_sql.py\n")
|
||||
if not rows:
|
||||
f.write(f"-- No rows for {table}\n")
|
||||
return
|
||||
|
||||
for batch in chunked(rows, BATCH_SIZE):
|
||||
values_sql = ",\n".join(
|
||||
"(" + ",".join(sql_value(row[col]) for col in columns) + ")"
|
||||
for row in batch
|
||||
)
|
||||
f.write(f"INSERT INTO {table} ({', '.join(columns)}) VALUES\n{values_sql};\n\n")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
snapshot_path: Path = args.snapshot_path
|
||||
output_dir: Path = args.output_dir
|
||||
if snapshot_path is None:
|
||||
raise SystemExit("Snapshot path is required. Pass --snapshot-path or set SNAPSHOT_PATH.")
|
||||
|
||||
print(f"Reading snapshot: {snapshot_path}")
|
||||
rows = load_pairs(snapshot_path)
|
||||
years = sorted({int(row["year"]) for row in rows})
|
||||
print(f" Loaded {len(rows):,} observed engine<->transmission pairs across {len(years)} years")
|
||||
|
||||
engines = build_engine_dimension(rows)
|
||||
transmissions = build_transmission_dimension(rows)
|
||||
vehicle_options = build_vehicle_options(rows, engines, transmissions)
|
||||
|
||||
print(f"Engines: {len(engines):,}")
|
||||
print(f"Transmissions: {len(transmissions):,}")
|
||||
print(f"Vehicle options (observed pairs): {len(vehicle_options):,}")
|
||||
|
||||
write_insert_file(
|
||||
output_dir / "01_engines.sql",
|
||||
"engines",
|
||||
["id", "name", "fuel_type"],
|
||||
engines.values(),
|
||||
)
|
||||
write_insert_file(
|
||||
output_dir / "02_transmissions.sql",
|
||||
"transmissions",
|
||||
["id", "type"],
|
||||
transmissions.values(),
|
||||
)
|
||||
write_insert_file(
|
||||
output_dir / "03_vehicle_options.sql",
|
||||
"vehicle_options",
|
||||
["year", "make", "model", "trim", "engine_id", "transmission_id"],
|
||||
vehicle_options,
|
||||
)
|
||||
|
||||
print("\nSQL files generated:")
|
||||
print(f" - {output_dir / '01_engines.sql'}")
|
||||
print(f" - {output_dir / '02_transmissions.sql'}")
|
||||
print(f" - {output_dir / '03_vehicle_options.sql'}")
|
||||
print(f"\nYear coverage: {years[0]}-{years[-1]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
238374
data/vehicle-etl/import/vehicle-catalog-master.csv
Normal file
238374
data/vehicle-etl/import/vehicle-catalog-master.csv
Normal file
File diff suppressed because it is too large
Load Diff
117
data/vehicle-etl/import_catalog.sh
Executable file
117
data/vehicle-etl/import_catalog.sh
Executable file
@@ -0,0 +1,117 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Vehicle Catalog CSV Bulk Import Wrapper
|
||||
#
|
||||
# Copies CSV file into mvp-backend container and executes bulk import script.
|
||||
# Handles large CSV files (250k+ rows) that fail in web import.
|
||||
#
|
||||
# Usage:
|
||||
# ./import_catalog.sh <path_to_csv_file>
|
||||
#
|
||||
# Example:
|
||||
# ./import_catalog.sh data/vehicle-etl/import/vehicle-catalog-master.csv
|
||||
#
|
||||
# Requirements:
|
||||
# - mvp-backend container must be running
|
||||
# - CSV file must have headers: year, make, model, trim
|
||||
# - Optional headers: engine_name, transmission_type
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER="mvp-backend"
|
||||
TEMP_CSV_PATH="/tmp/catalog-import.csv"
|
||||
SCRIPT_PATH="dist/features/admin/scripts/bulk-import-catalog.js"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print error and exit
|
||||
error() {
|
||||
echo -e "${RED}Error: $1${NC}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Print success message
|
||||
success() {
|
||||
echo -e "${GREEN}$1${NC}"
|
||||
}
|
||||
|
||||
# Print warning message
|
||||
warn() {
|
||||
echo -e "${YELLOW}$1${NC}"
|
||||
}
|
||||
|
||||
# Check if CSV file argument provided
|
||||
if [ $# -eq 0 ]; then
|
||||
error "No CSV file specified.
|
||||
|
||||
Usage: $0 <path_to_csv_file>
|
||||
|
||||
Example:
|
||||
$0 data/vehicle-etl/import/vehicle-catalog-master.csv"
|
||||
fi
|
||||
|
||||
CSV_FILE="$1"
|
||||
|
||||
# Validate CSV file exists
|
||||
if [ ! -f "$CSV_FILE" ]; then
|
||||
error "CSV file not found: $CSV_FILE"
|
||||
fi
|
||||
|
||||
# Get absolute path to CSV file
|
||||
CSV_FILE_ABS=$(cd "$(dirname "$CSV_FILE")" && pwd)/$(basename "$CSV_FILE")
|
||||
|
||||
# Check if container is running
|
||||
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER}$"; then
|
||||
error "Container '${CONTAINER}' is not running. Start it with: make start"
|
||||
fi
|
||||
|
||||
echo "=========================================="
|
||||
echo "Vehicle Catalog Bulk Import"
|
||||
echo "=========================================="
|
||||
echo "CSV File: $CSV_FILE_ABS"
|
||||
echo "Container: $CONTAINER"
|
||||
echo ""
|
||||
|
||||
# Copy CSV file into container
|
||||
echo "Step 1: Copying CSV file into container..."
|
||||
if ! docker cp "$CSV_FILE_ABS" "${CONTAINER}:${TEMP_CSV_PATH}"; then
|
||||
error "Failed to copy CSV file into container"
|
||||
fi
|
||||
success "CSV file copied successfully"
|
||||
echo ""
|
||||
|
||||
# Execute import script inside container
|
||||
echo "Step 2: Running import script..."
|
||||
echo ""
|
||||
|
||||
if docker exec -it "$CONTAINER" node "$SCRIPT_PATH"; then
|
||||
success "Import completed successfully!"
|
||||
IMPORT_SUCCESS=true
|
||||
else
|
||||
error "Import failed. Check the logs above for details."
|
||||
IMPORT_SUCCESS=false
|
||||
fi
|
||||
|
||||
# Cleanup: Remove temp CSV file from container
|
||||
echo ""
|
||||
echo "Step 3: Cleaning up..."
|
||||
if docker exec "$CONTAINER" rm -f "$TEMP_CSV_PATH" 2>/dev/null; then
|
||||
success "Temporary files cleaned up"
|
||||
else
|
||||
warn "Warning: Failed to cleanup temp CSV file in container"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [ "$IMPORT_SUCCESS" = true ]; then
|
||||
echo "=========================================="
|
||||
success "Import process completed successfully!"
|
||||
echo "=========================================="
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,514 +0,0 @@
|
||||
# vPIC ETL Implementation Plan v2
|
||||
|
||||
## Overview
|
||||
|
||||
Extract vehicle dropdown data from NHTSA vPIC database for MY2022+ to supplement existing VehAPI data. This revised plan uses a make-specific extraction approach with proper VIN schema parsing.
|
||||
|
||||
## Key Changes from v1
|
||||
|
||||
1. **Limit to VehAPI makes only** - Only extract the 48 makes that exist in VehAPI data
|
||||
2. **VIN schema-based extraction** - Extract directly from VIN patterns, not defs_model
|
||||
3. **Proper field formatting** - Match VehAPI display string formats
|
||||
4. **Make-specific logic** - Handle different manufacturers' data patterns
|
||||
|
||||
## Critical Discovery: WMI Linkage
|
||||
|
||||
**Must use `wmi_make` junction table (many-to-many), NOT `wmi.makeid` (one-to-many):**
|
||||
```sql
|
||||
-- CORRECT: via wmi_make (finds all makes including Toyota, Hyundai, etc.)
|
||||
FROM vpic.make m
|
||||
JOIN vpic.wmi_make wm ON wm.makeid = m.id
|
||||
JOIN vpic.wmi w ON w.id = wm.wmiid
|
||||
|
||||
-- WRONG: via wmi.makeid (misses many major brands)
|
||||
FROM vpic.make m
|
||||
JOIN vpic.wmi w ON w.makeid = m.id
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Make Availability Summary
|
||||
|
||||
| Status | Count | Makes |
|
||||
|--------|-------|-------|
|
||||
| **Available (2022+ schemas)** | 46 | See table below |
|
||||
| **No 2022+ data** | 2 | Hummer (discontinued 2010), Scion (discontinued 2016) |
|
||||
|
||||
---
|
||||
|
||||
## Per-Make Analysis
|
||||
|
||||
### Group 1: Japanese Manufacturers (Honda/Acura, Toyota/Lexus, Nissan/Infiniti)
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Acura | Acura | Acura | 48 | Ready |
|
||||
| Honda | Honda | Honda | 238 | Ready |
|
||||
| Lexus | Lexus | Lexus | 90 | Ready |
|
||||
| Toyota | Toyota | Toyota | 152 | Ready |
|
||||
| Infiniti | INFINITI | Infiniti | 76 | Ready |
|
||||
| Nissan | Nissan | Nissan | 85 | Ready |
|
||||
| Mazda | Mazda | Mazda | 37 | Ready |
|
||||
| Mitsubishi | Mitsubishi | Mitsubishi | 11 | Ready |
|
||||
| Subaru | Subaru | Subaru | 75 | Ready |
|
||||
| Isuzu | Isuzu | Isuzu | 11 | Ready |
|
||||
|
||||
### Group 2: Korean Manufacturers (Hyundai/Kia/Genesis)
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Genesis | Genesis | Genesis | 74 | Ready |
|
||||
| Hyundai | Hyundai | Hyundai | 177 | Ready |
|
||||
| Kia | Kia | Kia | 72 | Ready |
|
||||
|
||||
### Group 3: American - GM (Chevrolet, GMC, Buick, Cadillac)
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Buick | Buick | Buick | 20 | Ready |
|
||||
| Cadillac | Cadillac | Cadillac | 50 | Ready |
|
||||
| Chevrolet | Chevrolet | Chevrolet | 185 | Ready |
|
||||
| GMC | GMC | GMC | 107 | Ready |
|
||||
| Oldsmobile | Oldsmobile | Oldsmobile | 1 | Limited |
|
||||
| Pontiac | Pontiac | Pontiac | 5 | Limited (2022-2024) |
|
||||
|
||||
### Group 4: American - Stellantis (Chrysler, Dodge, Jeep, Ram, Fiat)
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Chrysler | Chrysler | Chrysler | 81 | Ready |
|
||||
| Dodge | Dodge | Dodge | 86 | Ready |
|
||||
| FIAT | FIAT | Fiat | 91 | Ready (case diff) |
|
||||
| Jeep | Jeep | Jeep | 81 | Ready |
|
||||
| RAM | RAM | Ram | 81 | Ready (case diff) |
|
||||
| Plymouth | Plymouth | Plymouth | 4 | Limited |
|
||||
|
||||
### Group 5: American - Ford
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Ford | Ford | Ford | 108 | Ready |
|
||||
| Lincoln | Lincoln | Lincoln | 21 | Ready |
|
||||
| Mercury | Mercury | Mercury | 0 | No data (discontinued 2011) |
|
||||
|
||||
### Group 6: American - EV Startups
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Polestar | Polestar | Polestar | 12 | Ready |
|
||||
| Rivian | Rivian | RIVIAN | 10 | Ready (case diff) |
|
||||
| Tesla | Tesla | Tesla | 14 | Ready |
|
||||
|
||||
### Group 7: German Manufacturers
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Audi | Audi | Audi | 55 | Ready |
|
||||
| BMW | BMW | BMW | 61 | Ready |
|
||||
| Mercedes-Benz | Mercedes-Benz | Mercedes-Benz | 39 | Ready |
|
||||
| MINI | MINI | MINI | 10 | Ready |
|
||||
| Porsche | Porsche | Porsche | 23 | Ready |
|
||||
| smart | smart | smart | 5 | Ready |
|
||||
| Volkswagen | Volkswagen | Volkswagen | 134 | Ready |
|
||||
|
||||
### Group 8: European Luxury
|
||||
|
||||
| Make | VehAPI Name | vPIC Name | Schemas (2022+) | Status |
|
||||
|------|-------------|-----------|-----------------|--------|
|
||||
| Bentley | Bentley | Bentley | 48 | Ready |
|
||||
| Ferrari | Ferrari | Ferrari | 9 | Ready |
|
||||
| Jaguar | Jaguar | Jaguar | 17 | Ready |
|
||||
| Lamborghini | Lamborghini | Lamborghini | 10 | Ready |
|
||||
| Lotus | Lotus | Lotus | 5 | Ready |
|
||||
| Maserati | Maserati | Maserati | 19 | Ready |
|
||||
| McLaren | McLaren | McLaren | 4 | Ready |
|
||||
| Volvo | Volvo | Volvo | 80 | Ready |
|
||||
|
||||
### Group 9: Discontinued (No 2022+ Data)
|
||||
|
||||
| Make | VehAPI Name | Reason | Action |
|
||||
|------|-------------|--------|--------|
|
||||
| Hummer | Hummer | Discontinued 2010 (new EV under GMC) | Skip - use existing VehAPI |
|
||||
| Scion | Scion | Discontinued 2016 | Skip - use existing VehAPI |
|
||||
| Saab | Saab | Discontinued 2012 | Limited schemas (9) |
|
||||
| Mercury | Mercury | Discontinued 2011 | No schemas |
|
||||
|
||||
---
|
||||
|
||||
## Extraction Architecture
|
||||
|
||||
### Data Flow
|
||||
```
|
||||
vPIC VIN Schemas → Pattern Extraction → Format Transformation → SQLite Pairs
|
||||
↓
|
||||
Filter by:
|
||||
- 48 VehAPI makes only
|
||||
- Year >= 2022
|
||||
- Vehicle types (exclude motorcycles, trailers, buses)
|
||||
```
|
||||
|
||||
### Core Query Strategy
|
||||
|
||||
For each allowed make:
|
||||
1. Find WMIs linked to that make
|
||||
2. Get VIN schemas for years 2022+
|
||||
3. Extract from patterns:
|
||||
- Model (from schema name or pattern)
|
||||
- Trim (Element: Trim)
|
||||
- Displacement (Element: DisplacementL)
|
||||
- Horsepower (Element: EngineHP)
|
||||
- Cylinders (Element: EngineCylinders)
|
||||
- Engine Config (Element: EngineConfiguration)
|
||||
- Transmission Style (Element: TransmissionStyle)
|
||||
- Transmission Speeds (Element: TransmissionSpeeds)
|
||||
|
||||
---
|
||||
|
||||
## Acura Extraction Template
|
||||
|
||||
This pattern applies to Honda/Acura and similar well-structured manufacturers.
|
||||
|
||||
### Sample VIN Schema: Acura MDX 2025 (schema_id: 26929)
|
||||
|
||||
| Element | Code | Values |
|
||||
|---------|------|--------|
|
||||
| Trim | Trim | MDX, Technology, SH-AWD, SH-AWD Technology, SH-AWD A-Spec, SH-AWD Advance, SH-AWD A-Spec Advance, SH-AWD TYPE S ADVANCE |
|
||||
| Displacement | DisplacementL | 3.5, 3.0 |
|
||||
| Horsepower | EngineHP | 290, 355 |
|
||||
| Cylinders | EngineCylinders | 6 |
|
||||
| Engine Config | EngineConfiguration | V-Shaped |
|
||||
| Trans Style | TransmissionStyle | Automatic |
|
||||
| Trans Speeds | TransmissionSpeeds | 10 |
|
||||
|
||||
### Output Format
|
||||
|
||||
**Engine Display** (match VehAPI):
|
||||
```
|
||||
{DisplacementL}L {EngineHP} hp V{EngineCylinders}
|
||||
→ "3.5L 290 hp V6"
|
||||
```
|
||||
|
||||
**Transmission Display** (match VehAPI):
|
||||
```
|
||||
{TransmissionSpeeds}-Speed {TransmissionStyle}
|
||||
→ "10-Speed Automatic"
|
||||
```
|
||||
|
||||
### Extraction SQL Template
|
||||
|
||||
```sql
|
||||
WITH schema_data AS (
|
||||
SELECT DISTINCT
|
||||
vs.id AS schema_id,
|
||||
vs.name AS schema_name,
|
||||
wvs.yearfrom,
|
||||
COALESCE(wvs.yearto, 2027) AS yearto,
|
||||
m.name AS make_name
|
||||
FROM vpic.wmi w
|
||||
JOIN vpic.make m ON w.makeid = m.id
|
||||
JOIN vpic.wmi_vinschema wvs ON w.id = wvs.wmiid
|
||||
JOIN vpic.vinschema vs ON wvs.vinschemaid = vs.id
|
||||
WHERE LOWER(m.name) IN ('acura', 'honda', ...) -- VehAPI makes
|
||||
AND wvs.yearfrom >= 2022 OR (wvs.yearto >= 2022)
|
||||
),
|
||||
trim_data AS (
|
||||
SELECT DISTINCT sd.schema_id, p.attributeid AS trim
|
||||
FROM schema_data sd
|
||||
JOIN vpic.pattern p ON p.vinschemaid = sd.schema_id
|
||||
JOIN vpic.element e ON p.elementid = e.id
|
||||
WHERE e.code = 'Trim'
|
||||
),
|
||||
engine_data AS (
|
||||
SELECT DISTINCT
|
||||
sd.schema_id,
|
||||
MAX(CASE WHEN e.code = 'DisplacementL' THEN p.attributeid END) AS displacement,
|
||||
MAX(CASE WHEN e.code = 'EngineHP' THEN p.attributeid END) AS hp,
|
||||
MAX(CASE WHEN e.code = 'EngineCylinders' THEN p.attributeid END) AS cylinders,
|
||||
MAX(CASE WHEN e.code = 'EngineConfiguration' THEN ec.name END) AS config
|
||||
FROM schema_data sd
|
||||
JOIN vpic.pattern p ON p.vinschemaid = sd.schema_id
|
||||
JOIN vpic.element e ON p.elementid = e.id
|
||||
LEFT JOIN vpic.engineconfiguration ec ON e.code = 'EngineConfiguration'
|
||||
AND p.attributeid ~ '^[0-9]+$' AND ec.id = CAST(p.attributeid AS INT)
|
||||
WHERE e.code IN ('DisplacementL', 'EngineHP', 'EngineCylinders', 'EngineConfiguration')
|
||||
GROUP BY sd.schema_id, p.keys -- Group by VIN pattern position
|
||||
),
|
||||
trans_data AS (
|
||||
SELECT DISTINCT
|
||||
sd.schema_id,
|
||||
t.name AS style,
|
||||
MAX(CASE WHEN e.code = 'TransmissionSpeeds' THEN p.attributeid END) AS speeds
|
||||
FROM schema_data sd
|
||||
JOIN vpic.pattern p ON p.vinschemaid = sd.schema_id
|
||||
JOIN vpic.element e ON p.elementid = e.id
|
||||
LEFT JOIN vpic.transmission t ON e.code = 'TransmissionStyle'
|
||||
AND p.attributeid ~ '^[0-9]+$' AND t.id = CAST(p.attributeid AS INT)
|
||||
WHERE e.code IN ('TransmissionStyle', 'TransmissionSpeeds')
|
||||
GROUP BY sd.schema_id, t.name
|
||||
)
|
||||
SELECT ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Allowed Makes (48 from VehAPI)
|
||||
|
||||
```python
|
||||
ALLOWED_MAKES = [
|
||||
'Acura', 'Audi', 'Bentley', 'BMW', 'Buick', 'Cadillac', 'Chevrolet',
|
||||
'Chrysler', 'Dodge', 'Ferrari', 'FIAT', 'Ford', 'Genesis', 'GMC',
|
||||
'Honda', 'Hummer', 'Hyundai', 'INFINITI', 'Isuzu', 'Jaguar', 'Jeep',
|
||||
'Kia', 'Lamborghini', 'Lexus', 'Lincoln', 'Lotus', 'Maserati', 'Mazda',
|
||||
'McLaren', 'Mercedes-Benz', 'Mercury', 'MINI', 'Mitsubishi', 'Nissan',
|
||||
'Oldsmobile', 'Plymouth', 'Polestar', 'Pontiac', 'Porsche', 'RAM',
|
||||
'Rivian', 'Saab', 'Scion', 'smart', 'Subaru', 'Tesla', 'Toyota',
|
||||
'Volkswagen', 'Volvo'
|
||||
]
|
||||
```
|
||||
|
||||
Note: Some makes may have different names in vPIC (case variations, abbreviations).
|
||||
|
||||
---
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
### Phase 1: Rewrite vpic_extract.py
|
||||
|
||||
**File:** `vpic_extract.py`
|
||||
|
||||
Core extraction query (uses wmi_make junction table):
|
||||
```sql
|
||||
WITH base AS (
|
||||
SELECT DISTINCT
|
||||
m.name AS make_name,
|
||||
vs.id AS schema_id,
|
||||
vs.name AS schema_name,
|
||||
generate_series(
|
||||
GREATEST(wvs.yearfrom, 2022),
|
||||
COALESCE(wvs.yearto, EXTRACT(YEAR FROM NOW()) + 2)
|
||||
)::INT AS year
|
||||
FROM vpic.make m
|
||||
JOIN vpic.wmi_make wm ON wm.makeid = m.id
|
||||
JOIN vpic.wmi w ON w.id = wm.wmiid
|
||||
JOIN vpic.wmi_vinschema wvs ON w.id = wvs.wmiid
|
||||
JOIN vpic.vinschema vs ON wvs.vinschemaid = vs.id
|
||||
WHERE LOWER(m.name) IN ({allowed_makes})
|
||||
AND (wvs.yearfrom >= 2022 OR wvs.yearto >= 2022)
|
||||
)
|
||||
SELECT ...
|
||||
```
|
||||
|
||||
**Key functions to implement:**
|
||||
1. `extract_model_from_schema_name(schema_name)` - Parse "Acura MDX Schema..." → "MDX"
|
||||
2. `get_schema_patterns(schema_id)` - Get all pattern data for a schema
|
||||
3. `format_engine_display(disp, hp, cyl, config)` - Format as "3.5L 290 hp V6"
|
||||
4. `format_trans_display(style, speeds)` - Format as "10-Speed Automatic"
|
||||
5. `generate_trans_records(has_data, style, speeds)` - Return 1 or 2 records
|
||||
|
||||
**Make name normalization:**
|
||||
```python
|
||||
MAKE_MAPPING = {
|
||||
'INFINITI': 'INFINITI', # VehAPI uses all-caps
|
||||
'FIAT': 'FIAT',
|
||||
'RAM': 'RAM',
|
||||
'RIVIAN': 'Rivian', # vPIC uses all-caps, normalize
|
||||
# ... etc
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 2: Test Extraction
|
||||
|
||||
Test with validated VINs:
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
python3 vpic_extract.py --test-vin 5J8YE1H05SL018611 # Acura MDX
|
||||
python3 vpic_extract.py --test-vin 5TFJA5DB4SX327537 # Toyota Tundra
|
||||
python3 vpic_extract.py --test-vin 3GTUUFEL6PG140748 # GMC Sierra
|
||||
```
|
||||
|
||||
### Phase 3: Full Extraction
|
||||
|
||||
```bash
|
||||
python3 vpic_extract.py --min-year 2022 --output-dir snapshots/vpic-2025-12
|
||||
```
|
||||
|
||||
### Phase 4: Merge & Import
|
||||
|
||||
```bash
|
||||
# Merge vPIC with existing VehAPI data
|
||||
sqlite3 snapshots/merged/snapshot.sqlite "
|
||||
CREATE TABLE pairs(...);
|
||||
ATTACH 'snapshots/vehicle-drop-down.sqlite' AS db1;
|
||||
ATTACH 'snapshots/vpic-2025-12/snapshot.sqlite' AS db2;
|
||||
INSERT OR IGNORE INTO pairs SELECT * FROM db1.pairs WHERE year < 2022;
|
||||
INSERT OR IGNORE INTO pairs SELECT * FROM db2.pairs;
|
||||
"
|
||||
|
||||
# Generate SQL and import
|
||||
python3 etl_generate_sql.py --snapshot-path snapshots/merged/snapshot.sqlite
|
||||
./import_data.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify
|
||||
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `vpic_extract.py` | Complete rewrite: VIN schema extraction, dual-record trans logic |
|
||||
| `README.md` | Already updated with workflow |
|
||||
|
||||
---
|
||||
|
||||
## Success Criteria
|
||||
|
||||
1. Extract all 41 makes with 2022+ VIN schemas
|
||||
2. ~2,500-5,000 unique vehicle configurations (Year/Make/Model/Trim/Engine)
|
||||
3. Transmission: Use vPIC data where available (7 makes), dual-record elsewhere
|
||||
4. Output format matches VehAPI: "3.5L 290 hp V6" / "10-Speed Automatic"
|
||||
5. Merge preserves 2015-2021 VehAPI data
|
||||
6. QA validation passes after import
|
||||
|
||||
---
|
||||
|
||||
## Make Analysis Status (All Families Validated)
|
||||
|
||||
| Family | Makes | Status | Trans Data | Strategy |
|
||||
|--------|-------|--------|------------|----------|
|
||||
| Honda/Acura | Acura, Honda | VALIDATED | YES (93-97%) | Use vPIC trans data |
|
||||
| Toyota/Lexus | Toyota, Lexus | VALIDATED | PARTIAL (Toyota 23%, Lexus 0%) | Dual-record for Lexus |
|
||||
| Nissan/Infiniti | Nissan, Infiniti, Mitsubishi | VALIDATED | LOW (5%) | Dual-record |
|
||||
| GM | Chevrolet, GMC, Buick, Cadillac | VALIDATED | LOW (0-7%) | Dual-record |
|
||||
| Stellantis | Chrysler, Dodge, Jeep, Ram, Fiat | VALIDATED | NONE (0%) | Dual-record |
|
||||
| Ford | Ford, Lincoln | VALIDATED | NONE (0%) | Dual-record |
|
||||
| VW Group | Volkswagen, Audi, Porsche, Bentley, Lamborghini | VALIDATED | MIXED (0-84%) | VW/Audi use vPIC; others dual-record |
|
||||
| BMW | BMW, MINI | VALIDATED | NONE (0%) | Dual-record |
|
||||
| Mercedes | Mercedes-Benz, smart | VALIDATED | YES (52%) | Use vPIC trans data |
|
||||
| Hyundai/Kia/Genesis | Hyundai, Kia, Genesis | VALIDATED | NONE (0%) | Dual-record |
|
||||
| Subaru | Subaru | VALIDATED | YES (64%) | Use vPIC trans data |
|
||||
| Mazda | Mazda | VALIDATED | LOW (11%) | Dual-record |
|
||||
| Volvo | Volvo, Polestar | VALIDATED | LOW (3%/0%) | Dual-record |
|
||||
| Exotics | Ferrari, Maserati, Jaguar, Lotus, McLaren | VALIDATED | MIXED | Per-make handling |
|
||||
| EV | Tesla, Rivian | VALIDATED | NONE (0%) | Dual-record (though EVs don't have "manual") |
|
||||
|
||||
### Special Cases
|
||||
|
||||
1. **Electric Vehicles** (Tesla, Rivian, Polestar): Don't have manual transmissions
|
||||
- Still create dual-record for consistency with dropdown
|
||||
- User can select "Automatic" (single-speed EV)
|
||||
|
||||
2. **Luxury Exotics** (Ferrari, Lamborghini, etc.): Mix of automated manual/DCT
|
||||
- Dual-record covers all options
|
||||
|
||||
---
|
||||
|
||||
## CRITICAL FINDING: Transmission Data Availability
|
||||
|
||||
**Most manufacturers do NOT encode transmission info in VINs.**
|
||||
|
||||
### VIN Decode Validation Results (12 Families)
|
||||
|
||||
| Family | VIN | Make | Model | Year | Trim | Engine | Trans |
|
||||
|--------|-----|------|-------|------|------|--------|-------|
|
||||
| Honda/Acura | 5J8YE1H05SL018611 | ACURA | MDX | 2025 | SH-AWD A-Spec | 3.5L V6 290hp | 10-Spd Auto |
|
||||
| Honda/Acura | 2HGFE4F88SH315466 | HONDA | Civic | 2025 | Sport Hybrid | 2.0L I4 141hp | e-CVT |
|
||||
| Toyota/Lexus | 5TFJA5DB4SX327537 | TOYOTA | Tundra | 2025 | Limited | 3.4L V6 389hp | 10-Spd Auto |
|
||||
| Nissan/Infiniti | 5N1AL1FW9TC332353 | INFINITI | QX60 | 2026 | Luxe | 2.0L (no cyl/hp) | **MISSING** |
|
||||
| GM | 3GTUUFEL6PG140748 | GMC | Sierra | 2023 | AT4X | 6.2L V8 (no hp) | **MISSING** |
|
||||
| Stellantis | 1C4HJXEG7PW506480 | JEEP | Wrangler | 2023 | Sahara | 3.6L V6 285hp | **MISSING** |
|
||||
| Ford | 1FTFW4L59SFC03038 | FORD | F-150 | 2025 | Tremor | 5.0L V8 (no hp) | **MISSING** |
|
||||
| VW Group | WVWEB7CD9RW229116 | VOLKSWAGEN | Golf R | 2024 | **MISSING** | 2.0L 4cyl 315hp | Auto (no spd) |
|
||||
| BMW | 5YM13ET06R9S31554 | BMW | X5 | 2024 | X5 M Competition | 4.4L 8cyl 617hp | **MISSING** |
|
||||
| Mercedes | W1KAF4HB1SR287126 | MERCEDES-BENZ | C-Class | 2025 | C300 4MATIC | 2.0L I4 255hp | 9-Spd Auto |
|
||||
| Hyundai/Kia | 5XYRLDJC0SG336002 | KIA | Sorento | 2025 | S | 2.5L 4cyl 191hp | **MISSING** |
|
||||
| Subaru | JF1VBAF67P9806852 | SUBARU | WRX | 2023 | Premium | 2.4L 4cyl 271hp | 6-Spd Manual |
|
||||
| Mazda | JM3KFBCL3R0522361 | MAZDA | CX-5 | 2024 | Preferred Pkg | 2.5L I4 187hp | 6-Spd Auto |
|
||||
| Volvo | YV4M12RJ9S1094167 | VOLVO | XC60 | 2025 | Core | 2.0L 4cyl 247hp | 8-Spd Auto |
|
||||
|
||||
### Transmission Data Coverage in vPIC Schemas
|
||||
|
||||
| Coverage | Makes | Trans Schemas / Total |
|
||||
|----------|-------|----------------------|
|
||||
| **HIGH (>40%)** | Honda, Acura, Subaru, Audi, VW, Mercedes, Jaguar | 225/233, 42/45, 47/74, 46/55, 47/132, 13/25, 17/17 |
|
||||
| **LOW (<10%)** | Chevrolet, Cadillac, Nissan, Infiniti, Mazda, Volvo | 4/164, 7/43, 4/82, 4/74, 4/36, 2/72 |
|
||||
| **NONE (0%)** | GMC, Buick, Ford, Lincoln, Jeep, Dodge, Chrysler, Ram, Fiat, BMW, MINI, Porsche, Hyundai, Kia, Genesis, Lexus, Tesla, Rivian, Polestar | 0% |
|
||||
|
||||
### Makes WITHOUT Transmission Data (22 of 41 makes = 54%)
|
||||
- **ALL Stellantis**: Chrysler, Dodge, Jeep, Ram, Fiat
|
||||
- **ALL Ford**: Ford, Lincoln
|
||||
- **ALL Korean**: Hyundai, Kia, Genesis
|
||||
- **ALL BMW Group**: BMW, MINI
|
||||
- **GM (partial)**: GMC, Buick (Chevy/Cadillac have minimal)
|
||||
- **Others**: Lexus, Porsche, Bentley, Lamborghini, Tesla, Rivian, Polestar
|
||||
|
||||
---
|
||||
|
||||
## Extraction Strategy (SELECTED)
|
||||
|
||||
### Dual-Record Strategy for Missing Transmission Data
|
||||
|
||||
When transmission data is NOT available from vPIC:
|
||||
- **Create TWO records** for each vehicle configuration
|
||||
- One with `trans_display = "Automatic"`, `trans_canon = "automatic"`
|
||||
- One with `trans_display = "Manual"`, `trans_canon = "manual"`
|
||||
|
||||
This ensures:
|
||||
- All transmission options available in dropdown for user selection
|
||||
- User can select the correct transmission type
|
||||
- No false "Unknown" values that break filtering
|
||||
|
||||
### Implementation Logic
|
||||
|
||||
```python
|
||||
def generate_trans_records(has_trans_data: bool, trans_style: str, trans_speeds: str):
|
||||
if has_trans_data:
|
||||
# Use actual vPIC data
|
||||
return [(format_trans_display(trans_style, trans_speeds),
|
||||
canonicalize_trans(trans_style))]
|
||||
else:
|
||||
# Generate both options
|
||||
return [
|
||||
("Automatic", "automatic"),
|
||||
("Manual", "manual")
|
||||
]
|
||||
```
|
||||
|
||||
### Expected Output Growth
|
||||
|
||||
For makes without trans data, record count approximately doubles:
|
||||
- GMC Sierra AT4X + 6.2L V8 → 2 records (Auto + Manual)
|
||||
- Ford F-150 Tremor + 5.0L V8 → 2 records (Auto + Manual)
|
||||
|
||||
This is acceptable as it provides complete dropdown coverage.
|
||||
|
||||
---
|
||||
|
||||
## Validated Extraction Examples
|
||||
|
||||
### Acura MDX 2025 (VIN: 5J8YE1H05SL018611)
|
||||
- **vPIC**: Make=ACURA, Model=MDX, Trim=SH-AWD A-Spec, Engine=3.5L V6 290hp, Trans=10-Speed Automatic
|
||||
- **Output**: `3.5L 290 hp V6` | `10-Speed Automatic`
|
||||
|
||||
### Honda Civic 2025 (VIN: 2HGFE4F88SH315466)
|
||||
- **vPIC**: Make=HONDA, Model=Civic, Trim=Sport Hybrid / Sport Touring Hybrid, Engine=2L I4 141hp, Trans=e-CVT
|
||||
- **Output**: `2.0L 141 hp I4` | `Electronic Continuously Variable (e-CVT)`
|
||||
|
||||
### Toyota Tundra 2025 (VIN: 5TFJA5DB4SX327537)
|
||||
- **vPIC**: Make=TOYOTA, Model=Tundra, Trim=Limited, Engine=3.4L V6 389hp, Trans=10-Speed Automatic
|
||||
- **Output**: `3.4L 389 hp V6` | `10-Speed Automatic`
|
||||
|
||||
### Mercedes C-Class 2025 (VIN: W1KAF4HB1SR287126)
|
||||
- **vPIC**: Make=MERCEDES-BENZ, Model=C-Class, Trim=C300 4MATIC, Engine=2.0L I4 255hp, Trans=9-Speed Automatic
|
||||
- **Output**: `2.0L 255 hp I4` | `9-Speed Automatic`
|
||||
|
||||
### Subaru WRX 2023 (VIN: JF1VBAF67P9806852)
|
||||
- **vPIC**: Make=SUBARU, Model=WRX, Trim=Premium, Engine=2.4L 4cyl 271hp, Trans=6-Speed Manual
|
||||
- **Output**: `2.4L 271 hp 4cyl` | `6-Speed Manual`
|
||||
|
||||
### Mazda CX-5 2024 (VIN: JM3KFBCL3R0522361)
|
||||
- **vPIC**: Make=MAZDA, Model=CX-5, Trim=Preferred Package, Engine=2.5L I4 187hp, Trans=6-Speed Automatic
|
||||
- **Output**: `2.5L 187 hp I4` | `6-Speed Automatic`
|
||||
|
||||
### Volvo XC60 2025 (VIN: YV4M12RJ9S1094167)
|
||||
- **vPIC**: Make=VOLVO, Model=XC60, Trim=Core, Engine=2.0L 4cyl 247hp, Trans=8-Speed Automatic
|
||||
- **Output**: `2.0L 247 hp 4cyl` | `8-Speed Automatic`
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Merges two VehAPI snapshot databases into a single consolidated database.
|
||||
Handles deduplication via PRIMARY KEY constraint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def merge_databases(db1_path: Path, db2_path: Path, output_path: Path) -> dict:
|
||||
"""Merge two snapshot databases into one, deduplicating by PRIMARY KEY."""
|
||||
|
||||
if output_path.exists():
|
||||
output_path.unlink()
|
||||
print(f"[info] Removed existing output file: {output_path}")
|
||||
|
||||
conn = sqlite3.connect(output_path)
|
||||
conn.execute("PRAGMA journal_mode=WAL;")
|
||||
conn.execute("PRAGMA synchronous=NORMAL;")
|
||||
|
||||
# Create target schema (pairs table only)
|
||||
conn.execute("""
|
||||
CREATE TABLE pairs(
|
||||
year INT,
|
||||
make TEXT,
|
||||
model TEXT,
|
||||
trim TEXT,
|
||||
engine_display TEXT,
|
||||
engine_canon TEXT,
|
||||
engine_bucket TEXT,
|
||||
trans_display TEXT,
|
||||
trans_canon TEXT,
|
||||
trans_bucket TEXT,
|
||||
PRIMARY KEY(year, make, model, trim, engine_canon, trans_canon)
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
# Attach source databases
|
||||
conn.execute(f"ATTACH DATABASE '{db1_path}' AS db1")
|
||||
conn.execute(f"ATTACH DATABASE '{db2_path}' AS db2")
|
||||
|
||||
# Insert from first database
|
||||
print(f"[info] Inserting records from {db1_path.name}...")
|
||||
cursor = conn.execute("""
|
||||
INSERT OR IGNORE INTO pairs
|
||||
SELECT year, make, model, trim, engine_display, engine_canon, engine_bucket,
|
||||
trans_display, trans_canon, trans_bucket
|
||||
FROM db1.pairs
|
||||
""")
|
||||
db1_inserted = cursor.rowcount
|
||||
conn.commit()
|
||||
print(f"[info] Inserted {db1_inserted:,} records from {db1_path.name}")
|
||||
|
||||
# Insert from second database (duplicates ignored)
|
||||
print(f"[info] Inserting records from {db2_path.name}...")
|
||||
cursor = conn.execute("""
|
||||
INSERT OR IGNORE INTO pairs
|
||||
SELECT year, make, model, trim, engine_display, engine_canon, engine_bucket,
|
||||
trans_display, trans_canon, trans_bucket
|
||||
FROM db2.pairs
|
||||
""")
|
||||
db2_inserted = cursor.rowcount
|
||||
conn.commit()
|
||||
print(f"[info] Inserted {db2_inserted:,} new records from {db2_path.name}")
|
||||
|
||||
# Detach source databases
|
||||
conn.execute("DETACH DATABASE db1")
|
||||
conn.execute("DETACH DATABASE db2")
|
||||
|
||||
# Get final stats
|
||||
total_count = conn.execute("SELECT COUNT(*) FROM pairs").fetchone()[0]
|
||||
min_year = conn.execute("SELECT MIN(year) FROM pairs").fetchone()[0]
|
||||
max_year = conn.execute("SELECT MAX(year) FROM pairs").fetchone()[0]
|
||||
|
||||
# Optimize the database
|
||||
print("[info] Running VACUUM to optimize database...")
|
||||
conn.execute("VACUUM")
|
||||
conn.close()
|
||||
|
||||
stats = {
|
||||
"db1_inserted": db1_inserted,
|
||||
"db2_inserted": db2_inserted,
|
||||
"total_records": total_count,
|
||||
"min_year": min_year,
|
||||
"max_year": max_year,
|
||||
"output_path": str(output_path),
|
||||
}
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
base_dir = Path(__file__).resolve().parent
|
||||
snapshots_dir = base_dir / "snapshots"
|
||||
|
||||
db1_path = snapshots_dir / "1980-2007.sqlite"
|
||||
db2_path = snapshots_dir / "2007-2022.sqlite"
|
||||
output_path = snapshots_dir / "1980-2022-vehicles.sqlite"
|
||||
|
||||
# Validate source files exist
|
||||
if not db1_path.exists():
|
||||
print(f"[error] Source database not found: {db1_path}", file=sys.stderr)
|
||||
return 1
|
||||
if not db2_path.exists():
|
||||
print(f"[error] Source database not found: {db2_path}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"[info] Merging databases...")
|
||||
print(f" Source 1: {db1_path}")
|
||||
print(f" Source 2: {db2_path}")
|
||||
print(f" Output: {output_path}")
|
||||
print()
|
||||
|
||||
stats = merge_databases(db1_path, db2_path, output_path)
|
||||
|
||||
print()
|
||||
print(f"[done] Merge complete!")
|
||||
print(f" Total records: {stats['total_records']:,}")
|
||||
print(f" Year range: {stats['min_year']} - {stats['max_year']}")
|
||||
print(f" Output: {stats['output_path']}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,513 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetches VehAPI data into an offline snapshot (SQLite + meta.json).
|
||||
|
||||
Workflow:
|
||||
1. Walks Year -> Make -> Model -> Trim -> Transmission -> Engine using VehAPI.
|
||||
2. Persists observed compatibility pairs to snapshot.sqlite (no Cartesian products).
|
||||
3. Stores request/response cache for resume; obeys rate limits and 429 retry-after.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError: # pragma: no cover - env guard
|
||||
print("[error] Missing dependency 'requests'. Install with `pip install requests`.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
SCRIPT_VERSION = "vehapi_fetch_snapshot.py@1.1.0"
|
||||
DEFAULT_MIN_YEAR = 2015
|
||||
DEFAULT_MAX_YEAR = 2022
|
||||
DEFAULT_RATE_PER_MIN = 55 # stays under the 60 req/min ceiling
|
||||
MAX_ATTEMPTS = 5
|
||||
FALLBACK_TRIMS = ["Base"]
|
||||
FALLBACK_TRANSMISSIONS = ["Manual", "Automatic"]
|
||||
DEFAULT_BASE_URL = "https://vehapi.com/api/v1/car-lists/get/car"
|
||||
|
||||
|
||||
def canonicalize(value: str) -> str:
|
||||
"""Lowercase, trim, collapse spaces, and normalize hyphens for dedupe keys."""
|
||||
import re
|
||||
|
||||
cleaned = (value or "").strip()
|
||||
cleaned = re.sub(r"[\s\u00A0]+", " ", cleaned)
|
||||
cleaned = re.sub(r"[-\u2010-\u2015]+", "-", cleaned)
|
||||
return cleaned.lower()
|
||||
|
||||
|
||||
def infer_trans_bucket(trans_str: str) -> str:
|
||||
lowered = (trans_str or "").lower()
|
||||
if "manual" in lowered or "mt" in lowered or "m/t" in lowered:
|
||||
return "Manual"
|
||||
return "Automatic"
|
||||
|
||||
|
||||
def infer_fuel_bucket(engine_str: str, trans_str: str, trim_str: str) -> str:
|
||||
target = " ".join([engine_str or "", trans_str or "", trim_str or ""]).lower()
|
||||
if any(token in target for token in ["electric", "ev", "battery", "motor", "kwh"]):
|
||||
return "Electric"
|
||||
if any(token in target for token in ["hybrid", "phev", "plug-in", "hev", "e-hybrid"]):
|
||||
return "Hybrid"
|
||||
if any(token in target for token in ["diesel", "tdi", "dci", "duramax", "power stroke", "cummins"]):
|
||||
return "Diesel"
|
||||
return "Gas"
|
||||
|
||||
|
||||
def read_text_file(path: Path) -> str:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def read_lines(path: Path) -> List[str]:
|
||||
return [line.strip() for line in read_text_file(path).splitlines() if line.strip()]
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(8192), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def ensure_snapshot_dir(root: Path, custom_dir: Optional[str]) -> Path:
|
||||
if custom_dir:
|
||||
snapshot_dir = Path(custom_dir)
|
||||
else:
|
||||
today = datetime.now(timezone.utc).date().isoformat()
|
||||
snapshot_dir = root / today
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
return snapshot_dir
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Fixed delay limiter to stay below the VehAPI threshold (60 req/min)."""
|
||||
|
||||
def __init__(self, max_per_min: int) -> None:
|
||||
self.delay = 60.0 / max_per_min # ~1.09 sec for 55 rpm
|
||||
self._last_request = 0.0
|
||||
|
||||
def acquire(self) -> None:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_request
|
||||
if elapsed < self.delay:
|
||||
time.sleep(self.delay - elapsed)
|
||||
self._last_request = time.monotonic()
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchCounts:
|
||||
pairs_inserted: int = 0
|
||||
cache_hits: int = 0
|
||||
fallback_transmissions: int = 0
|
||||
fallback_engines: int = 0
|
||||
|
||||
|
||||
class VehapiFetcher:
|
||||
def __init__(
|
||||
self,
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
token: str,
|
||||
min_year: int,
|
||||
max_year: int,
|
||||
allowed_makes: Sequence[str],
|
||||
snapshot_path: Path,
|
||||
responses_cache: bool = True,
|
||||
rate_per_min: int = DEFAULT_RATE_PER_MIN,
|
||||
) -> None:
|
||||
self.session = session
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.token = token
|
||||
self.min_year = min_year
|
||||
self.max_year = max_year
|
||||
self.allowed_makes = {canonicalize(m): m for m in allowed_makes}
|
||||
self.snapshot_path = snapshot_path
|
||||
self.conn = sqlite3.connect(self.snapshot_path)
|
||||
self.conn.execute("PRAGMA journal_mode=WAL;")
|
||||
self.conn.execute("PRAGMA synchronous=NORMAL;")
|
||||
self._init_schema()
|
||||
self.responses_cache = responses_cache
|
||||
self.rate_limiter = RateLimiter(rate_per_min)
|
||||
self.counts = FetchCounts()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
self.conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS pairs(
|
||||
year INT,
|
||||
make TEXT,
|
||||
model TEXT,
|
||||
trim TEXT,
|
||||
engine_display TEXT,
|
||||
engine_canon TEXT,
|
||||
engine_bucket TEXT,
|
||||
trans_display TEXT,
|
||||
trans_canon TEXT,
|
||||
trans_bucket TEXT,
|
||||
PRIMARY KEY(year, make, model, trim, engine_canon, trans_canon)
|
||||
)
|
||||
"""
|
||||
)
|
||||
self.conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS meta(
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
self.conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS responses(
|
||||
request_key TEXT PRIMARY KEY,
|
||||
url TEXT,
|
||||
status INT,
|
||||
headers_json TEXT,
|
||||
body_json TEXT,
|
||||
fetched_at TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def _store_meta(self, meta: Dict[str, Any]) -> None:
|
||||
rows = [(k, str(v)) for k, v in meta.items()]
|
||||
self.conn.executemany("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)", rows)
|
||||
self.conn.commit()
|
||||
|
||||
def _load_cached_response(self, request_key: str) -> Optional[Any]:
|
||||
if not self.responses_cache:
|
||||
return None
|
||||
cur = self.conn.execute("SELECT body_json FROM responses WHERE request_key = ?", (request_key,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
self.counts.cache_hits += 1
|
||||
try:
|
||||
return json.loads(row[0])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _save_response(self, request_key: str, url: str, status: int, headers: Dict[str, Any], body: Any) -> None:
|
||||
self.conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO responses(request_key, url, status, headers_json, body_json, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
request_key,
|
||||
url,
|
||||
status,
|
||||
json.dumps(dict(headers), default=str),
|
||||
json.dumps(body, default=str),
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def _request_json(self, path_parts: Sequence[str], label: str) -> Any:
|
||||
path_parts = [str(p) for p in path_parts]
|
||||
request_key = "/".join(path_parts)
|
||||
cached = self._load_cached_response(request_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
url = f"{self.base_url}/" + "/".join(quote(p, safe="") for p in path_parts)
|
||||
attempts = 0
|
||||
backoff = 1.0
|
||||
while attempts < MAX_ATTEMPTS:
|
||||
attempts += 1
|
||||
self.rate_limiter.acquire()
|
||||
try:
|
||||
resp = self.session.get(url, headers={"Authorization": f"Bearer {self.token}"}, timeout=30)
|
||||
except requests.RequestException as exc:
|
||||
print(f"[warn] {label}: request error {exc}; retrying...", file=sys.stderr)
|
||||
time.sleep(backoff + random.uniform(0, 0.5))
|
||||
backoff = min(backoff * 2, 30)
|
||||
continue
|
||||
|
||||
if resp.status_code == 429:
|
||||
retry_after = resp.headers.get("retry-after") or resp.headers.get("Retry-After")
|
||||
try:
|
||||
retry_seconds = float(retry_after)
|
||||
except (TypeError, ValueError):
|
||||
retry_seconds = 30.0
|
||||
sleep_for = retry_seconds + random.uniform(0, 0.5)
|
||||
print(f"[info] {label}: hit 429, sleeping {sleep_for:.1f}s before retry", file=sys.stderr)
|
||||
time.sleep(sleep_for)
|
||||
backoff = min(backoff * 2, 30)
|
||||
continue
|
||||
|
||||
if resp.status_code >= 500:
|
||||
print(f"[warn] {label}: server {resp.status_code}, retrying...", file=sys.stderr)
|
||||
time.sleep(backoff + random.uniform(0, 0.5))
|
||||
backoff = min(backoff * 2, 30)
|
||||
continue
|
||||
|
||||
if not resp.ok:
|
||||
print(f"[warn] {label}: HTTP {resp.status_code}, skipping", file=sys.stderr)
|
||||
return []
|
||||
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError:
|
||||
print(f"[warn] {label}: non-JSON response, skipping", file=sys.stderr)
|
||||
return []
|
||||
|
||||
self._save_response(request_key, url, resp.status_code, resp.headers, body)
|
||||
return body
|
||||
|
||||
print(f"[error] {label}: exhausted retries", file=sys.stderr)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _extract_values(payload: Any, keys: Sequence[str]) -> List[str]:
|
||||
values: List[str] = []
|
||||
if isinstance(payload, dict):
|
||||
payload = payload.get("data") or payload.get("results") or payload.get("items") or payload
|
||||
if not payload:
|
||||
return values
|
||||
if isinstance(payload, list):
|
||||
for item in payload:
|
||||
if isinstance(item, str):
|
||||
if item.strip():
|
||||
values.append(item.strip())
|
||||
continue
|
||||
if isinstance(item, dict):
|
||||
for key in keys:
|
||||
val = item.get(key)
|
||||
if val:
|
||||
values.append(str(val).strip())
|
||||
break
|
||||
return values
|
||||
|
||||
def _record_pair(
|
||||
self,
|
||||
year: int,
|
||||
make: str,
|
||||
model: str,
|
||||
trim: str,
|
||||
engine_display: str,
|
||||
engine_bucket: str,
|
||||
trans_display: str,
|
||||
trans_bucket: str,
|
||||
) -> None:
|
||||
engine_canon = canonicalize(engine_display)
|
||||
trans_canon = canonicalize(trans_display)
|
||||
cur = self.conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO pairs(
|
||||
year, make, model, trim,
|
||||
engine_display, engine_canon, engine_bucket,
|
||||
trans_display, trans_canon, trans_bucket
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
year,
|
||||
make,
|
||||
model,
|
||||
trim,
|
||||
engine_display.strip(),
|
||||
engine_canon,
|
||||
engine_bucket,
|
||||
trans_display.strip(),
|
||||
trans_canon,
|
||||
trans_bucket,
|
||||
),
|
||||
)
|
||||
if cur.rowcount:
|
||||
self.counts.pairs_inserted += 1
|
||||
|
||||
def _fetch_engines_for_transmission(
|
||||
self, year: int, make: str, model: str, trim: str, transmission: str, trans_bucket: str
|
||||
) -> None:
|
||||
path = ["engines", str(year), make, model, trim, transmission]
|
||||
label = f"engines:{year}/{make}/{model}/{trim}/{transmission}"
|
||||
engines_payload = self._request_json(path, label)
|
||||
engines = self._extract_values(engines_payload, ["engine"])
|
||||
if not engines:
|
||||
engine_bucket = infer_fuel_bucket("", transmission, trim)
|
||||
fallback_engine = engine_bucket
|
||||
self._record_pair(year, make, model, trim, fallback_engine, engine_bucket, transmission, trans_bucket)
|
||||
self.counts.fallback_engines += 1
|
||||
return
|
||||
|
||||
for engine in engines:
|
||||
engine_bucket = infer_fuel_bucket(engine, transmission, trim)
|
||||
self._record_pair(year, make, model, trim, engine, engine_bucket, transmission, trans_bucket)
|
||||
|
||||
def _fetch_transmissions_for_trim(self, year: int, make: str, model: str, trim: str) -> None:
|
||||
path = ["transmissions", str(year), make, model, trim]
|
||||
label = f"transmissions:{year}/{make}/{model}/{trim}"
|
||||
transmissions_payload = self._request_json(path, label)
|
||||
transmissions = self._extract_values(transmissions_payload, ["transmission"])
|
||||
if not transmissions:
|
||||
for fallback in FALLBACK_TRANSMISSIONS:
|
||||
trans_bucket = infer_trans_bucket(fallback)
|
||||
engine_bucket = infer_fuel_bucket("", fallback, trim)
|
||||
self._record_pair(year, make, model, trim, engine_bucket, engine_bucket, fallback, trans_bucket)
|
||||
self.counts.fallback_transmissions += 1
|
||||
self.counts.fallback_engines += 1
|
||||
return
|
||||
|
||||
for trans in transmissions:
|
||||
trans_bucket = infer_trans_bucket(trans)
|
||||
self._fetch_engines_for_transmission(year, make, model, trim, trans, trans_bucket)
|
||||
|
||||
def _fetch_trims_for_model(self, year: int, make: str, model: str) -> None:
|
||||
print(f" -> {year} {make} {model}", file=sys.stderr)
|
||||
path = ["trims", str(year), make, model]
|
||||
label = f"trims:{year}/{make}/{model}"
|
||||
trims_payload = self._request_json(path, label)
|
||||
trims = self._extract_values(trims_payload, ["trim"])
|
||||
|
||||
if not trims:
|
||||
trims = FALLBACK_TRIMS
|
||||
|
||||
for trim in trims:
|
||||
self._fetch_transmissions_for_trim(year, make, model, trim)
|
||||
self.conn.commit()
|
||||
|
||||
def _fetch_models_for_make(self, year: int, make: str) -> None:
|
||||
path = ["models", str(year), make]
|
||||
label = f"models:{year}/{make}"
|
||||
models_payload = self._request_json(path, label)
|
||||
models = self._extract_values(models_payload, ["model"])
|
||||
if not models:
|
||||
print(f"[warn] {label}: no models returned", file=sys.stderr)
|
||||
return
|
||||
for model in models:
|
||||
self._fetch_trims_for_model(year, make, model)
|
||||
|
||||
def _fetch_makes_for_year(self, year: int) -> List[str]:
|
||||
path = ["makes", str(year)]
|
||||
label = f"makes:{year}"
|
||||
makes_payload = self._request_json(path, label)
|
||||
makes = self._extract_values(makes_payload, ["make"])
|
||||
filtered = []
|
||||
for make in makes:
|
||||
canon = canonicalize(make)
|
||||
if canon in self.allowed_makes:
|
||||
filtered.append(make)
|
||||
return filtered
|
||||
|
||||
def run(self) -> FetchCounts:
|
||||
for year in range(self.min_year, self.max_year + 1):
|
||||
makes = self._fetch_makes_for_year(year)
|
||||
if not makes:
|
||||
print(f"[info] {year}: no allowed makes found, skipping", file=sys.stderr)
|
||||
continue
|
||||
print(f"[info] {year}: {len(makes)} makes", file=sys.stderr)
|
||||
for idx, make in enumerate(makes, 1):
|
||||
print(f"[{year}] ({idx}/{len(makes)}) {make}", file=sys.stderr)
|
||||
self._fetch_models_for_make(year, make)
|
||||
print(f" [{self.counts.pairs_inserted} pairs so far]", file=sys.stderr)
|
||||
self.conn.commit()
|
||||
return self.counts
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Fetch VehAPI snapshot into SQLite.")
|
||||
parser.add_argument("--min-year", type=int, default=int(read_env("MIN_YEAR", DEFAULT_MIN_YEAR)), help="Inclusive min year (default env MIN_YEAR or 2017)")
|
||||
parser.add_argument("--max-year", type=int, default=int(read_env("MAX_YEAR", DEFAULT_MAX_YEAR)), help="Inclusive max year (default env MAX_YEAR or 2026)")
|
||||
parser.add_argument("--snapshot-dir", type=str, help="Target snapshot directory (default snapshots/<today>)")
|
||||
parser.add_argument("--base-url", type=str, default=read_env("VEHAPI_BASE_URL", DEFAULT_BASE_URL), help="VehAPI base URL (e.g. https://vehapi.com/api/v1/car-lists/get/car)")
|
||||
parser.add_argument("--rate-per-min", type=int, default=int(read_env("VEHAPI_MAX_RPM", DEFAULT_RATE_PER_MIN)), help="Max requests per minute (<=60)")
|
||||
parser.add_argument("--makes-file", type=str, default="source-makes.txt", help="Path to source-makes.txt")
|
||||
parser.add_argument("--api-key-file", type=str, default="vehapi.key", help="Path to VehAPI bearer token file")
|
||||
parser.add_argument("--no-response-cache", action="store_true", help="Disable request cache stored in snapshot.sqlite")
|
||||
return parser
|
||||
|
||||
|
||||
def read_env(key: str, default: Any) -> Any:
|
||||
import os
|
||||
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
base_dir = Path(__file__).resolve().parent
|
||||
snapshot_root = base_dir / "snapshots"
|
||||
snapshot_dir = ensure_snapshot_dir(snapshot_root, args.snapshot_dir)
|
||||
snapshot_path = snapshot_dir / "snapshot.sqlite"
|
||||
meta_path = snapshot_dir / "meta.json"
|
||||
|
||||
makes_file = (base_dir / args.makes_file).resolve()
|
||||
api_key_file = (base_dir / args.api_key_file).resolve()
|
||||
if not makes_file.exists():
|
||||
print(f"[error] makes file not found: {makes_file}", file=sys.stderr)
|
||||
return 1
|
||||
if not api_key_file.exists():
|
||||
print(f"[error] api key file not found: {api_key_file}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
allowed_makes = read_lines(makes_file)
|
||||
token = read_text_file(api_key_file).strip()
|
||||
if not token:
|
||||
print("[error] vehapi.key is empty", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
session = requests.Session()
|
||||
fetcher = VehapiFetcher(
|
||||
session=session,
|
||||
base_url=args.base_url,
|
||||
token=token,
|
||||
min_year=args.min_year,
|
||||
max_year=args.max_year,
|
||||
allowed_makes=allowed_makes,
|
||||
snapshot_path=snapshot_path,
|
||||
responses_cache=not args.no_response_cache,
|
||||
rate_per_min=args.rate_per_min,
|
||||
)
|
||||
|
||||
started_at = datetime.now(timezone.utc)
|
||||
counts = fetcher.run()
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
|
||||
meta = {
|
||||
"generated_at": finished_at.isoformat(),
|
||||
"started_at": started_at.isoformat(),
|
||||
"min_year": args.min_year,
|
||||
"max_year": args.max_year,
|
||||
"script_version": SCRIPT_VERSION,
|
||||
"makes_file": str(makes_file),
|
||||
"makes_hash": sha256_file(makes_file),
|
||||
"api_base_url": args.base_url,
|
||||
"snapshot_path": str(snapshot_path),
|
||||
"pairs_inserted": counts.pairs_inserted,
|
||||
"fallback_transmissions": counts.fallback_transmissions,
|
||||
"fallback_engines": counts.fallback_engines,
|
||||
"response_cache_hits": counts.cache_hits,
|
||||
}
|
||||
fetcher._store_meta(meta)
|
||||
with meta_path.open("w", encoding="utf-8") as fh:
|
||||
json.dump(meta, fh, indent=2)
|
||||
|
||||
print(
|
||||
f"[done] wrote snapshot to {snapshot_path} with {counts.pairs_inserted} pairs "
|
||||
f"(fallback trans={counts.fallback_transmissions}, fallback engines={counts.fallback_engines}, cache hits={counts.cache_hits})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user