Initial Commit
This commit is contained in:
Eric Gullickson
Binary file not shown.
@@ -0,0 +1,381 @@
|
||||
"""
|
||||
Unit Tests for EngineSpecParser
|
||||
|
||||
Tests the engine specification parsing functionality including:
|
||||
- Standard engine format parsing (displacement, configuration, cylinders)
|
||||
- CRITICAL: L→I normalization (L3 becomes I3)
|
||||
- Hybrid and electric vehicle detection
|
||||
- Fuel type and aspiration parsing
|
||||
- Electric motor creation for empty engines arrays
|
||||
- Error handling for unparseable engines
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
# Import the class we're testing
|
||||
from ..utils.engine_spec_parser import EngineSpecParser, EngineSpec
|
||||
|
||||
|
||||
class TestEngineSpecParser(unittest.TestCase):
|
||||
"""Test cases for EngineSpecParser utility"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test environment before each test"""
|
||||
self.parser = EngineSpecParser()
|
||||
|
||||
def test_parse_standard_engines(self):
|
||||
"""Test parsing of standard engine formats"""
|
||||
test_cases = [
|
||||
# Format: (input, expected_displacement, expected_config, expected_cylinders)
|
||||
("2.0L I4", 2.0, "I", 4),
|
||||
("3.5L V6", 3.5, "V", 6),
|
||||
("5.6L V8", 5.6, "V", 8),
|
||||
("1.6L I4", 1.6, "I", 4),
|
||||
("6.2L V8", 6.2, "V", 8),
|
||||
]
|
||||
|
||||
for engine_str, expected_disp, expected_config, expected_cyl in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
self.assertEqual(spec.displacement_l, expected_disp,
|
||||
f"Displacement: expected {expected_disp}, got {spec.displacement_l}")
|
||||
self.assertEqual(spec.configuration, expected_config,
|
||||
f"Configuration: expected {expected_config}, got {spec.configuration}")
|
||||
self.assertEqual(spec.cylinders, expected_cyl,
|
||||
f"Cylinders: expected {expected_cyl}, got {spec.cylinders}")
|
||||
self.assertEqual(spec.fuel_type, "Gasoline") # Default
|
||||
self.assertEqual(spec.aspiration, "Natural") # Default
|
||||
|
||||
def test_l_to_i_normalization(self):
|
||||
"""Test CRITICAL L→I configuration normalization"""
|
||||
test_cases = [
|
||||
# L-configuration should become I (Inline)
|
||||
("1.5L L3", "I", 3),
|
||||
("2.0L L4", "I", 4),
|
||||
("1.2L L3", "I", 3),
|
||||
]
|
||||
|
||||
for engine_str, expected_config, expected_cyl in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
self.assertEqual(spec.configuration, expected_config,
|
||||
f"L→I normalization failed: '{engine_str}' should become I{expected_cyl}")
|
||||
self.assertEqual(spec.cylinders, expected_cyl)
|
||||
self.assertEqual(spec.raw_string, engine_str) # Original preserved
|
||||
|
||||
def test_subaru_boxer_engines(self):
|
||||
"""Test Subaru Boxer (H-configuration) engines"""
|
||||
test_cases = [
|
||||
("2.4L H4", 2.4, "H", 4),
|
||||
("2.0L H4", 2.0, "H", 4),
|
||||
("2.5L H4", 2.5, "H", 4),
|
||||
]
|
||||
|
||||
for engine_str, expected_disp, expected_config, expected_cyl in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
self.assertEqual(spec.displacement_l, expected_disp)
|
||||
self.assertEqual(spec.configuration, expected_config) # Should remain H
|
||||
self.assertEqual(spec.cylinders, expected_cyl)
|
||||
|
||||
def test_w_configuration_engines(self):
|
||||
"""Test W-configuration engines (VW Group, Bentley)"""
|
||||
test_cases = [
|
||||
("6.0L W12", 6.0, "W", 12),
|
||||
("4.0L W8", 4.0, "W", 8),
|
||||
("8.0L W16", 8.0, "W", 16), # Theoretical case
|
||||
]
|
||||
|
||||
for engine_str, expected_disp, expected_config, expected_cyl in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
self.assertEqual(spec.displacement_l, expected_disp)
|
||||
self.assertEqual(spec.configuration, expected_config) # Should remain W
|
||||
self.assertEqual(spec.cylinders, expected_cyl)
|
||||
|
||||
def test_hybrid_detection(self):
|
||||
"""Test hybrid engine detection patterns"""
|
||||
test_cases = [
|
||||
# Format: (input, expected_fuel_type)
|
||||
("2.5L I4 FULL HYBRID EV- (FHEV)", "Full Hybrid"),
|
||||
("1.5L L3 PLUG-IN HYBRID EV- (PHEV)", "Plug-in Hybrid"),
|
||||
("2.0L I4 FULL HYBRID EV- (FHEV)", "Full Hybrid"),
|
||||
("1.8L I4 HYBRID", "Hybrid"), # Generic hybrid
|
||||
]
|
||||
|
||||
for engine_str, expected_fuel_type in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
self.assertEqual(spec.fuel_type, expected_fuel_type,
|
||||
f"Expected fuel type '{expected_fuel_type}' for '{engine_str}'")
|
||||
|
||||
def test_l_to_i_with_hybrid(self):
|
||||
"""Test L→I normalization combined with hybrid detection"""
|
||||
test_cases = [
|
||||
("1.5L L3 PLUG-IN HYBRID EV- (PHEV)", "I", 3, "Plug-in Hybrid"),
|
||||
("1.2L L3 FULL HYBRID EV- (FHEV)", "I", 3, "Full Hybrid"),
|
||||
]
|
||||
|
||||
for engine_str, expected_config, expected_cyl, expected_fuel in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
# Test both L→I normalization AND hybrid detection
|
||||
self.assertEqual(spec.configuration, expected_config,
|
||||
f"L→I normalization failed for hybrid: '{engine_str}'")
|
||||
self.assertEqual(spec.cylinders, expected_cyl)
|
||||
self.assertEqual(spec.fuel_type, expected_fuel,
|
||||
f"Hybrid detection failed: '{engine_str}'")
|
||||
|
||||
def test_flex_fuel_detection(self):
|
||||
"""Test flex fuel detection"""
|
||||
test_cases = [
|
||||
("5.6L V8 FLEX", "Flex Fuel"),
|
||||
("4.0L V6 FLEX", "Flex Fuel"),
|
||||
]
|
||||
|
||||
for engine_str, expected_fuel_type in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
self.assertEqual(spec.fuel_type, expected_fuel_type)
|
||||
|
||||
def test_electric_detection(self):
|
||||
"""Test electric engine detection"""
|
||||
test_cases = [
|
||||
("1.8L I4 ELECTRIC", "Electric"),
|
||||
]
|
||||
|
||||
for engine_str, expected_fuel_type in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
self.assertEqual(spec.fuel_type, expected_fuel_type)
|
||||
|
||||
def test_aspiration_detection(self):
|
||||
"""Test turbo/supercharged detection"""
|
||||
# Note: These patterns are less common in current JSON data
|
||||
test_cases = [
|
||||
("2.0L I4 TURBO", "Turbocharged"),
|
||||
("6.2L V8 SUPERCHARGED", "Supercharged"),
|
||||
("3.0L V6 SC", "Supercharged"), # SC abbreviation
|
||||
]
|
||||
|
||||
for engine_str, expected_aspiration in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
self.assertEqual(spec.aspiration, expected_aspiration)
|
||||
|
||||
def test_create_electric_motor(self):
|
||||
"""Test electric motor creation for empty engines arrays"""
|
||||
spec = self.parser.create_electric_motor()
|
||||
|
||||
self.assertIsNone(spec.displacement_l)
|
||||
self.assertEqual(spec.configuration, "Electric")
|
||||
self.assertIsNone(spec.cylinders)
|
||||
self.assertEqual(spec.fuel_type, "Electric")
|
||||
self.assertIsNone(spec.aspiration)
|
||||
self.assertEqual(spec.raw_string, "Electric Motor")
|
||||
|
||||
def test_parse_multiple_engines_empty_array(self):
|
||||
"""Test handling of empty engines array (electric vehicles)"""
|
||||
# Empty array should create electric motor
|
||||
specs = self.parser.parse_multiple_engines([])
|
||||
|
||||
self.assertEqual(len(specs), 1)
|
||||
self.assertEqual(specs[0].raw_string, "Electric Motor")
|
||||
self.assertEqual(specs[0].fuel_type, "Electric")
|
||||
|
||||
def test_parse_multiple_engines_normal(self):
|
||||
"""Test parsing multiple engine specifications"""
|
||||
engine_strings = [
|
||||
"2.0L I4",
|
||||
"3.5L V6",
|
||||
"1.5L L3 PLUG-IN HYBRID EV- (PHEV)" # Includes L→I normalization
|
||||
]
|
||||
|
||||
specs = self.parser.parse_multiple_engines(engine_strings)
|
||||
|
||||
self.assertEqual(len(specs), 3)
|
||||
|
||||
# Check first engine
|
||||
self.assertEqual(specs[0].displacement_l, 2.0)
|
||||
self.assertEqual(specs[0].configuration, "I")
|
||||
|
||||
# Check third engine (L→I normalization + hybrid)
|
||||
self.assertEqual(specs[2].configuration, "I") # L normalized to I
|
||||
self.assertEqual(specs[2].fuel_type, "Plug-in Hybrid")
|
||||
|
||||
def test_unparseable_engines(self):
|
||||
"""Test handling of unparseable engine strings"""
|
||||
unparseable_cases = [
|
||||
"Custom Hybrid System",
|
||||
"V12 Twin-Turbo Custom",
|
||||
"V10 Plus",
|
||||
"Unknown Engine Type",
|
||||
"", # Empty string
|
||||
]
|
||||
|
||||
for engine_str in unparseable_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
# Should create fallback engine
|
||||
self.assertEqual(spec.configuration, "Unknown")
|
||||
self.assertEqual(spec.fuel_type, "Unknown")
|
||||
self.assertEqual(spec.aspiration, "Natural")
|
||||
self.assertIsNone(spec.displacement_l)
|
||||
self.assertIsNone(spec.cylinders)
|
||||
self.assertEqual(spec.raw_string, engine_str or "Empty Engine String")
|
||||
|
||||
def test_case_insensitive_parsing(self):
|
||||
"""Test that parsing is case insensitive"""
|
||||
test_cases = [
|
||||
("2.0l i4", 2.0, "I", 4), # Lowercase
|
||||
("3.5L v6", 3.5, "V", 6), # Mixed case
|
||||
("1.5L l3 plug-in hybrid ev- (phev)", "I", 3, "Plug-in Hybrid"), # Lowercase with hybrid
|
||||
]
|
||||
|
||||
for engine_str, expected_disp, expected_config, expected_cyl in test_cases[:2]:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
|
||||
self.assertEqual(spec.displacement_l, expected_disp)
|
||||
self.assertEqual(spec.configuration, expected_config)
|
||||
self.assertEqual(spec.cylinders, expected_cyl)
|
||||
|
||||
# Test hybrid case separately
|
||||
spec = self.parser.parse_engine_string("1.5L l3 plug-in hybrid ev- (phev)")
|
||||
self.assertEqual(spec.configuration, "I") # L→I normalization
|
||||
self.assertEqual(spec.fuel_type, "Plug-in Hybrid")
|
||||
|
||||
def test_get_unique_engines(self):
|
||||
"""Test deduplication of engine specifications"""
|
||||
# Create list with duplicates
|
||||
engine_specs = [
|
||||
self.parser.parse_engine_string("2.0L I4"),
|
||||
self.parser.parse_engine_string("2.0L I4"), # Duplicate
|
||||
self.parser.parse_engine_string("3.5L V6"),
|
||||
self.parser.parse_engine_string("2.0L I4"), # Another duplicate
|
||||
]
|
||||
|
||||
unique_specs = self.parser.get_unique_engines(engine_specs)
|
||||
|
||||
# Should have only 2 unique engines
|
||||
self.assertEqual(len(unique_specs), 2)
|
||||
|
||||
# Check that we have the expected unique engines
|
||||
displacement_configs = [(spec.displacement_l, spec.configuration, spec.cylinders)
|
||||
for spec in unique_specs]
|
||||
|
||||
self.assertIn((2.0, "I", 4), displacement_configs)
|
||||
self.assertIn((3.5, "V", 6), displacement_configs)
|
||||
|
||||
def test_validate_engine_spec(self):
|
||||
"""Test engine specification validation"""
|
||||
# Valid engine
|
||||
valid_spec = self.parser.parse_engine_string("2.0L I4")
|
||||
warnings = self.parser.validate_engine_spec(valid_spec)
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
# Invalid displacement
|
||||
invalid_spec = EngineSpec(
|
||||
displacement_l=-1.0, # Invalid
|
||||
configuration="I",
|
||||
cylinders=4,
|
||||
fuel_type="Gasoline",
|
||||
aspiration="Natural",
|
||||
raw_string="Invalid Engine"
|
||||
)
|
||||
warnings = self.parser.validate_engine_spec(invalid_spec)
|
||||
self.assertGreater(len(warnings), 0)
|
||||
self.assertTrue(any("displacement" in w for w in warnings))
|
||||
|
||||
# Electric motor should be valid
|
||||
electric_spec = self.parser.create_electric_motor()
|
||||
warnings = self.parser.validate_engine_spec(electric_spec)
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
def test_normalize_configuration_directly(self):
|
||||
"""Test the normalize_configuration method directly"""
|
||||
# Test L→I normalization
|
||||
self.assertEqual(self.parser.normalize_configuration('L'), 'I')
|
||||
self.assertEqual(self.parser.normalize_configuration('l'), 'I') # Case insensitive
|
||||
|
||||
# Test other configurations remain unchanged
|
||||
self.assertEqual(self.parser.normalize_configuration('I'), 'I')
|
||||
self.assertEqual(self.parser.normalize_configuration('V'), 'V')
|
||||
self.assertEqual(self.parser.normalize_configuration('H'), 'H')
|
||||
self.assertEqual(self.parser.normalize_configuration('i'), 'I') # Uppercase
|
||||
|
||||
def test_decimal_displacement(self):
|
||||
"""Test engines with decimal displacements"""
|
||||
test_cases = [
|
||||
("1.5L I4", 1.5),
|
||||
("2.3L I4", 2.3),
|
||||
("3.7L V6", 3.7),
|
||||
]
|
||||
|
||||
for engine_str, expected_disp in test_cases:
|
||||
with self.subTest(engine_str=engine_str):
|
||||
spec = self.parser.parse_engine_string(engine_str)
|
||||
self.assertEqual(spec.displacement_l, expected_disp)
|
||||
|
||||
def test_edge_case_patterns(self):
|
||||
"""Test edge cases and boundary conditions"""
|
||||
# Large engines
|
||||
spec = self.parser.parse_engine_string("8.4L V10")
|
||||
self.assertEqual(spec.displacement_l, 8.4)
|
||||
self.assertEqual(spec.cylinders, 10)
|
||||
|
||||
# Small engines
|
||||
spec = self.parser.parse_engine_string("1.0L I3")
|
||||
self.assertEqual(spec.displacement_l, 1.0)
|
||||
self.assertEqual(spec.cylinders, 3)
|
||||
|
||||
# Very small displacement
|
||||
spec = self.parser.parse_engine_string("0.7L I3")
|
||||
self.assertEqual(spec.displacement_l, 0.7)
|
||||
|
||||
|
||||
class TestEngineSpec(unittest.TestCase):
|
||||
"""Test cases for EngineSpec dataclass"""
|
||||
|
||||
def test_engine_spec_creation(self):
|
||||
"""Test EngineSpec creation and string representation"""
|
||||
spec = EngineSpec(
|
||||
displacement_l=2.0,
|
||||
configuration="I",
|
||||
cylinders=4,
|
||||
fuel_type="Gasoline",
|
||||
aspiration="Natural",
|
||||
raw_string="2.0L I4"
|
||||
)
|
||||
|
||||
self.assertEqual(spec.displacement_l, 2.0)
|
||||
self.assertEqual(spec.configuration, "I")
|
||||
self.assertEqual(spec.cylinders, 4)
|
||||
self.assertEqual(spec.fuel_type, "Gasoline")
|
||||
self.assertEqual(spec.aspiration, "Natural")
|
||||
self.assertEqual(spec.raw_string, "2.0L I4")
|
||||
|
||||
# Test string representation
|
||||
str_repr = str(spec)
|
||||
self.assertIn("2.0L I4", str_repr)
|
||||
self.assertIn("Gasoline", str_repr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Run specific test for L→I normalization if needed
|
||||
if len(unittest.sys.argv) > 1 and 'l_to_i' in unittest.sys.argv[1].lower():
|
||||
suite = unittest.TestSuite()
|
||||
suite.addTest(TestEngineSpecParser('test_l_to_i_normalization'))
|
||||
suite.addTest(TestEngineSpecParser('test_l_to_i_with_hybrid'))
|
||||
runner = unittest.TextTestRunner(verbosity=2)
|
||||
runner.run(suite)
|
||||
else:
|
||||
unittest.main(verbosity=2)
|
||||
427
mvp-platform-services/vehicles/etl/tests/test_json_extractor.py
Normal file
427
mvp-platform-services/vehicles/etl/tests/test_json_extractor.py
Normal file
@@ -0,0 +1,427 @@
|
||||
"""
|
||||
Unit Tests for JsonExtractor
|
||||
|
||||
Tests the JSON extraction functionality including:
|
||||
- JSON structure validation
|
||||
- Make/model/year/trim/engine extraction
|
||||
- Electric vehicle handling (empty engines arrays)
|
||||
- Data normalization and quality assurance
|
||||
- Error handling and reporting
|
||||
- Integration with MakeNameMapper and EngineSpecParser
|
||||
"""
|
||||
|
||||
import unittest
|
||||
import tempfile
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
# Import the classes we're testing
|
||||
from ..extractors.json_extractor import (
|
||||
JsonExtractor, MakeData, ModelData, ExtractionResult, ValidationResult
|
||||
)
|
||||
from ..utils.make_name_mapper import MakeNameMapper
|
||||
from ..utils.engine_spec_parser import EngineSpecParser, EngineSpec
|
||||
|
||||
|
||||
class TestJsonExtractor(unittest.TestCase):
|
||||
"""Test cases for JsonExtractor functionality"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test environment before each test"""
|
||||
self.make_mapper = MakeNameMapper()
|
||||
self.engine_parser = EngineSpecParser()
|
||||
self.extractor = JsonExtractor(self.make_mapper, self.engine_parser)
|
||||
|
||||
def create_test_json_file(self, filename: str, content: dict) -> str:
|
||||
"""Create a temporary JSON file for testing"""
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
file_path = os.path.join(temp_dir, filename)
|
||||
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(content, f)
|
||||
|
||||
return file_path
|
||||
|
||||
def test_validate_json_structure_valid(self):
|
||||
"""Test JSON structure validation with valid data"""
|
||||
valid_json = {
|
||||
"toyota": [
|
||||
{
|
||||
"year": "2024",
|
||||
"models": [
|
||||
{
|
||||
"name": "camry",
|
||||
"engines": ["2.5L I4", "3.5L V6"],
|
||||
"submodels": ["LE", "XLE", "XSE"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
result = self.extractor.validate_json_structure(valid_json, "toyota.json")
|
||||
|
||||
self.assertTrue(result.is_valid)
|
||||
self.assertEqual(len(result.errors), 0)
|
||||
|
||||
def test_validate_json_structure_invalid_top_level(self):
|
||||
"""Test JSON validation with invalid top-level structure"""
|
||||
invalid_json = ["not", "a", "dict"]
|
||||
|
||||
result = self.extractor.validate_json_structure(invalid_json, "test.json")
|
||||
|
||||
self.assertFalse(result.is_valid)
|
||||
self.assertGreater(len(result.errors), 0)
|
||||
self.assertIn("must be a dictionary", result.errors[0])
|
||||
|
||||
def test_validate_json_structure_multiple_keys(self):
|
||||
"""Test JSON validation with multiple top-level keys"""
|
||||
invalid_json = {
|
||||
"toyota": [],
|
||||
"honda": []
|
||||
}
|
||||
|
||||
result = self.extractor.validate_json_structure(invalid_json, "test.json")
|
||||
|
||||
self.assertFalse(result.is_valid)
|
||||
self.assertIn("exactly one top-level key", result.errors[0])
|
||||
|
||||
def test_validate_json_structure_missing_required_fields(self):
|
||||
"""Test JSON validation with missing required fields"""
|
||||
invalid_json = {
|
||||
"toyota": [
|
||||
{
|
||||
# Missing 'year' field
|
||||
"models": [
|
||||
{
|
||||
# Missing 'name' field
|
||||
"engines": ["2.5L I4"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
result = self.extractor.validate_json_structure(invalid_json, "test.json")
|
||||
|
||||
self.assertFalse(result.is_valid)
|
||||
self.assertTrue(any("missing 'year' field" in error for error in result.errors))
|
||||
self.assertTrue(any("missing 'name' field" in error for error in result.errors))
|
||||
|
||||
def test_extract_make_data_simple(self):
|
||||
"""Test extraction of simple make data"""
|
||||
test_json = {
|
||||
"toyota": [
|
||||
{
|
||||
"year": "2024",
|
||||
"models": [
|
||||
{
|
||||
"name": "camry",
|
||||
"engines": ["2.5L I4", "3.5L V6"],
|
||||
"submodels": ["LE", "XLE"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
json_file = self.create_test_json_file("toyota.json", test_json)
|
||||
|
||||
try:
|
||||
make_data = self.extractor.extract_make_data(json_file)
|
||||
|
||||
self.assertEqual(make_data.name, "Toyota")
|
||||
self.assertEqual(make_data.filename, "toyota.json")
|
||||
self.assertEqual(len(make_data.models), 1)
|
||||
self.assertEqual(len(make_data.processing_errors), 0)
|
||||
|
||||
# Check model data
|
||||
model = make_data.models[0]
|
||||
self.assertEqual(model.name, "camry")
|
||||
self.assertEqual(model.years, [2024])
|
||||
self.assertEqual(len(model.engines), 2)
|
||||
self.assertEqual(len(model.trims), 2)
|
||||
self.assertFalse(model.is_electric)
|
||||
|
||||
finally:
|
||||
os.unlink(json_file)
|
||||
|
||||
def test_extract_make_data_electric_vehicle(self):
|
||||
"""Test extraction with electric vehicle (empty engines array)"""
|
||||
test_json = {
|
||||
"tesla": [
|
||||
{
|
||||
"year": "2024",
|
||||
"models": [
|
||||
{
|
||||
"name": "model s",
|
||||
"engines": [], # Empty engines - electric vehicle
|
||||
"submodels": ["Base", "Plaid"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
json_file = self.create_test_json_file("tesla.json", test_json)
|
||||
|
||||
try:
|
||||
make_data = self.extractor.extract_make_data(json_file)
|
||||
|
||||
self.assertEqual(make_data.name, "Tesla")
|
||||
self.assertEqual(len(make_data.models), 1)
|
||||
|
||||
model = make_data.models[0]
|
||||
self.assertTrue(model.is_electric)
|
||||
self.assertEqual(len(model.engines), 1) # Should get default electric motor
|
||||
self.assertEqual(model.engines[0].fuel_type, "Electric")
|
||||
self.assertEqual(model.engines[0].configuration, "Electric")
|
||||
|
||||
finally:
|
||||
os.unlink(json_file)
|
||||
|
||||
def test_extract_make_data_multiple_years(self):
|
||||
"""Test extraction with model appearing across multiple years"""
|
||||
test_json = {
|
||||
"honda": [
|
||||
{
|
||||
"year": "2023",
|
||||
"models": [
|
||||
{
|
||||
"name": "civic",
|
||||
"engines": ["1.5L I4"],
|
||||
"submodels": ["LX", "EX"]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"models": [
|
||||
{
|
||||
"name": "civic",
|
||||
"engines": ["1.5L I4", "2.0L I4"],
|
||||
"submodels": ["LX", "EX", "Type R"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
json_file = self.create_test_json_file("honda.json", test_json)
|
||||
|
||||
try:
|
||||
make_data = self.extractor.extract_make_data(json_file)
|
||||
|
||||
self.assertEqual(len(make_data.models), 1) # Should merge into one model
|
||||
|
||||
model = make_data.models[0]
|
||||
self.assertEqual(model.name, "civic")
|
||||
self.assertEqual(sorted(model.years), [2023, 2024])
|
||||
self.assertEqual(len(model.engines), 2) # Should have both engines
|
||||
self.assertEqual(len(model.trims), 3) # Should have unique trims
|
||||
|
||||
finally:
|
||||
os.unlink(json_file)
|
||||
|
||||
def test_extract_make_data_l_to_i_normalization(self):
|
||||
"""Test that L→I normalization is applied during extraction"""
|
||||
test_json = {
|
||||
"geo": [
|
||||
{
|
||||
"year": "1995",
|
||||
"models": [
|
||||
{
|
||||
"name": "metro",
|
||||
"engines": ["1.0L L3", "1.3L I4"], # L3 should become I3
|
||||
"submodels": ["Base", "LSi"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
json_file = self.create_test_json_file("geo.json", test_json)
|
||||
|
||||
try:
|
||||
make_data = self.extractor.extract_make_data(json_file)
|
||||
|
||||
model = make_data.models[0]
|
||||
|
||||
# Find the L3 engine (should be normalized to I3)
|
||||
l3_engine = None
|
||||
for engine in model.engines:
|
||||
if engine.displacement_l == 1.0 and engine.cylinders == 3:
|
||||
l3_engine = engine
|
||||
break
|
||||
|
||||
self.assertIsNotNone(l3_engine)
|
||||
self.assertEqual(l3_engine.configuration, "I") # Should be normalized from L
|
||||
|
||||
finally:
|
||||
os.unlink(json_file)
|
||||
|
||||
def test_extract_make_data_invalid_json(self):
|
||||
"""Test extraction with invalid JSON file"""
|
||||
json_file = self.create_test_json_file("invalid.json", {"invalid": "structure"})
|
||||
|
||||
try:
|
||||
make_data = self.extractor.extract_make_data(json_file)
|
||||
|
||||
# Should return make data with errors
|
||||
self.assertEqual(make_data.name, "Invalid")
|
||||
self.assertEqual(len(make_data.models), 0)
|
||||
self.assertGreater(len(make_data.processing_errors), 0)
|
||||
|
||||
finally:
|
||||
os.unlink(json_file)
|
||||
|
||||
def test_extract_all_makes_multiple_files(self):
|
||||
"""Test extraction of multiple make files"""
|
||||
# Create temporary directory with multiple JSON files
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
|
||||
try:
|
||||
# Create test files
|
||||
toyota_json = {"toyota": [{"year": "2024", "models": [{"name": "camry", "engines": ["2.5L I4"], "submodels": ["LE"]}]}]}
|
||||
tesla_json = {"tesla": [{"year": "2024", "models": [{"name": "model s", "engines": [], "submodels": ["Base"]}]}]}
|
||||
|
||||
toyota_file = os.path.join(temp_dir, "toyota.json")
|
||||
tesla_file = os.path.join(temp_dir, "tesla.json")
|
||||
|
||||
with open(toyota_file, 'w') as f:
|
||||
json.dump(toyota_json, f)
|
||||
with open(tesla_file, 'w') as f:
|
||||
json.dump(tesla_json, f)
|
||||
|
||||
# Extract all makes
|
||||
result = self.extractor.extract_all_makes(temp_dir)
|
||||
|
||||
self.assertEqual(result.total_files_processed, 2)
|
||||
self.assertEqual(result.successful_extractions, 2)
|
||||
self.assertEqual(result.failed_extractions, 0)
|
||||
self.assertEqual(len(result.makes), 2)
|
||||
self.assertEqual(result.total_models, 2)
|
||||
self.assertEqual(result.total_engines, 2) # Toyota: 1, Tesla: 1 (electric)
|
||||
self.assertEqual(result.total_electric_models, 1) # Tesla
|
||||
|
||||
# Check make names
|
||||
make_names = [make.name for make in result.makes]
|
||||
self.assertIn("Toyota", make_names)
|
||||
self.assertIn("Tesla", make_names)
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
for file in os.listdir(temp_dir):
|
||||
os.unlink(os.path.join(temp_dir, file))
|
||||
os.rmdir(temp_dir)
|
||||
|
||||
def test_extract_all_makes_empty_directory(self):
|
||||
"""Test extraction from empty directory"""
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
|
||||
try:
|
||||
result = self.extractor.extract_all_makes(temp_dir)
|
||||
|
||||
self.assertEqual(result.total_files_processed, 0)
|
||||
self.assertEqual(result.successful_extractions, 0)
|
||||
self.assertEqual(result.failed_extractions, 0)
|
||||
self.assertEqual(len(result.makes), 0)
|
||||
|
||||
finally:
|
||||
os.rmdir(temp_dir)
|
||||
|
||||
def test_get_extraction_statistics(self):
|
||||
"""Test extraction statistics generation"""
|
||||
# Create mock extraction result
|
||||
make1 = MakeData("Toyota", "toyota.json", [], [], [])
|
||||
make1.models = [ModelData("camry", [2024], [], [], False)]
|
||||
|
||||
make2 = MakeData("Tesla", "tesla.json", [], [], [])
|
||||
make2.models = [ModelData("model s", [2024], [], [], True)]
|
||||
|
||||
result = ExtractionResult(
|
||||
makes=[make1, make2],
|
||||
total_files_processed=2,
|
||||
successful_extractions=2,
|
||||
failed_extractions=0,
|
||||
total_models=2,
|
||||
total_engines=2,
|
||||
total_electric_models=1
|
||||
)
|
||||
|
||||
stats = self.extractor.get_extraction_statistics(result)
|
||||
|
||||
self.assertEqual(stats['files']['total_processed'], 2)
|
||||
self.assertEqual(stats['files']['successful'], 2)
|
||||
self.assertEqual(stats['files']['success_rate'], 1.0)
|
||||
self.assertEqual(stats['data']['total_makes'], 2)
|
||||
self.assertEqual(stats['data']['total_models'], 2)
|
||||
self.assertEqual(stats['data']['electric_models'], 1)
|
||||
self.assertEqual(len(stats['makes']), 2)
|
||||
|
||||
|
||||
class TestDataStructures(unittest.TestCase):
|
||||
"""Test cases for data structure classes"""
|
||||
|
||||
def test_validation_result(self):
|
||||
"""Test ValidationResult properties"""
|
||||
result = ValidationResult(True, [], ["warning"])
|
||||
|
||||
self.assertTrue(result.is_valid)
|
||||
self.assertFalse(result.has_errors)
|
||||
self.assertTrue(result.has_warnings)
|
||||
|
||||
def test_model_data_properties(self):
|
||||
"""Test ModelData calculated properties"""
|
||||
# Create mock engine specs
|
||||
engines = [
|
||||
EngineSpec(2.5, "I", 4, "Gasoline", "Natural", "2.5L I4"),
|
||||
EngineSpec(3.5, "V", 6, "Gasoline", "Natural", "3.5L V6")
|
||||
]
|
||||
|
||||
model = ModelData(
|
||||
name="camry",
|
||||
years=[2023, 2024],
|
||||
engines=engines,
|
||||
trims=["LE", "XLE", "XSE"],
|
||||
is_electric=False
|
||||
)
|
||||
|
||||
self.assertEqual(model.total_trims, 3)
|
||||
self.assertEqual(model.total_engines, 2)
|
||||
self.assertEqual(model.year_range, "2023-2024")
|
||||
|
||||
def test_model_data_single_year(self):
|
||||
"""Test ModelData with single year"""
|
||||
model = ModelData("camry", [2024], [], ["LE"])
|
||||
self.assertEqual(model.year_range, "2024")
|
||||
|
||||
def test_make_data_properties(self):
|
||||
"""Test MakeData calculated properties"""
|
||||
model1 = ModelData("camry", [2024], [], ["LE", "XLE"], False)
|
||||
model2 = ModelData("prius", [2024], [], ["L", "LE"], True) # Electric
|
||||
|
||||
make = MakeData("Toyota", "toyota.json", [model1, model2], [], [])
|
||||
|
||||
self.assertEqual(make.total_models, 2)
|
||||
self.assertEqual(make.total_trims, 4)
|
||||
self.assertEqual(make.electric_models_count, 1)
|
||||
|
||||
def test_extraction_result_properties(self):
|
||||
"""Test ExtractionResult calculated properties"""
|
||||
result = ExtractionResult(
|
||||
makes=[],
|
||||
total_files_processed=10,
|
||||
successful_extractions=8,
|
||||
failed_extractions=2,
|
||||
total_models=100,
|
||||
total_engines=500,
|
||||
total_electric_models=25
|
||||
)
|
||||
|
||||
self.assertEqual(result.success_rate, 0.8)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,443 @@
|
||||
"""
|
||||
Unit Tests for JsonManualLoader
|
||||
|
||||
Tests the database loading functionality including:
|
||||
- Clear/Append mode operations
|
||||
- Referential integrity maintenance
|
||||
- Duplicate handling and conflict resolution
|
||||
- Load statistics and reporting
|
||||
- Error handling and rollback scenarios
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from typing import List
|
||||
|
||||
# Import the classes we're testing
|
||||
from ..loaders.json_manual_loader import (
|
||||
JsonManualLoader, LoadMode, LoadResult, LoadStatistics
|
||||
)
|
||||
from ..extractors.json_extractor import MakeData, ModelData
|
||||
from ..utils.engine_spec_parser import EngineSpec
|
||||
|
||||
|
||||
class TestJsonManualLoader(unittest.TestCase):
|
||||
"""Test cases for JsonManualLoader functionality"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test environment before each test"""
|
||||
self.mock_postgres_loader = Mock()
|
||||
self.loader = JsonManualLoader(self.mock_postgres_loader)
|
||||
|
||||
def create_test_engine_spec(self, displacement: float, config: str, cylinders: int, fuel_type: str = "Gasoline") -> EngineSpec:
|
||||
"""Create a test engine specification"""
|
||||
return EngineSpec(
|
||||
displacement_l=displacement,
|
||||
configuration=config,
|
||||
cylinders=cylinders,
|
||||
fuel_type=fuel_type,
|
||||
aspiration="Natural",
|
||||
raw_string=f"{displacement}L {config}{cylinders}"
|
||||
)
|
||||
|
||||
def create_test_model_data(self, name: str, years: List[int], engine_count: int = 1, trim_count: int = 2, is_electric: bool = False) -> ModelData:
|
||||
"""Create test model data"""
|
||||
engines = []
|
||||
if is_electric:
|
||||
engines = [EngineSpec(None, "Electric", None, "Electric", None, "Electric Motor")]
|
||||
else:
|
||||
for i in range(engine_count):
|
||||
engines.append(self.create_test_engine_spec(2.0 + i, "I", 4))
|
||||
|
||||
trims = [f"Trim_{i}" for i in range(trim_count)]
|
||||
|
||||
return ModelData(
|
||||
name=name,
|
||||
years=years,
|
||||
engines=engines,
|
||||
trims=trims,
|
||||
is_electric=is_electric
|
||||
)
|
||||
|
||||
def create_test_make_data(self, name: str, model_count: int = 2) -> MakeData:
|
||||
"""Create test make data"""
|
||||
models = []
|
||||
for i in range(model_count):
|
||||
models.append(self.create_test_model_data(f"Model_{i}", [2024], 1, 2))
|
||||
|
||||
return MakeData(
|
||||
name=name,
|
||||
filename=f"{name.lower()}.json",
|
||||
models=models,
|
||||
processing_errors=[],
|
||||
processing_warnings=[]
|
||||
)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_load_make_append_mode_new_make(self, mock_conn_context):
|
||||
"""Test loading a new make in append mode"""
|
||||
# Mock database connection and cursor
|
||||
mock_conn = Mock()
|
||||
mock_cursor = Mock()
|
||||
mock_conn_context.return_value.__enter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
|
||||
# Mock make doesn't exist (returns None)
|
||||
mock_cursor.fetchone.return_value = None
|
||||
# Mock make insert returns ID
|
||||
mock_cursor.fetchone.side_effect = [10] # make ID
|
||||
|
||||
# Create test data
|
||||
make_data = self.create_test_make_data("Toyota", 1)
|
||||
stats = LoadStatistics()
|
||||
|
||||
# Test loading
|
||||
make_id = self.loader.load_make(make_data, LoadMode.APPEND, stats)
|
||||
|
||||
self.assertEqual(make_id, 10)
|
||||
self.assertEqual(stats.makes_processed, 1)
|
||||
|
||||
# Verify make insert was called
|
||||
mock_cursor.execute.assert_any_call(
|
||||
"INSERT INTO vehicles.make (name) VALUES (%s) RETURNING id",
|
||||
("Toyota",)
|
||||
)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_load_make_append_mode_existing_make(self, mock_conn_context):
|
||||
"""Test loading an existing make in append mode"""
|
||||
# Mock database connection and cursor
|
||||
mock_conn = Mock()
|
||||
mock_cursor = Mock()
|
||||
mock_conn_context.return_value.__enter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
|
||||
# Mock make exists (returns ID)
|
||||
mock_cursor.fetchone.return_value = (5,) # existing make ID
|
||||
|
||||
# Create test data
|
||||
make_data = self.create_test_make_data("Honda", 1)
|
||||
stats = LoadStatistics()
|
||||
|
||||
# Test loading
|
||||
make_id = self.loader.load_make(make_data, LoadMode.APPEND, stats)
|
||||
|
||||
self.assertEqual(make_id, 5)
|
||||
self.assertEqual(stats.duplicate_makes, 1)
|
||||
|
||||
# Verify make select was called, but not insert
|
||||
mock_cursor.execute.assert_any_call(
|
||||
"SELECT id FROM vehicles.make WHERE name = %s",
|
||||
("Honda",)
|
||||
)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_load_engine_with_full_spec(self, mock_conn_context):
|
||||
"""Test loading engine with complete specification"""
|
||||
# Mock database cursor
|
||||
mock_cursor = Mock()
|
||||
|
||||
# Mock engine doesn't exist
|
||||
mock_cursor.fetchone.side_effect = [None, 100] # check, then insert
|
||||
|
||||
# Create test engine
|
||||
engine_spec = self.create_test_engine_spec(2.5, "V", 6, "Gasoline")
|
||||
stats = LoadStatistics()
|
||||
|
||||
# Test loading
|
||||
engine_id = self.loader.load_engine(mock_cursor, engine_spec, LoadMode.APPEND, stats)
|
||||
|
||||
self.assertEqual(engine_id, 100)
|
||||
self.assertEqual(stats.engines_inserted, 1)
|
||||
|
||||
# Verify engine insert with correct parameters
|
||||
expected_call = mock_cursor.execute.call_args_list[-1]
|
||||
args = expected_call[0]
|
||||
|
||||
self.assertIn("INSERT INTO vehicles.engine", args[0])
|
||||
self.assertEqual(args[1][0], "2.5L V6") # engine name
|
||||
self.assertEqual(args[1][1], "2.5lv6") # engine code
|
||||
self.assertEqual(args[1][2], 2.5) # displacement
|
||||
self.assertEqual(args[1][3], 6) # cylinders
|
||||
self.assertEqual(args[1][4], "Gasoline") # fuel type
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_load_engine_electric(self, mock_conn_context):
|
||||
"""Test loading electric motor"""
|
||||
# Mock database cursor
|
||||
mock_cursor = Mock()
|
||||
|
||||
# Mock engine doesn't exist
|
||||
mock_cursor.fetchone.side_effect = [None, 200] # check, then insert
|
||||
|
||||
# Create electric engine
|
||||
engine_spec = EngineSpec(None, "Electric", None, "Electric", None, "Electric Motor")
|
||||
stats = LoadStatistics()
|
||||
|
||||
# Test loading
|
||||
engine_id = self.loader.load_engine(mock_cursor, engine_spec, LoadMode.APPEND, stats)
|
||||
|
||||
self.assertEqual(engine_id, 200)
|
||||
self.assertEqual(stats.engines_inserted, 1)
|
||||
|
||||
# Verify electric motor was inserted correctly
|
||||
expected_call = mock_cursor.execute.call_args_list[-1]
|
||||
args = expected_call[0]
|
||||
|
||||
self.assertIn("INSERT INTO vehicles.engine", args[0])
|
||||
self.assertEqual(args[1][0], "Electric Motor") # engine name
|
||||
self.assertEqual(args[1][4], "Electric") # fuel type
|
||||
|
||||
def test_load_mode_enum(self):
|
||||
"""Test LoadMode enum values"""
|
||||
self.assertEqual(LoadMode.CLEAR.value, "clear")
|
||||
self.assertEqual(LoadMode.APPEND.value, "append")
|
||||
|
||||
def test_load_result_properties(self):
|
||||
"""Test LoadResult calculated properties"""
|
||||
result = LoadResult(
|
||||
total_makes=10,
|
||||
total_models=50,
|
||||
total_model_years=100,
|
||||
total_trims=200,
|
||||
total_engines=75,
|
||||
total_trim_engine_mappings=400,
|
||||
failed_makes=["BMW", "Audi"],
|
||||
warnings=["Warning 1"],
|
||||
load_mode=LoadMode.APPEND
|
||||
)
|
||||
|
||||
self.assertEqual(result.success_count, 8) # 10 - 2 failed
|
||||
self.assertEqual(result.success_rate, 0.8) # 8/10
|
||||
|
||||
def test_load_statistics_initialization(self):
|
||||
"""Test LoadStatistics initialization"""
|
||||
stats = LoadStatistics()
|
||||
|
||||
self.assertEqual(stats.makes_processed, 0)
|
||||
self.assertEqual(stats.makes_skipped, 0)
|
||||
self.assertEqual(len(stats.errors), 0)
|
||||
self.assertEqual(len(stats.warnings), 0)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_clear_all_tables(self, mock_conn_context):
|
||||
"""Test clearing all tables in correct order"""
|
||||
# Mock database connection and cursor
|
||||
mock_conn = Mock()
|
||||
mock_cursor = Mock()
|
||||
mock_conn_context.return_value.__enter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
|
||||
# Test clearing
|
||||
self.loader.clear_all_tables()
|
||||
|
||||
# Verify truncate calls were made
|
||||
truncate_calls = [call for call in mock_cursor.execute.call_args_list
|
||||
if 'TRUNCATE' in str(call)]
|
||||
|
||||
self.assertGreater(len(truncate_calls), 0)
|
||||
|
||||
# Verify commit was called
|
||||
mock_conn.commit.assert_called()
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_get_database_statistics(self, mock_conn_context):
|
||||
"""Test getting database statistics"""
|
||||
# Mock database connection and cursor
|
||||
mock_conn = Mock()
|
||||
mock_cursor = Mock()
|
||||
mock_conn_context.return_value.__enter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
|
||||
# Mock count queries return values
|
||||
mock_cursor.fetchone.side_effect = [(10,), (50,), (100,), (200,), (75,), (400,)]
|
||||
|
||||
# Test getting statistics
|
||||
stats = self.loader.get_database_statistics()
|
||||
|
||||
expected_stats = {
|
||||
'make': 10,
|
||||
'model': 50,
|
||||
'model_year': 100,
|
||||
'trim': 200,
|
||||
'engine': 75,
|
||||
'trim_engine': 400
|
||||
}
|
||||
|
||||
self.assertEqual(stats, expected_stats)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_validate_referential_integrity_clean(self, mock_conn_context):
|
||||
"""Test referential integrity validation with clean data"""
|
||||
# Mock database connection and cursor
|
||||
mock_conn = Mock()
|
||||
mock_cursor = Mock()
|
||||
mock_conn_context.return_value.__enter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
|
||||
# Mock all integrity checks return 0 (no issues)
|
||||
mock_cursor.fetchone.side_effect = [(0,), (0,), (0,), (0,)]
|
||||
|
||||
# Test validation
|
||||
issues = self.loader.validate_referential_integrity()
|
||||
|
||||
self.assertEqual(len(issues), 0)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_validate_referential_integrity_issues(self, mock_conn_context):
|
||||
"""Test referential integrity validation with issues found"""
|
||||
# Mock database connection and cursor
|
||||
mock_conn = Mock()
|
||||
mock_cursor = Mock()
|
||||
mock_conn_context.return_value.__enter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
|
||||
# Mock integrity checks return issues
|
||||
mock_cursor.fetchone.side_effect = [(2,), (0,), (1,), (0,)] # 2 orphaned models, 1 orphaned trim
|
||||
|
||||
# Test validation
|
||||
issues = self.loader.validate_referential_integrity()
|
||||
|
||||
self.assertEqual(len(issues), 2)
|
||||
self.assertIn("orphaned models", issues[0])
|
||||
self.assertIn("orphaned trims", issues[1])
|
||||
|
||||
def test_load_all_makes_with_errors(self):
|
||||
"""Test loading makes where some have processing errors"""
|
||||
# Create test data with errors
|
||||
good_make = self.create_test_make_data("Toyota", 1)
|
||||
bad_make = MakeData(
|
||||
name="ErrorMake",
|
||||
filename="error.json",
|
||||
models=[],
|
||||
processing_errors=["JSON parse error"],
|
||||
processing_warnings=[]
|
||||
)
|
||||
|
||||
makes_data = [good_make, bad_make]
|
||||
|
||||
# Mock load_make to work for good make
|
||||
with patch.object(self.loader, 'load_make') as mock_load_make:
|
||||
mock_load_make.return_value = 1
|
||||
|
||||
# Test loading
|
||||
result = self.loader.load_all_makes(makes_data, LoadMode.APPEND)
|
||||
|
||||
self.assertEqual(result.total_makes, 2)
|
||||
self.assertEqual(result.success_count, 1) # Only good make succeeded
|
||||
self.assertIn("ErrorMake", result.failed_makes)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_load_model_new_model(self, mock_conn_context):
|
||||
"""Test loading a new model"""
|
||||
# Mock database cursor
|
||||
mock_cursor = Mock()
|
||||
|
||||
# Mock model doesn't exist, then return new ID
|
||||
mock_cursor.fetchone.side_effect = [None, 25] # check, then insert
|
||||
|
||||
# Create test data
|
||||
model_data = self.create_test_model_data("Camry", [2024], 1, 2)
|
||||
stats = LoadStatistics()
|
||||
|
||||
# Mock load_model_year
|
||||
with patch.object(self.loader, 'load_model_year') as mock_load_model_year:
|
||||
mock_load_model_year.return_value = 100
|
||||
|
||||
# Test loading
|
||||
model_id = self.loader.load_model(mock_cursor, 1, model_data, LoadMode.APPEND, stats)
|
||||
|
||||
self.assertEqual(model_id, 25)
|
||||
self.assertEqual(stats.models_inserted, 1)
|
||||
|
||||
# Verify model_year was loaded for each year
|
||||
mock_load_model_year.assert_called_once_with(mock_cursor, 25, 2024, model_data, LoadMode.APPEND, stats)
|
||||
|
||||
@patch('loaders.json_manual_loader.db_connections.postgres_connection')
|
||||
def test_load_trim_with_engines(self, mock_conn_context):
|
||||
"""Test loading trim and connecting to engines"""
|
||||
# Mock database cursor
|
||||
mock_cursor = Mock()
|
||||
|
||||
# Mock trim doesn't exist, then return new ID
|
||||
mock_cursor.fetchone.side_effect = [None, 50, None] # trim check, insert, mapping check
|
||||
|
||||
# Test data
|
||||
stats = LoadStatistics()
|
||||
engine_ids = [100, 101]
|
||||
|
||||
# Test loading
|
||||
trim_id = self.loader.load_trim(mock_cursor, 1, "Sport", engine_ids, LoadMode.APPEND, stats)
|
||||
|
||||
self.assertEqual(trim_id, 50)
|
||||
self.assertEqual(stats.trims_inserted, 1)
|
||||
self.assertEqual(stats.trim_engine_mappings_inserted, 2) # One for each engine
|
||||
|
||||
|
||||
class TestLoadModeAndResults(unittest.TestCase):
|
||||
"""Test LoadMode enum and result classes"""
|
||||
|
||||
def test_load_mode_values(self):
|
||||
"""Test LoadMode enum has correct values"""
|
||||
self.assertEqual(LoadMode.CLEAR.value, "clear")
|
||||
self.assertEqual(LoadMode.APPEND.value, "append")
|
||||
|
||||
def test_load_result_with_no_failures(self):
|
||||
"""Test LoadResult with perfect success"""
|
||||
result = LoadResult(
|
||||
total_makes=5,
|
||||
total_models=25,
|
||||
total_model_years=50,
|
||||
total_trims=100,
|
||||
total_engines=40,
|
||||
total_trim_engine_mappings=200,
|
||||
failed_makes=[],
|
||||
warnings=[],
|
||||
load_mode=LoadMode.APPEND
|
||||
)
|
||||
|
||||
self.assertEqual(result.success_count, 5)
|
||||
self.assertEqual(result.success_rate, 1.0)
|
||||
|
||||
def test_load_result_with_failures(self):
|
||||
"""Test LoadResult with some failures"""
|
||||
result = LoadResult(
|
||||
total_makes=10,
|
||||
total_models=40,
|
||||
total_model_years=80,
|
||||
total_trims=160,
|
||||
total_engines=60,
|
||||
total_trim_engine_mappings=320,
|
||||
failed_makes=["BMW", "Audi", "Mercedes"],
|
||||
warnings=["Warning 1", "Warning 2"],
|
||||
load_mode=LoadMode.CLEAR
|
||||
)
|
||||
|
||||
self.assertEqual(result.success_count, 7) # 10 - 3 failed
|
||||
self.assertEqual(result.success_rate, 0.7)
|
||||
|
||||
def test_load_statistics_defaults(self):
|
||||
"""Test LoadStatistics default values"""
|
||||
stats = LoadStatistics()
|
||||
|
||||
# Check all counters start at 0
|
||||
self.assertEqual(stats.makes_processed, 0)
|
||||
self.assertEqual(stats.makes_skipped, 0)
|
||||
self.assertEqual(stats.models_inserted, 0)
|
||||
self.assertEqual(stats.model_years_inserted, 0)
|
||||
self.assertEqual(stats.trims_inserted, 0)
|
||||
self.assertEqual(stats.engines_inserted, 0)
|
||||
self.assertEqual(stats.trim_engine_mappings_inserted, 0)
|
||||
self.assertEqual(stats.duplicate_makes, 0)
|
||||
self.assertEqual(stats.duplicate_models, 0)
|
||||
self.assertEqual(stats.duplicate_engines, 0)
|
||||
|
||||
# Check lists are initialized
|
||||
self.assertIsInstance(stats.errors, list)
|
||||
self.assertIsInstance(stats.warnings, list)
|
||||
self.assertEqual(len(stats.errors), 0)
|
||||
self.assertEqual(len(stats.warnings), 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,285 @@
|
||||
"""
|
||||
Unit Tests for MakeNameMapper
|
||||
|
||||
Tests the make name normalization functionality including:
|
||||
- Standard filename to display name conversion
|
||||
- Special capitalization cases (BMW, GMC, etc.)
|
||||
- Multi-word names (underscore to space conversion)
|
||||
- Validation against authoritative makes list
|
||||
- Error handling for edge cases
|
||||
"""
|
||||
|
||||
import unittest
|
||||
import tempfile
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch, mock_open
|
||||
|
||||
# Import the class we're testing
|
||||
from ..utils.make_name_mapper import MakeNameMapper, ValidationReport
|
||||
|
||||
|
||||
class TestMakeNameMapper(unittest.TestCase):
|
||||
"""Test cases for MakeNameMapper utility"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test environment before each test"""
|
||||
self.mapper = MakeNameMapper()
|
||||
|
||||
def test_normalize_single_word_makes(self):
|
||||
"""Test normalization of single-word make names"""
|
||||
test_cases = [
|
||||
('toyota.json', 'Toyota'),
|
||||
('honda.json', 'Honda'),
|
||||
('ford.json', 'Ford'),
|
||||
('audi.json', 'Audi'),
|
||||
('tesla.json', 'Tesla'),
|
||||
]
|
||||
|
||||
for filename, expected in test_cases:
|
||||
with self.subTest(filename=filename):
|
||||
result = self.mapper.normalize_make_name(filename)
|
||||
self.assertEqual(result, expected,
|
||||
f"Expected '{expected}' for '{filename}', got '{result}'")
|
||||
|
||||
def test_normalize_multi_word_makes(self):
|
||||
"""Test normalization of multi-word make names (underscore to space)"""
|
||||
test_cases = [
|
||||
('alfa_romeo.json', 'Alfa Romeo'),
|
||||
('land_rover.json', 'Land Rover'),
|
||||
('rolls_royce.json', 'Rolls Royce'),
|
||||
('aston_martin.json', 'Aston Martin'),
|
||||
]
|
||||
|
||||
for filename, expected in test_cases:
|
||||
with self.subTest(filename=filename):
|
||||
result = self.mapper.normalize_make_name(filename)
|
||||
self.assertEqual(result, expected,
|
||||
f"Expected '{expected}' for '{filename}', got '{result}'")
|
||||
|
||||
def test_normalize_special_cases(self):
|
||||
"""Test special capitalization cases"""
|
||||
test_cases = [
|
||||
('bmw.json', 'BMW'),
|
||||
('gmc.json', 'GMC'),
|
||||
('mini.json', 'MINI'),
|
||||
('mclaren.json', 'McLaren'),
|
||||
]
|
||||
|
||||
for filename, expected in test_cases:
|
||||
with self.subTest(filename=filename):
|
||||
result = self.mapper.normalize_make_name(filename)
|
||||
self.assertEqual(result, expected,
|
||||
f"Expected '{expected}' for '{filename}', got '{result}'")
|
||||
|
||||
def test_normalize_edge_cases(self):
|
||||
"""Test edge cases and error handling"""
|
||||
test_cases = [
|
||||
# Edge cases that should still work
|
||||
('test.json', 'Test'),
|
||||
('test_brand.json', 'Test Brand'),
|
||||
# Error cases that should return "Unknown"
|
||||
('.json', 'Unknown'),
|
||||
('', 'Unknown'),
|
||||
]
|
||||
|
||||
for filename, expected in test_cases:
|
||||
with self.subTest(filename=filename):
|
||||
result = self.mapper.normalize_make_name(filename)
|
||||
self.assertEqual(result, expected,
|
||||
f"Expected '{expected}' for '{filename}', got '{result}'")
|
||||
|
||||
def test_validate_mapping_valid_makes(self):
|
||||
"""Test validation of valid make names"""
|
||||
# These should be in the authoritative list
|
||||
valid_cases = [
|
||||
('toyota.json', 'Toyota'),
|
||||
('alfa_romeo.json', 'Alfa Romeo'),
|
||||
('bmw.json', 'BMW'),
|
||||
('land_rover.json', 'Land Rover'),
|
||||
]
|
||||
|
||||
for filename, display_name in valid_cases:
|
||||
with self.subTest(filename=filename):
|
||||
is_valid = self.mapper.validate_mapping(filename, display_name)
|
||||
self.assertTrue(is_valid, f"'{display_name}' should be valid")
|
||||
|
||||
def test_validate_mapping_invalid_makes(self):
|
||||
"""Test validation of invalid make names"""
|
||||
invalid_cases = [
|
||||
('unknown.json', 'Unknown Brand'),
|
||||
('fake.json', 'Fake'),
|
||||
('test.json', 'Test Make'),
|
||||
]
|
||||
|
||||
for filename, display_name in invalid_cases:
|
||||
with self.subTest(filename=filename):
|
||||
is_valid = self.mapper.validate_mapping(filename, display_name)
|
||||
self.assertFalse(is_valid, f"'{display_name}' should be invalid")
|
||||
|
||||
def test_reverse_lookup(self):
|
||||
"""Test reverse lookup functionality"""
|
||||
test_cases = [
|
||||
('Toyota', 'toyota.json'),
|
||||
('Alfa Romeo', 'alfa_romeo.json'),
|
||||
('BMW', 'bmw.json'),
|
||||
('Land Rover', 'land_rover.json'),
|
||||
('McLaren', 'mclaren.json'),
|
||||
]
|
||||
|
||||
for display_name, expected_filename in test_cases:
|
||||
with self.subTest(display_name=display_name):
|
||||
result = self.mapper.get_filename_for_display_name(display_name)
|
||||
self.assertEqual(result, expected_filename,
|
||||
f"Expected '{expected_filename}' for '{display_name}', got '{result}'")
|
||||
|
||||
def test_get_all_mappings(self):
|
||||
"""Test getting all mappings from a directory"""
|
||||
# Create temporary directory with test files
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Create test JSON files
|
||||
test_files = ['toyota.json', 'alfa_romeo.json', 'bmw.json']
|
||||
|
||||
for filename in test_files:
|
||||
file_path = os.path.join(temp_dir, filename)
|
||||
with open(file_path, 'w') as f:
|
||||
json.dump({"test": "data"}, f)
|
||||
|
||||
mappings = self.mapper.get_all_mappings(temp_dir)
|
||||
|
||||
# Check that all files were processed
|
||||
self.assertEqual(len(mappings), len(test_files))
|
||||
|
||||
# Check specific mappings
|
||||
expected_mappings = {
|
||||
'toyota.json': 'Toyota',
|
||||
'alfa_romeo.json': 'Alfa Romeo',
|
||||
'bmw.json': 'BMW'
|
||||
}
|
||||
|
||||
for filename, expected_display in expected_mappings.items():
|
||||
self.assertIn(filename, mappings)
|
||||
self.assertEqual(mappings[filename], expected_display)
|
||||
|
||||
def test_validation_report(self):
|
||||
"""Test validation report generation"""
|
||||
# Create temporary directory with test files
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Mix of valid and invalid files
|
||||
test_files = {
|
||||
'toyota.json': {"test": "data"}, # Valid
|
||||
'alfa_romeo.json': {"test": "data"}, # Valid
|
||||
'unknown_brand.json': {"test": "data"} # Invalid
|
||||
}
|
||||
|
||||
for filename, content in test_files.items():
|
||||
file_path = os.path.join(temp_dir, filename)
|
||||
with open(file_path, 'w') as f:
|
||||
json.dump(content, f)
|
||||
|
||||
report = self.mapper.validate_all_mappings(temp_dir)
|
||||
|
||||
# Check report structure
|
||||
self.assertIsInstance(report, ValidationReport)
|
||||
self.assertEqual(report.total_files, 3)
|
||||
self.assertEqual(report.valid_mappings, 2) # toyota and alfa_romeo should be valid
|
||||
self.assertEqual(len(report.mismatches), 1) # unknown_brand should be invalid
|
||||
|
||||
# Check success rate
|
||||
expected_rate = 2/3 # 2 valid out of 3 total
|
||||
self.assertAlmostEqual(report.success_rate, expected_rate, places=2)
|
||||
|
||||
# Check mismatch details
|
||||
mismatch = report.mismatches[0]
|
||||
self.assertEqual(mismatch['filename'], 'unknown_brand.json')
|
||||
self.assertEqual(mismatch['mapped_name'], 'Unknown Brand')
|
||||
self.assertEqual(mismatch['status'], 'NOT_FOUND_IN_AUTHORITATIVE')
|
||||
|
||||
@patch('builtins.open', mock_open(read_data='{"manufacturers": ["Toyota", "BMW", "Custom Brand"]}'))
|
||||
def test_load_custom_authoritative_makes(self):
|
||||
"""Test loading custom authoritative makes from file"""
|
||||
with patch('os.path.exists', return_value=True):
|
||||
mapper = MakeNameMapper(sources_dir='test_sources')
|
||||
|
||||
# Should have loaded the custom list
|
||||
self.assertIn('Toyota', mapper.authoritative_makes)
|
||||
self.assertIn('BMW', mapper.authoritative_makes)
|
||||
self.assertIn('Custom Brand', mapper.authoritative_makes)
|
||||
|
||||
# Test validation with custom list
|
||||
self.assertTrue(mapper.validate_mapping('custom.json', 'Custom Brand'))
|
||||
|
||||
def test_make_statistics(self):
|
||||
"""Test make statistics calculation"""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
# Create test files representing different categories
|
||||
test_files = [
|
||||
'toyota.json', # Single word
|
||||
'honda.json', # Single word
|
||||
'alfa_romeo.json', # Multi word
|
||||
'land_rover.json', # Multi word
|
||||
'bmw.json', # Special case
|
||||
'gmc.json', # Special case
|
||||
]
|
||||
|
||||
for filename in test_files:
|
||||
file_path = os.path.join(temp_dir, filename)
|
||||
with open(file_path, 'w') as f:
|
||||
json.dump({"test": "data"}, f)
|
||||
|
||||
stats = self.mapper.get_make_statistics(temp_dir)
|
||||
|
||||
expected_stats = {
|
||||
'total': 6,
|
||||
'single_words': 2, # toyota, honda
|
||||
'multi_words': 2, # alfa_romeo, land_rover
|
||||
'special_cases': 2 # bmw, gmc
|
||||
}
|
||||
|
||||
self.assertEqual(stats, expected_stats)
|
||||
|
||||
def test_error_handling(self):
|
||||
"""Test error handling for various failure scenarios"""
|
||||
# Test with non-existent directory
|
||||
mappings = self.mapper.get_all_mappings('/non/existent/directory')
|
||||
self.assertEqual(mappings, {})
|
||||
|
||||
# Test validation report with non-existent directory
|
||||
report = self.mapper.validate_all_mappings('/non/existent/directory')
|
||||
self.assertEqual(report.total_files, 0)
|
||||
self.assertEqual(report.valid_mappings, 0)
|
||||
self.assertEqual(len(report.mismatches), 0)
|
||||
|
||||
|
||||
class TestValidationReport(unittest.TestCase):
|
||||
"""Test cases for ValidationReport dataclass"""
|
||||
|
||||
def test_success_rate_calculation(self):
|
||||
"""Test success rate calculation"""
|
||||
# Test normal case
|
||||
report = ValidationReport(
|
||||
total_files=10,
|
||||
valid_mappings=8,
|
||||
mismatches=[]
|
||||
)
|
||||
self.assertEqual(report.success_rate, 0.8)
|
||||
|
||||
# Test zero division case
|
||||
report_empty = ValidationReport(
|
||||
total_files=0,
|
||||
valid_mappings=0,
|
||||
mismatches=[]
|
||||
)
|
||||
self.assertEqual(report_empty.success_rate, 0.0)
|
||||
|
||||
# Test perfect score
|
||||
report_perfect = ValidationReport(
|
||||
total_files=5,
|
||||
valid_mappings=5,
|
||||
mismatches=[]
|
||||
)
|
||||
self.assertEqual(report_perfect.success_rate, 1.0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user