Initial Commit

mvp-platform-services/vehicles/etl/config.py

@@ -0,0 +1,39 @@
+import os
+from typing import Optional
+
+class ETLConfig:
+    """ETL Configuration using environment variables"""
+    
+    # MS SQL Server settings
+    MSSQL_HOST: str = os.getenv("MSSQL_HOST", "mvp-platform-vehicles-mssql")
+    MSSQL_PORT: int = int(os.getenv("MSSQL_PORT", "1433"))
+    MSSQL_DATABASE: str = os.getenv("MSSQL_DATABASE", "VPICList")
+    MSSQL_USER: str = os.getenv("MSSQL_USER", "sa")
+    MSSQL_PASSWORD: str = os.getenv("MSSQL_PASSWORD", "Platform123!")
+    
+    # PostgreSQL settings  
+    POSTGRES_HOST: str = os.getenv("POSTGRES_HOST", "mvp-platform-vehicles-db")
+    POSTGRES_PORT: int = int(os.getenv("POSTGRES_PORT", "5432"))
+    POSTGRES_DATABASE: str = os.getenv("POSTGRES_DATABASE", "vehicles")
+    POSTGRES_USER: str = os.getenv("POSTGRES_USER", "mvp_platform_user")
+    POSTGRES_PASSWORD: str = os.getenv("POSTGRES_PASSWORD", "platform123")
+    
+    # Redis settings
+    REDIS_HOST: str = os.getenv("REDIS_HOST", "mvp-platform-vehicles-redis")
+    REDIS_PORT: int = int(os.getenv("REDIS_PORT", "6379"))
+    REDIS_DB: int = int(os.getenv("REDIS_DB", "0"))
+    
+    # ETL Scheduling
+    ETL_SCHEDULE: str = os.getenv("ETL_SCHEDULE", "0 2 * * 0")  # Weekly at 2 AM on Sunday
+    
+    # ETL settings
+    BATCH_SIZE: int = int(os.getenv("BATCH_SIZE", "10000"))
+    PARALLEL_WORKERS: int = int(os.getenv("PARALLEL_WORKERS", "4"))
+    LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
+    
+    # Confidence thresholds
+    MIN_CONFIDENCE_SCORE: int = int(os.getenv("MIN_CONFIDENCE_SCORE", "50"))
+    # ETL behavior toggles
+    DISABLE_ALL_MODELS_FALLBACK: bool = os.getenv("DISABLE_ALL_MODELS_FALLBACK", "true").lower() in ("1", "true", "yes")
+
+config = ETLConfig()