40 lines
1.6 KiB
Python
Executable File
40 lines
1.6 KiB
Python
Executable File
import os
|
|
from typing import Optional
|
|
|
|
class ETLConfig:
|
|
"""ETL Configuration using environment variables"""
|
|
|
|
# MS SQL Server settings
|
|
MSSQL_HOST: str = os.getenv("MSSQL_HOST", "mvp-platform-vehicles-mssql")
|
|
MSSQL_PORT: int = int(os.getenv("MSSQL_PORT", "1433"))
|
|
MSSQL_DATABASE: str = os.getenv("MSSQL_DATABASE", "VPICList")
|
|
MSSQL_USER: str = os.getenv("MSSQL_USER", "sa")
|
|
MSSQL_PASSWORD: str = os.getenv("MSSQL_PASSWORD", "Platform123!")
|
|
|
|
# PostgreSQL settings
|
|
POSTGRES_HOST: str = os.getenv("POSTGRES_HOST", "mvp-platform-vehicles-db")
|
|
POSTGRES_PORT: int = int(os.getenv("POSTGRES_PORT", "5432"))
|
|
POSTGRES_DATABASE: str = os.getenv("POSTGRES_DATABASE", "vehicles")
|
|
POSTGRES_USER: str = os.getenv("POSTGRES_USER", "mvp_platform_user")
|
|
POSTGRES_PASSWORD: str = os.getenv("POSTGRES_PASSWORD", "platform123")
|
|
|
|
# Redis settings
|
|
REDIS_HOST: str = os.getenv("REDIS_HOST", "mvp-platform-vehicles-redis")
|
|
REDIS_PORT: int = int(os.getenv("REDIS_PORT", "6379"))
|
|
REDIS_DB: int = int(os.getenv("REDIS_DB", "0"))
|
|
|
|
# ETL Scheduling
|
|
ETL_SCHEDULE: str = os.getenv("ETL_SCHEDULE", "0 2 * * 0") # Weekly at 2 AM on Sunday
|
|
|
|
# ETL settings
|
|
BATCH_SIZE: int = int(os.getenv("BATCH_SIZE", "10000"))
|
|
PARALLEL_WORKERS: int = int(os.getenv("PARALLEL_WORKERS", "4"))
|
|
LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
|
|
|
|
# Confidence thresholds
|
|
MIN_CONFIDENCE_SCORE: int = int(os.getenv("MIN_CONFIDENCE_SCORE", "50"))
|
|
# ETL behavior toggles
|
|
DISABLE_ALL_MODELS_FALLBACK: bool = os.getenv("DISABLE_ALL_MODELS_FALLBACK", "true").lower() in ("1", "true", "yes")
|
|
|
|
config = ETLConfig()
|