bc437cbf8c
- автообнаружение Docker Airflow через docker inspect и .env - мониторинг failed и long-running DAG-запусков с автоматическим retry - экспорт данных в файлы для Zabbix Agent через UserParameter - офлайн-сборка ZIP-архива для закрытых контуров - Zabbix шаблоны для 5.x (XML) и 6.x+ (YAML) - systemd сервис с graceful shutdown и lock file
101 lines
2.8 KiB
Python
101 lines
2.8 KiB
Python
"""Configuration loader for Airflow DAG Monitor."""
|
|
|
|
import dataclasses
|
|
import pathlib
|
|
|
|
import yaml
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class AirflowConfig:
|
|
"""Airflow API connection settings."""
|
|
|
|
base_url: str = "http://localhost:8080"
|
|
username: str = "airflow"
|
|
password: str = "airflow"
|
|
api_version: str = "auto" # "auto", "v1", or "experimental"
|
|
timeout: int = 30
|
|
verify_ssl: bool = True
|
|
request_delay: float = 0.5 # delay between API calls (seconds)
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class MonitorConfig:
|
|
"""Monitoring behavior settings."""
|
|
|
|
cycle_interval: int = 300 # seconds between full cycles
|
|
long_running_threshold: int = 1800 # 30 minutes
|
|
retry_wait: int = 120 # wait after retry before re-check
|
|
max_retries: int = 1
|
|
state_file: str = "/var/lib/airflow-monitor/state.json"
|
|
lock_file: str = "/var/run/airflow-monitor.lock"
|
|
state_max_age: int = 86400 # 24 hours
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class ZabbixConfig:
|
|
"""Zabbix agent integration settings.
|
|
|
|
Monitor writes data to files, Zabbix agent reads via UserParameter.
|
|
"""
|
|
|
|
enabled: bool = True
|
|
data_dir: str = "/var/lib/airflow-monitor"
|
|
problems_file: str = "problems.json"
|
|
heartbeat_file: str = "heartbeat"
|
|
status_file: str = "status.json"
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class LoggingConfig:
|
|
"""Logging settings."""
|
|
|
|
level: str = "INFO"
|
|
file: str = "/var/log/airflow-monitor/monitor.log"
|
|
max_bytes: int = 10_485_760 # 10 MB
|
|
backup_count: int = 5
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class AppConfig:
|
|
"""Top-level application configuration."""
|
|
|
|
airflow: AirflowConfig
|
|
monitor: MonitorConfig
|
|
zabbix: ZabbixConfig
|
|
logging: LoggingConfig
|
|
|
|
|
|
def _build_dataclass(cls, data: dict):
|
|
"""Build a dataclass instance from a dict, ignoring unknown keys."""
|
|
if data is None:
|
|
return cls()
|
|
fields = {f.name for f in dataclasses.fields(cls)}
|
|
filtered = {k: v for k, v in data.items() if k in fields}
|
|
return cls(**filtered)
|
|
|
|
|
|
def load_config(path: str) -> AppConfig:
|
|
"""Load configuration from a YAML file.
|
|
|
|
Missing sections fall back to defaults.
|
|
Raises FileNotFoundError if the file does not exist.
|
|
Raises ValueError on invalid YAML.
|
|
"""
|
|
config_path = pathlib.Path(path)
|
|
if not config_path.exists():
|
|
raise FileNotFoundError(f"Config file not found: {path}")
|
|
|
|
with open(config_path, "r", encoding="utf-8") as f:
|
|
raw = yaml.safe_load(f)
|
|
|
|
if not isinstance(raw, dict):
|
|
raise ValueError(f"Config file must be a YAML mapping, got {type(raw).__name__}")
|
|
|
|
return AppConfig(
|
|
airflow=_build_dataclass(AirflowConfig, raw.get("airflow")),
|
|
monitor=_build_dataclass(MonitorConfig, raw.get("monitor")),
|
|
zabbix=_build_dataclass(ZabbixConfig, raw.get("zabbix")),
|
|
logging=_build_dataclass(LoggingConfig, raw.get("logging")),
|
|
)
|