fix: добавил max_failed_age — игнорировать старые failed runs (>24ч)

- failed runs старше 24 часов пропускаются как историческая ошибка
- предотвращает массовый перезапуск DAG-ов от 2024 года
- параметр max_failed_age настраивается в config.yaml
This commit is contained in:
Maksim Totmin 2026-04-08 18:54:33 +07:00
parent cc4dfc0692
commit e99fe6afc3
4 changed files with 21 additions and 3 deletions

View File

@ -27,11 +27,13 @@ class DagIssue:
class DagAnalyzer: class DagAnalyzer:
"""Analyzes DAG runs and produces a list of issues.""" """Analyzes DAG runs and produces a list of issues."""
def __init__(self, long_running_threshold: int): def __init__(self, long_running_threshold: int, max_failed_age: int = 86400):
self._threshold = long_running_threshold self._threshold = long_running_threshold
self._max_failed_age = max_failed_age # ignore failed runs older than this (seconds)
logger.debug( logger.debug(
"DagAnalyzer initialized: long_running_threshold=%ds (%.1f min)", "DagAnalyzer initialized: long_running_threshold=%ds (%.1f min), max_failed_age=%ds (%.1f hours)",
long_running_threshold, long_running_threshold / 60, long_running_threshold, long_running_threshold / 60,
max_failed_age, max_failed_age / 3600,
) )
def analyze_dag_runs( def analyze_dag_runs(
@ -81,6 +83,14 @@ class DagAnalyzer:
) )
if state == "failed": if state == "failed":
# Skip old failed runs — only alert on recent failures
if duration > self._max_failed_age:
logger.debug(
" → SKIP: failed run too old (%.0fs > %ds max_failed_age)",
duration, self._max_failed_age,
)
continue
error_info = self._extract_error(dag_id, run_id, task_instances_fn) error_info = self._extract_error(dag_id, run_id, task_instances_fn)
issue = DagIssue( issue = DagIssue(
dag_id=dag_id, dag_id=dag_id,

View File

@ -25,6 +25,7 @@ class MonitorConfig:
cycle_interval: int = 300 # seconds between full cycles cycle_interval: int = 300 # seconds between full cycles
long_running_threshold: int = 1800 # 30 minutes long_running_threshold: int = 1800 # 30 minutes
max_failed_age: int = 86400 # ignore failed runs older than 24 hours
retry_wait: int = 120 # wait after retry before re-check retry_wait: int = 120 # wait after retry before re-check
max_retries: int = 1 max_retries: int = 1
state_file: str = "/var/lib/airflow-monitor/state.json" state_file: str = "/var/lib/airflow-monitor/state.json"

View File

@ -32,7 +32,10 @@ class Monitor:
self._shutdown = shutdown_event self._shutdown = shutdown_event
self._client = AirflowClient(config.airflow) self._client = AirflowClient(config.airflow)
self._state = StateManager(config.monitor.state_file, config.monitor.state_max_age) self._state = StateManager(config.monitor.state_file, config.monitor.state_max_age)
self._analyzer = DagAnalyzer(config.monitor.long_running_threshold) self._analyzer = DagAnalyzer(
config.monitor.long_running_threshold,
config.monitor.max_failed_age,
)
self._exporter = DataExporter(config.zabbix) self._exporter = DataExporter(config.zabbix)
self._actions = ActionHandler( self._actions = ActionHandler(
self._client, self._state, self._exporter, config.monitor, self._client, self._state, self._exporter, config.monitor,

View File

@ -36,6 +36,10 @@ monitor:
# DAG run duration threshold to consider as "long running" (seconds) # DAG run duration threshold to consider as "long running" (seconds)
long_running_threshold: 1800 # 30 minutes long_running_threshold: 1800 # 30 minutes
# Игнорировать failed runs старше этого порога (seconds)
# Старые ошибки — это история, не текущая проблема
max_failed_age: 86400 # 24 hours
# Time to wait after retry before re-checking (seconds) # Time to wait after retry before re-checking (seconds)
retry_wait: 120 # 2 minutes retry_wait: 120 # 2 minutes