fix: защита от параллельного выполнения DAG при retry

- двойная проверка: в actions (по списку issues) и в client (API запрос)
- если DAG уже running — retry отклоняется, сразу alert
- предотвращает дублирование выполнения отчётов
This commit is contained in:
Maksim Totmin 2026-04-08 18:57:51 +07:00
parent e99fe6afc3
commit 4e390b7dad
3 changed files with 51 additions and 2 deletions

View File

@ -131,13 +131,18 @@ class ActionHandler:
config.max_retries, config.retry_wait, config.max_retries, config.retry_wait,
) )
def handle_issue(self, issue: DagIssue) -> str: def handle_issue(self, issue: DagIssue, all_issues: list[DagIssue]) -> str:
"""Process a single issue. """Process a single issue.
Args:
issue: The issue to handle.
all_issues: All issues in this cycle (to check for running runs of same DAG).
Returns: Returns:
"already_handled" - previously alerted, skip "already_handled" - previously alerted, skip
"retried" - retry initiated, needs re-check "retried" - retry initiated, needs re-check
"needs_alert" - retry exhausted, alert needed "needs_alert" - retry exhausted, alert needed
"skip_running" - DAG already has a running instance, unsafe to retry
""" """
dag_id = issue.dag_id dag_id = issue.dag_id
run_id = issue.dag_run_id run_id = issue.dag_run_id
@ -161,6 +166,21 @@ class ActionHandler:
logger.debug(" → already_handled: alert was sent previously") logger.debug(" → already_handled: alert was sent previously")
return "already_handled" return "already_handled"
# SAFETY: Do not retry if this DAG already has a running instance.
# Prevents parallel execution of the same DAG (critical for reports).
if issue.issue_type == "failed":
has_running = any(
i.dag_id == dag_id and i.state == "running"
for i in all_issues
)
if has_running:
logger.warning(
" → skip_running: DAG %s has a running instance, "
"will NOT retry failed run %s to avoid parallel execution",
dag_id, run_id,
)
return "needs_alert"
# Can we retry? # Can we retry?
if retry_count < self._config.max_retries: if retry_count < self._config.max_retries:
logger.debug( logger.debug(

View File

@ -259,11 +259,40 @@ class AirflowClient:
) )
return [] return []
def has_running_run(self, dag_id: str) -> bool:
"""Check if DAG has any currently running runs.
Used as a safety check before retrying a failed run
to prevent parallel execution.
"""
try:
runs = self.get_dag_runs(dag_id, states=["running"])
if runs:
logger.warning(
"DAG %s has %d running run(s), unsafe to retry",
dag_id, len(runs),
)
return True
return False
except AirflowAPIError:
# If we can't check, assume it's running (safe side)
logger.warning("Cannot check running state for %s, assuming running", dag_id)
return True
def clear_dag_run(self, dag_id: str, dag_run_id: str) -> bool: def clear_dag_run(self, dag_id: str, dag_run_id: str) -> bool:
"""Clear (retry) a DAG run by resetting failed task instances. """Clear (retry) a DAG run by resetting failed task instances.
Safety: checks for running instances before clearing.
Returns True on success, False on failure. Returns True on success, False on failure.
""" """
# Double-check: do not retry if DAG already has a running instance
if self.has_running_run(dag_id):
logger.warning(
"SAFETY: Refusing to clear %s/%s — DAG has running instance",
dag_id, dag_run_id,
)
return False
encoded_dag_id = quote(dag_id, safe="") encoded_dag_id = quote(dag_id, safe="")
encoded_run_id = quote(dag_run_id, safe="") encoded_run_id = quote(dag_run_id, safe="")

View File

@ -140,7 +140,7 @@ class Monitor:
phase_start = time.monotonic() phase_start = time.monotonic()
needs_recheck = [] needs_recheck = []
for issue in all_issues: for issue in all_issues:
action = self._actions.handle_issue(issue) action = self._actions.handle_issue(issue, all_issues)
logger.info( logger.info(
"Phase 2/5 [Handle]: DAG %s run %s%s (type=%s, duration=%.0fs)", "Phase 2/5 [Handle]: DAG %s run %s%s (type=%s, duration=%.0fs)",
issue.dag_id, issue.dag_run_id, action, issue.dag_id, issue.dag_run_id, action,