fix: защита от параллельного выполнения DAG при retry
- двойная проверка: в actions (по списку issues) и в client (API запрос) - если DAG уже running — retry отклоняется, сразу alert - предотвращает дублирование выполнения отчётов
This commit is contained in:
parent
e99fe6afc3
commit
4e390b7dad
@ -131,13 +131,18 @@ class ActionHandler:
|
||||
config.max_retries, config.retry_wait,
|
||||
)
|
||||
|
||||
def handle_issue(self, issue: DagIssue) -> str:
|
||||
def handle_issue(self, issue: DagIssue, all_issues: list[DagIssue]) -> str:
|
||||
"""Process a single issue.
|
||||
|
||||
Args:
|
||||
issue: The issue to handle.
|
||||
all_issues: All issues in this cycle (to check for running runs of same DAG).
|
||||
|
||||
Returns:
|
||||
"already_handled" - previously alerted, skip
|
||||
"retried" - retry initiated, needs re-check
|
||||
"needs_alert" - retry exhausted, alert needed
|
||||
"skip_running" - DAG already has a running instance, unsafe to retry
|
||||
"""
|
||||
dag_id = issue.dag_id
|
||||
run_id = issue.dag_run_id
|
||||
@ -161,6 +166,21 @@ class ActionHandler:
|
||||
logger.debug(" → already_handled: alert was sent previously")
|
||||
return "already_handled"
|
||||
|
||||
# SAFETY: Do not retry if this DAG already has a running instance.
|
||||
# Prevents parallel execution of the same DAG (critical for reports).
|
||||
if issue.issue_type == "failed":
|
||||
has_running = any(
|
||||
i.dag_id == dag_id and i.state == "running"
|
||||
for i in all_issues
|
||||
)
|
||||
if has_running:
|
||||
logger.warning(
|
||||
" → skip_running: DAG %s has a running instance, "
|
||||
"will NOT retry failed run %s to avoid parallel execution",
|
||||
dag_id, run_id,
|
||||
)
|
||||
return "needs_alert"
|
||||
|
||||
# Can we retry?
|
||||
if retry_count < self._config.max_retries:
|
||||
logger.debug(
|
||||
|
||||
@ -259,11 +259,40 @@ class AirflowClient:
|
||||
)
|
||||
return []
|
||||
|
||||
def has_running_run(self, dag_id: str) -> bool:
|
||||
"""Check if DAG has any currently running runs.
|
||||
|
||||
Used as a safety check before retrying a failed run
|
||||
to prevent parallel execution.
|
||||
"""
|
||||
try:
|
||||
runs = self.get_dag_runs(dag_id, states=["running"])
|
||||
if runs:
|
||||
logger.warning(
|
||||
"DAG %s has %d running run(s), unsafe to retry",
|
||||
dag_id, len(runs),
|
||||
)
|
||||
return True
|
||||
return False
|
||||
except AirflowAPIError:
|
||||
# If we can't check, assume it's running (safe side)
|
||||
logger.warning("Cannot check running state for %s, assuming running", dag_id)
|
||||
return True
|
||||
|
||||
def clear_dag_run(self, dag_id: str, dag_run_id: str) -> bool:
|
||||
"""Clear (retry) a DAG run by resetting failed task instances.
|
||||
|
||||
Safety: checks for running instances before clearing.
|
||||
Returns True on success, False on failure.
|
||||
"""
|
||||
# Double-check: do not retry if DAG already has a running instance
|
||||
if self.has_running_run(dag_id):
|
||||
logger.warning(
|
||||
"SAFETY: Refusing to clear %s/%s — DAG has running instance",
|
||||
dag_id, dag_run_id,
|
||||
)
|
||||
return False
|
||||
|
||||
encoded_dag_id = quote(dag_id, safe="")
|
||||
encoded_run_id = quote(dag_run_id, safe="")
|
||||
|
||||
|
||||
@ -140,7 +140,7 @@ class Monitor:
|
||||
phase_start = time.monotonic()
|
||||
needs_recheck = []
|
||||
for issue in all_issues:
|
||||
action = self._actions.handle_issue(issue)
|
||||
action = self._actions.handle_issue(issue, all_issues)
|
||||
logger.info(
|
||||
"Phase 2/5 [Handle]: DAG %s run %s → %s (type=%s, duration=%.0fs)",
|
||||
issue.dag_id, issue.dag_run_id, action,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user