fix: защита от параллельного выполнения DAG при retry
- двойная проверка: в actions (по списку issues) и в client (API запрос) - если DAG уже running — retry отклоняется, сразу alert - предотвращает дублирование выполнения отчётов
This commit is contained in:
parent
e99fe6afc3
commit
4e390b7dad
@ -131,13 +131,18 @@ class ActionHandler:
|
|||||||
config.max_retries, config.retry_wait,
|
config.max_retries, config.retry_wait,
|
||||||
)
|
)
|
||||||
|
|
||||||
def handle_issue(self, issue: DagIssue) -> str:
|
def handle_issue(self, issue: DagIssue, all_issues: list[DagIssue]) -> str:
|
||||||
"""Process a single issue.
|
"""Process a single issue.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
issue: The issue to handle.
|
||||||
|
all_issues: All issues in this cycle (to check for running runs of same DAG).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
"already_handled" - previously alerted, skip
|
"already_handled" - previously alerted, skip
|
||||||
"retried" - retry initiated, needs re-check
|
"retried" - retry initiated, needs re-check
|
||||||
"needs_alert" - retry exhausted, alert needed
|
"needs_alert" - retry exhausted, alert needed
|
||||||
|
"skip_running" - DAG already has a running instance, unsafe to retry
|
||||||
"""
|
"""
|
||||||
dag_id = issue.dag_id
|
dag_id = issue.dag_id
|
||||||
run_id = issue.dag_run_id
|
run_id = issue.dag_run_id
|
||||||
@ -161,6 +166,21 @@ class ActionHandler:
|
|||||||
logger.debug(" → already_handled: alert was sent previously")
|
logger.debug(" → already_handled: alert was sent previously")
|
||||||
return "already_handled"
|
return "already_handled"
|
||||||
|
|
||||||
|
# SAFETY: Do not retry if this DAG already has a running instance.
|
||||||
|
# Prevents parallel execution of the same DAG (critical for reports).
|
||||||
|
if issue.issue_type == "failed":
|
||||||
|
has_running = any(
|
||||||
|
i.dag_id == dag_id and i.state == "running"
|
||||||
|
for i in all_issues
|
||||||
|
)
|
||||||
|
if has_running:
|
||||||
|
logger.warning(
|
||||||
|
" → skip_running: DAG %s has a running instance, "
|
||||||
|
"will NOT retry failed run %s to avoid parallel execution",
|
||||||
|
dag_id, run_id,
|
||||||
|
)
|
||||||
|
return "needs_alert"
|
||||||
|
|
||||||
# Can we retry?
|
# Can we retry?
|
||||||
if retry_count < self._config.max_retries:
|
if retry_count < self._config.max_retries:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|||||||
@ -259,11 +259,40 @@ class AirflowClient:
|
|||||||
)
|
)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
def has_running_run(self, dag_id: str) -> bool:
|
||||||
|
"""Check if DAG has any currently running runs.
|
||||||
|
|
||||||
|
Used as a safety check before retrying a failed run
|
||||||
|
to prevent parallel execution.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
runs = self.get_dag_runs(dag_id, states=["running"])
|
||||||
|
if runs:
|
||||||
|
logger.warning(
|
||||||
|
"DAG %s has %d running run(s), unsafe to retry",
|
||||||
|
dag_id, len(runs),
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
except AirflowAPIError:
|
||||||
|
# If we can't check, assume it's running (safe side)
|
||||||
|
logger.warning("Cannot check running state for %s, assuming running", dag_id)
|
||||||
|
return True
|
||||||
|
|
||||||
def clear_dag_run(self, dag_id: str, dag_run_id: str) -> bool:
|
def clear_dag_run(self, dag_id: str, dag_run_id: str) -> bool:
|
||||||
"""Clear (retry) a DAG run by resetting failed task instances.
|
"""Clear (retry) a DAG run by resetting failed task instances.
|
||||||
|
|
||||||
|
Safety: checks for running instances before clearing.
|
||||||
Returns True on success, False on failure.
|
Returns True on success, False on failure.
|
||||||
"""
|
"""
|
||||||
|
# Double-check: do not retry if DAG already has a running instance
|
||||||
|
if self.has_running_run(dag_id):
|
||||||
|
logger.warning(
|
||||||
|
"SAFETY: Refusing to clear %s/%s — DAG has running instance",
|
||||||
|
dag_id, dag_run_id,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
encoded_dag_id = quote(dag_id, safe="")
|
encoded_dag_id = quote(dag_id, safe="")
|
||||||
encoded_run_id = quote(dag_run_id, safe="")
|
encoded_run_id = quote(dag_run_id, safe="")
|
||||||
|
|
||||||
|
|||||||
@ -140,7 +140,7 @@ class Monitor:
|
|||||||
phase_start = time.monotonic()
|
phase_start = time.monotonic()
|
||||||
needs_recheck = []
|
needs_recheck = []
|
||||||
for issue in all_issues:
|
for issue in all_issues:
|
||||||
action = self._actions.handle_issue(issue)
|
action = self._actions.handle_issue(issue, all_issues)
|
||||||
logger.info(
|
logger.info(
|
||||||
"Phase 2/5 [Handle]: DAG %s run %s → %s (type=%s, duration=%.0fs)",
|
"Phase 2/5 [Handle]: DAG %s run %s → %s (type=%s, duration=%.0fs)",
|
||||||
issue.dag_id, issue.dag_run_id, action,
|
issue.dag_id, issue.dag_run_id, action,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user