fix: экспорт всех текущих проблем в Zabbix + совместимость шаблона с Zabbix 5.0
collect_and_alert теперь экспортирует ВСЕ текущие проблемные DAG-и, а не только новые — Zabbix видит проблему пока DAG не починен. Zabbix XML шаблон: version 5.0, убраны tags, preprocessing params, triggers/graphs вынесены на верхний уровень, history 7d / trends 365d. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Maksim Totmin
parent
289c308fcb
commit
f3e1ad7df3
@ -211,54 +211,54 @@ class ActionHandler:
|
||||
return "needs_alert"
|
||||
|
||||
def collect_and_alert(self, issues: list[DagIssue]):
|
||||
"""Export alerts for issues that need alerting.
|
||||
"""Export all current issues and mark new ones as alerted.
|
||||
|
||||
Filters to only unalerted issues, builds JSON payload,
|
||||
writes to file for Zabbix agent, and marks as alerted in state.
|
||||
Always exports the full list of current problems so Zabbix sees them
|
||||
until they are resolved. Only logs warnings for newly discovered issues.
|
||||
"""
|
||||
to_alert = []
|
||||
# Build export payload for ALL current issues
|
||||
all_problems = []
|
||||
new_issues = []
|
||||
for issue in issues:
|
||||
entry = self._state.ensure_entry(issue.dag_id, issue.dag_run_id)
|
||||
alert_data = {
|
||||
"dag_id": issue.dag_id,
|
||||
"dag_run_id": issue.dag_run_id,
|
||||
"issue_type": issue.issue_type,
|
||||
"status": issue.state,
|
||||
"duration_seconds": issue.duration_seconds,
|
||||
"error_info": issue.error_info,
|
||||
"retry_count": entry["retry_count"],
|
||||
}
|
||||
all_problems.append(alert_data)
|
||||
if not self._state.is_alerted(issue.dag_id, issue.dag_run_id):
|
||||
entry = self._state.ensure_entry(issue.dag_id, issue.dag_run_id)
|
||||
alert_data = {
|
||||
"dag_id": issue.dag_id,
|
||||
"dag_run_id": issue.dag_run_id,
|
||||
"issue_type": issue.issue_type,
|
||||
"status": issue.state,
|
||||
"duration_seconds": issue.duration_seconds,
|
||||
"error_info": issue.error_info,
|
||||
"retry_count": entry["retry_count"],
|
||||
}
|
||||
to_alert.append(alert_data)
|
||||
logger.debug(
|
||||
"Issue queued for alert: %s/%s type=%s retries=%d",
|
||||
issue.dag_id, issue.dag_run_id,
|
||||
issue.issue_type, entry["retry_count"],
|
||||
)
|
||||
new_issues.append(issue)
|
||||
|
||||
if not to_alert:
|
||||
# Write empty list to clear Zabbix trigger
|
||||
if not all_problems:
|
||||
self._exporter.export_problems([])
|
||||
logger.info("No issues to alert, exported empty problem list")
|
||||
logger.info("No issues, exported empty problem list")
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"Exporting %d problematic DAG runs for Zabbix: %s",
|
||||
len(to_alert),
|
||||
", ".join(
|
||||
f"{p['dag_id']}/{p['dag_run_id']}({p['issue_type']})"
|
||||
for p in to_alert
|
||||
),
|
||||
# Log new issues at warning level
|
||||
if new_issues:
|
||||
logger.warning(
|
||||
"New problematic DAG runs: %s",
|
||||
", ".join(
|
||||
f"{i.dag_id}/{i.dag_run_id}({i.issue_type})"
|
||||
for i in new_issues
|
||||
),
|
||||
)
|
||||
|
||||
logger.debug("Alert payload:\n%s", json.dumps(all_problems, indent=2, ensure_ascii=False))
|
||||
|
||||
self._exporter.export_problems(all_problems)
|
||||
|
||||
# Mark new issues as alerted
|
||||
for issue in new_issues:
|
||||
self._state.mark_alerted(issue.dag_id, issue.dag_run_id)
|
||||
logger.debug("Marked as alerted: %s/%s", issue.dag_id, issue.dag_run_id)
|
||||
|
||||
logger.info(
|
||||
"Exported %d issues to problems file (%d new)",
|
||||
len(all_problems), len(new_issues),
|
||||
)
|
||||
|
||||
# Log full alert payload at debug level
|
||||
logger.debug("Alert payload:\n%s", json.dumps(to_alert, indent=2, ensure_ascii=False))
|
||||
|
||||
self._exporter.export_problems(to_alert)
|
||||
|
||||
for issue in issues:
|
||||
if not self._state.is_alerted(issue.dag_id, issue.dag_run_id):
|
||||
self._state.mark_alerted(issue.dag_id, issue.dag_run_id)
|
||||
logger.debug("Marked as alerted: %s/%s", issue.dag_id, issue.dag_run_id)
|
||||
|
||||
logger.info("Exported %d issues to problems file for Zabbix agent", len(to_alert))
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<zabbix_export>
|
||||
<version>5.4</version>
|
||||
<version>5.0</version>
|
||||
<groups>
|
||||
<group>
|
||||
<name>Templates/Applications</name>
|
||||
@ -17,7 +17,7 @@
|
||||
- Установлен airflow-monitor (/opt/airflow-monitor)
|
||||
- Установлен конфиг /etc/zabbix/zabbix_agentd.conf.d/airflow-monitor.conf
|
||||
|
||||
Версия шаблона: 1.0.0 (Zabbix 5.x)</description>
|
||||
Версия шаблона: 1.0.1 (Zabbix 5.0)</description>
|
||||
<groups>
|
||||
<group>
|
||||
<name>Templates/Applications</name>
|
||||
@ -32,12 +32,6 @@
|
||||
<history>7d</history>
|
||||
<value_type>TEXT</value_type>
|
||||
<description>JSON-массив проблемных DAG-запусков. [] = нет проблем.</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>airflow</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- Problematic DAGs count -->
|
||||
@ -45,16 +39,10 @@
|
||||
<name>Airflow: Problematic DAGs count</name>
|
||||
<key>airflow.dag.problems.count</key>
|
||||
<delay>1m</delay>
|
||||
<history>90d</history>
|
||||
<history>7d</history>
|
||||
<trends>365d</trends>
|
||||
<value_type>UNSIGNED</value_type>
|
||||
<description>Количество DAG-запусков с проблемами.</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>airflow</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- Heartbeat -->
|
||||
@ -63,14 +51,9 @@
|
||||
<key>airflow.monitor.heartbeat</key>
|
||||
<delay>1m</delay>
|
||||
<history>7d</history>
|
||||
<trends>365d</trends>
|
||||
<value_type>UNSIGNED</value_type>
|
||||
<description>Epoch timestamp последнего успешного цикла мониторинга.</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>dag-monitor</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- Status JSON -->
|
||||
@ -81,12 +64,6 @@
|
||||
<history>7d</history>
|
||||
<value_type>TEXT</value_type>
|
||||
<description>JSON со статусом монитора.</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>dag-monitor</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- Service alive -->
|
||||
@ -94,19 +71,13 @@
|
||||
<name>Airflow Monitor: Service alive</name>
|
||||
<key>airflow.monitor.alive</key>
|
||||
<delay>1m</delay>
|
||||
<history>90d</history>
|
||||
<history>7d</history>
|
||||
<trends>365d</trends>
|
||||
<value_type>UNSIGNED</value_type>
|
||||
<description>1 = сервис запущен, 0 = остановлен.</description>
|
||||
<valuemap>
|
||||
<name>Service state</name>
|
||||
</valuemap>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>dag-monitor</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- DAG count (dependent from status) -->
|
||||
@ -114,7 +85,7 @@
|
||||
<name>Airflow Monitor: DAG count</name>
|
||||
<type>DEPENDENT</type>
|
||||
<key>airflow.monitor.dag_count</key>
|
||||
<history>90d</history>
|
||||
<history>7d</history>
|
||||
<trends>365d</trends>
|
||||
<value_type>UNSIGNED</value_type>
|
||||
<description>Количество отслеживаемых DAG-ов.</description>
|
||||
@ -124,23 +95,13 @@
|
||||
<preprocessing>
|
||||
<step>
|
||||
<type>JSONPATH</type>
|
||||
<parameters>
|
||||
<parameter>$.dag_count</parameter>
|
||||
</parameters>
|
||||
<params>$.dag_count</params>
|
||||
</step>
|
||||
<step>
|
||||
<type>DISCARD_UNCHANGED_HEARTBEAT</type>
|
||||
<parameters>
|
||||
<parameter>1h</parameter>
|
||||
</parameters>
|
||||
<params>1h</params>
|
||||
</step>
|
||||
</preprocessing>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>airflow</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- Cycle time (dependent from status) -->
|
||||
@ -148,7 +109,7 @@
|
||||
<name>Airflow Monitor: Cycle time (seconds)</name>
|
||||
<type>DEPENDENT</type>
|
||||
<key>airflow.monitor.cycle_time</key>
|
||||
<history>90d</history>
|
||||
<history>7d</history>
|
||||
<trends>365d</trends>
|
||||
<value_type>FLOAT</value_type>
|
||||
<units>s</units>
|
||||
@ -159,17 +120,9 @@
|
||||
<preprocessing>
|
||||
<step>
|
||||
<type>JSONPATH</type>
|
||||
<parameters>
|
||||
<parameter>$.cycle_time_seconds</parameter>
|
||||
</parameters>
|
||||
<params>$.cycle_time_seconds</params>
|
||||
</step>
|
||||
</preprocessing>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>dag-monitor</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
|
||||
<!-- Cycle count (dependent from status) -->
|
||||
@ -177,7 +130,7 @@
|
||||
<name>Airflow Monitor: Cycle count</name>
|
||||
<type>DEPENDENT</type>
|
||||
<key>airflow.monitor.cycle_count</key>
|
||||
<history>90d</history>
|
||||
<history>7d</history>
|
||||
<trends>365d</trends>
|
||||
<value_type>UNSIGNED</value_type>
|
||||
<description>Номер текущего цикла мониторинга.</description>
|
||||
@ -187,133 +140,16 @@
|
||||
<preprocessing>
|
||||
<step>
|
||||
<type>JSONPATH</type>
|
||||
<parameters>
|
||||
<parameter>$.cycle_count</parameter>
|
||||
</parameters>
|
||||
<params>$.cycle_count</params>
|
||||
</step>
|
||||
<step>
|
||||
<type>DISCARD_UNCHANGED_HEARTBEAT</type>
|
||||
<parameters>
|
||||
<parameter>1h</parameter>
|
||||
</parameters>
|
||||
<params>1h</params>
|
||||
</step>
|
||||
</preprocessing>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>component</tag>
|
||||
<value>dag-monitor</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</item>
|
||||
</items>
|
||||
|
||||
<triggers>
|
||||
<!-- Problematic DAGs detected -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.dag.problems.count.last()}>0</expression>
|
||||
<recovery_expression>{Airflow DAG Monitor:airflow.dag.problems.count.last()}=0</recovery_expression>
|
||||
<name>Airflow: {ITEM.LASTVALUE1} problematic DAG run(s) detected</name>
|
||||
<priority>HIGH</priority>
|
||||
<description>Обнаружены проблемные DAG-запуски (failed или long_running).
|
||||
Подробности в item "Airflow: Problematic DAGs (JSON)".
|
||||
Триггер снимается автоматически при разрешении всех проблем.</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>scope</tag>
|
||||
<value>availability</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</trigger>
|
||||
|
||||
<!-- Heartbeat stale -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.heartbeat.nodata(600)}=1 or (({Airflow DAG Monitor:airflow.monitor.heartbeat.last()}>0) and ({Airflow DAG Monitor:airflow.monitor.heartbeat.fuzzytime(600)}=0))</expression>
|
||||
<name>Airflow Monitor: Data is stale (no update for 10+ minutes)</name>
|
||||
<priority>DISASTER</priority>
|
||||
<description>Данные мониторинга не обновлялись более 10 минут.
|
||||
Возможные причины: сервис остановлен, Airflow API недоступен.</description>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<name>Airflow Monitor: Service is not running</name>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.alive.last()}=0</expression>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>scope</tag>
|
||||
<value>availability</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</trigger>
|
||||
|
||||
<!-- Service down -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.alive.last()}=0</expression>
|
||||
<name>Airflow Monitor: Service is not running</name>
|
||||
<priority>HIGH</priority>
|
||||
<description>Сервис airflow-monitor не запущен.
|
||||
Запустить: sudo systemctl start airflow-monitor</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>scope</tag>
|
||||
<value>availability</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</trigger>
|
||||
|
||||
<!-- Slow cycle -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.cycle_time.last()}>60</expression>
|
||||
<name>Airflow Monitor: Cycle time is high ({ITEM.LASTVALUE1}s)</name>
|
||||
<priority>WARNING</priority>
|
||||
<description>Цикл мониторинга выполняется дольше 60 секунд.</description>
|
||||
<tags>
|
||||
<tag>
|
||||
<tag>scope</tag>
|
||||
<value>performance</value>
|
||||
</tag>
|
||||
</tags>
|
||||
</trigger>
|
||||
</triggers>
|
||||
|
||||
<graphs>
|
||||
<!-- Problem count graph -->
|
||||
<graph>
|
||||
<name>Airflow: Problem DAG count</name>
|
||||
<graph_items>
|
||||
<graph_item>
|
||||
<color>FF0000</color>
|
||||
<item>
|
||||
<host>Airflow DAG Monitor</host>
|
||||
<key>airflow.dag.problems.count</key>
|
||||
</item>
|
||||
</graph_item>
|
||||
<graph_item>
|
||||
<color>00AA00</color>
|
||||
<yaxisside>RIGHT</yaxisside>
|
||||
<item>
|
||||
<host>Airflow DAG Monitor</host>
|
||||
<key>airflow.monitor.dag_count</key>
|
||||
</item>
|
||||
</graph_item>
|
||||
</graph_items>
|
||||
</graph>
|
||||
|
||||
<!-- Cycle time graph -->
|
||||
<graph>
|
||||
<name>Airflow Monitor: Cycle time</name>
|
||||
<graph_items>
|
||||
<graph_item>
|
||||
<color>0000FF</color>
|
||||
<item>
|
||||
<host>Airflow DAG Monitor</host>
|
||||
<key>airflow.monitor.cycle_time</key>
|
||||
</item>
|
||||
</graph_item>
|
||||
</graph_items>
|
||||
</graph>
|
||||
</graphs>
|
||||
|
||||
<macros>
|
||||
<macro>
|
||||
<macro>{$AIRFLOW.STALE_THRESHOLD}</macro>
|
||||
@ -329,6 +165,89 @@
|
||||
</template>
|
||||
</templates>
|
||||
|
||||
<triggers>
|
||||
<!-- Problematic DAGs detected -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.dag.problems.count.last()}>0</expression>
|
||||
<recovery_expression></recovery_expression>
|
||||
<name>Airflow: {ITEM.LASTVALUE1} problematic DAG run(s) detected</name>
|
||||
<priority>HIGH</priority>
|
||||
<description>Обнаружены проблемные DAG-запуски (failed или long_running).
|
||||
Подробности в item "Airflow: Problematic DAGs (JSON)".
|
||||
Триггер снимается автоматически при разрешении всех проблем.</description>
|
||||
</trigger>
|
||||
|
||||
<!-- Heartbeat stale -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.heartbeat.nodata(600)}=1 or (({Airflow DAG Monitor:airflow.monitor.heartbeat.last()}>0) and ({Airflow DAG Monitor:airflow.monitor.heartbeat.fuzzytime(600)}=0))</expression>
|
||||
<name>Airflow Monitor: Data is stale (no update for 10+ minutes)</name>
|
||||
<priority>DISASTER</priority>
|
||||
<description>Данные мониторинга не обновлялись более 10 минут.
|
||||
Возможные причины: сервис остановлен, Airflow API недоступен.</description>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<name>Airflow Monitor: Service is not running</name>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.alive.last()}=0</expression>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</trigger>
|
||||
|
||||
<!-- Service down -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.alive.last()}=0</expression>
|
||||
<name>Airflow Monitor: Service is not running</name>
|
||||
<priority>HIGH</priority>
|
||||
<description>Сервис airflow-monitor не запущен.
|
||||
Запустить: sudo systemctl start airflow-monitor</description>
|
||||
</trigger>
|
||||
|
||||
<!-- Slow cycle -->
|
||||
<trigger>
|
||||
<expression>{Airflow DAG Monitor:airflow.monitor.cycle_time.last()}>60</expression>
|
||||
<name>Airflow Monitor: Cycle time is high ({ITEM.LASTVALUE1}s)</name>
|
||||
<priority>WARNING</priority>
|
||||
<description>Цикл мониторинга выполняется дольше 60 секунд.</description>
|
||||
</trigger>
|
||||
</triggers>
|
||||
|
||||
<graphs>
|
||||
<!-- Problem count graph -->
|
||||
<graph>
|
||||
<name>Airflow: Problem DAG count</name>
|
||||
<graph_items>
|
||||
<graph_item>
|
||||
<color>FF0000</color>
|
||||
<item>
|
||||
<host>Airflow DAG Monitor</host>
|
||||
<key>airflow.dag.problems.count</key>
|
||||
</item>
|
||||
</graph_item>
|
||||
<graph_item>
|
||||
<color>00AA00</color>
|
||||
<yaxisside>RIGHT</yaxisside>
|
||||
<item>
|
||||
<host>Airflow DAG Monitor</host>
|
||||
<key>airflow.monitor.dag_count</key>
|
||||
</item>
|
||||
</graph_item>
|
||||
</graph_items>
|
||||
</graph>
|
||||
|
||||
<!-- Cycle time graph -->
|
||||
<graph>
|
||||
<name>Airflow Monitor: Cycle time</name>
|
||||
<graph_items>
|
||||
<graph_item>
|
||||
<color>0000FF</color>
|
||||
<item>
|
||||
<host>Airflow DAG Monitor</host>
|
||||
<key>airflow.monitor.cycle_time</key>
|
||||
</item>
|
||||
</graph_item>
|
||||
</graph_items>
|
||||
</graph>
|
||||
</graphs>
|
||||
|
||||
<value_maps>
|
||||
<value_map>
|
||||
<name>Service state</name>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user