bc437cbf8c
- автообнаружение Docker Airflow через docker inspect и .env - мониторинг failed и long-running DAG-запусков с автоматическим retry - экспорт данных в файлы для Zabbix Agent через UserParameter - офлайн-сборка ZIP-архива для закрытых контуров - Zabbix шаблоны для 5.x (XML) и 6.x+ (YAML) - systemd сервис с graceful shutdown и lock file
127 lines
4.7 KiB
Bash
Executable File
127 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# Airflow DAG Monitor - Setup Script
|
|
#
|
|
# Creates virtual environment, installs dependencies, prepares directories,
|
|
# and installs systemd service.
|
|
#
|
|
# По умолчанию устанавливает в /opt/airflow-monitor.
|
|
# Если запущен из другого каталога — копирует файлы в /opt/airflow-monitor.
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
INSTALL_DIR="/opt/airflow-monitor"
|
|
|
|
# Copy project files to /opt/airflow-monitor if running from another location
|
|
if [ "$SCRIPT_DIR" != "$INSTALL_DIR" ]; then
|
|
echo "Copying project to $INSTALL_DIR..."
|
|
sudo mkdir -p "$INSTALL_DIR"
|
|
sudo chown "$(whoami):$(id -gn)" "$INSTALL_DIR"
|
|
cp -r "$SCRIPT_DIR/airflow_monitor" "$INSTALL_DIR/"
|
|
cp "$SCRIPT_DIR/config.yaml" "$INSTALL_DIR/"
|
|
cp "$SCRIPT_DIR/requirements.txt" "$INSTALL_DIR/"
|
|
cp "$SCRIPT_DIR/airflow-monitor.service" "$INSTALL_DIR/"
|
|
cp "$SCRIPT_DIR/README.md" "$INSTALL_DIR/"
|
|
[ -f "$SCRIPT_DIR/setup.sh" ] && cp "$SCRIPT_DIR/setup.sh" "$INSTALL_DIR/"
|
|
[ -f "$SCRIPT_DIR/build.sh" ] && cp "$SCRIPT_DIR/build.sh" "$INSTALL_DIR/"
|
|
[ -d "$SCRIPT_DIR/zabbix" ] && cp -r "$SCRIPT_DIR/zabbix" "$INSTALL_DIR/"
|
|
fi
|
|
|
|
VENV_DIR="$INSTALL_DIR/venv"
|
|
SERVICE_NAME="airflow-monitor"
|
|
SERVICE_FILE="$INSTALL_DIR/${SERVICE_NAME}.service"
|
|
|
|
echo "=== Airflow DAG Monitor Setup ==="
|
|
echo "Install directory: $INSTALL_DIR"
|
|
echo ""
|
|
|
|
# --- Step 1: Create virtual environment ---
|
|
echo "[1/4] Creating Python virtual environment..."
|
|
if [ -d "$VENV_DIR" ]; then
|
|
echo " Virtual environment already exists, recreating..."
|
|
rm -rf "$VENV_DIR"
|
|
fi
|
|
python3 -m venv "$VENV_DIR"
|
|
echo " Created: $VENV_DIR"
|
|
|
|
# --- Step 2: Install dependencies ---
|
|
echo "[2/4] Installing dependencies..."
|
|
"$VENV_DIR/bin/pip" install --upgrade pip --quiet
|
|
"$VENV_DIR/bin/pip" install -r "$INSTALL_DIR/requirements.txt" --quiet
|
|
echo " Installed: $(cat "$INSTALL_DIR/requirements.txt" | grep -v '^#' | tr '\n' ' ')"
|
|
|
|
# --- Step 3: Create required directories ---
|
|
echo "[3/4] Creating directories..."
|
|
|
|
STATE_DIR="/var/lib/airflow-monitor"
|
|
LOG_DIR="/var/log/airflow-monitor"
|
|
|
|
for dir in "$STATE_DIR" "$LOG_DIR"; do
|
|
if [ ! -d "$dir" ]; then
|
|
sudo mkdir -p "$dir"
|
|
sudo chown "$(whoami):$(id -gn)" "$dir"
|
|
echo " Created: $dir"
|
|
else
|
|
echo " Exists: $dir"
|
|
fi
|
|
done
|
|
|
|
# --- Step 4: Install systemd service ---
|
|
echo "[4/5] Installing systemd service..."
|
|
if [ -f "$SERVICE_FILE" ]; then
|
|
sudo cp "$SERVICE_FILE" /etc/systemd/system/
|
|
sudo systemctl daemon-reload
|
|
echo " Installed: /etc/systemd/system/${SERVICE_NAME}.service"
|
|
else
|
|
echo " WARNING: ${SERVICE_FILE} not found, skipping service install"
|
|
fi
|
|
|
|
# --- Step 5: Install Zabbix agent config ---
|
|
echo "[5/5] Installing Zabbix agent config..."
|
|
ZABBIX_CONF_SRC="$INSTALL_DIR/zabbix/airflow-monitor.conf"
|
|
ZABBIX_CONF_DST="/etc/zabbix/zabbix_agentd.conf.d/airflow-monitor.conf"
|
|
|
|
if [ -f "$ZABBIX_CONF_SRC" ]; then
|
|
if [ -d "/etc/zabbix/zabbix_agentd.conf.d" ]; then
|
|
sudo cp "$ZABBIX_CONF_SRC" "$ZABBIX_CONF_DST"
|
|
# Zabbix agent must be able to read data files
|
|
sudo chmod 644 "$ZABBIX_CONF_DST"
|
|
echo " Installed: $ZABBIX_CONF_DST"
|
|
# Restart zabbix agent to pick up new UserParameters
|
|
if systemctl is-active zabbix-agent >/dev/null 2>&1; then
|
|
sudo systemctl restart zabbix-agent
|
|
echo " Restarted: zabbix-agent"
|
|
elif systemctl is-active zabbix-agent2 >/dev/null 2>&1; then
|
|
sudo systemctl restart zabbix-agent2
|
|
echo " Restarted: zabbix-agent2"
|
|
else
|
|
echo " WARNING: zabbix-agent not running, restart it manually"
|
|
fi
|
|
else
|
|
echo " WARNING: /etc/zabbix/zabbix_agentd.conf.d/ not found"
|
|
echo " Copy manually: cp $ZABBIX_CONF_SRC $ZABBIX_CONF_DST"
|
|
fi
|
|
else
|
|
echo " WARNING: zabbix config not found at $ZABBIX_CONF_SRC"
|
|
fi
|
|
|
|
# Ensure data directory is readable by zabbix agent
|
|
sudo chmod 755 "$STATE_DIR"
|
|
|
|
echo ""
|
|
echo "=== Setup Complete ==="
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Edit config.yaml:"
|
|
echo " vim $INSTALL_DIR/config.yaml"
|
|
echo " 2. Test manually:"
|
|
echo " $VENV_DIR/bin/python -m airflow_monitor --discover -c $INSTALL_DIR/config.yaml"
|
|
echo " 3. Enable and start:"
|
|
echo " sudo systemctl enable --now $SERVICE_NAME"
|
|
echo " 4. Check logs:"
|
|
echo " journalctl -u $SERVICE_NAME -f"
|
|
echo " 5. Verify Zabbix agent reads data:"
|
|
echo " zabbix_agentd -t airflow.monitor.heartbeat"
|
|
echo " zabbix_agentd -t airflow.dag.problems.count"
|