#!/usr/bin/env bash # ============================================================================= # Airflow DAG Monitor - Offline Build Script # # Собирает автономный tar.gz архив со всеми зависимостями: # - wheel-пакеты Python для версий 3.9-3.14 # - исходники Python 3.11 (для сборки на серверах со старым Python) # - исходный код, конфиги, Zabbix шаблоны # # Архив полностью автономен — на целевом сервере не нужен интернет. # # Использование: # ./build.sh # ============================================================================= set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" VERSION=$(python3 -c " import re, pathlib text = pathlib.Path('$SCRIPT_DIR/airflow_monitor/__init__.py').read_text() m = re.search(r'__version__\s*=\s*[\\x22\\x27]([^\\x22\\x27]+)', text) print(m.group(1) if m else '0.0.0') ") BUILD_DIR="$SCRIPT_DIR/build" DIST_DIR="$SCRIPT_DIR/dist" WHEELS_DIR="$BUILD_DIR/wheels" STAGE_DIR="$BUILD_DIR/airflow-monitor-${VERSION}" PYTHON_SRC_VERSION="3.11.12" PYTHON_SRC_URL="https://www.python.org/ftp/python/${PYTHON_SRC_VERSION}/Python-${PYTHON_SRC_VERSION}.tar.xz" PYTHON_SRC_FILE="Python-${PYTHON_SRC_VERSION}.tar.xz" echo "=== Airflow DAG Monitor — Offline Build ===" echo "Version: ${VERSION}" echo "Build dir: ${BUILD_DIR}" echo "" # --- Cleanup previous build --- rm -rf "$BUILD_DIR" mkdir -p "$WHEELS_DIR" "$STAGE_DIR" "$DIST_DIR" # --- Step 1: Download wheel packages --- echo "[1/5] Downloading dependencies as wheel packages..." PYTHON_VERSIONS="39 310 311 312 313 314" echo " Downloading for Python versions: ${PYTHON_VERSIONS}" for pyver in $PYTHON_VERSIONS; do python3 -m pip download \ -r "$SCRIPT_DIR/requirements.txt" \ --dest "$WHEELS_DIR" \ --python-version "$pyver" \ --only-binary=:all: \ --quiet 2>/dev/null || true done # Also download pure-python wheels python3 -m pip download \ -r "$SCRIPT_DIR/requirements.txt" \ --dest "$WHEELS_DIR" \ --quiet 2>/dev/null || true WHEEL_COUNT=$(find "$WHEELS_DIR" -type f | wc -l) echo " Downloaded $WHEEL_COUNT packages" # --- Step 2: Download Python source --- echo "[2/5] Downloading Python ${PYTHON_SRC_VERSION} source..." PYTHON_SRC_PATH="$STAGE_DIR/python-src" mkdir -p "$PYTHON_SRC_PATH" if [ -f "$SCRIPT_DIR/cache/$PYTHON_SRC_FILE" ]; then cp "$SCRIPT_DIR/cache/$PYTHON_SRC_FILE" "$PYTHON_SRC_PATH/" echo " Copied from cache" else curl -sL "$PYTHON_SRC_URL" -o "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" # Cache for future builds mkdir -p "$SCRIPT_DIR/cache" cp "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" "$SCRIPT_DIR/cache/" echo " Downloaded: $PYTHON_SRC_FILE ($(du -h "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" | cut -f1))" fi # --- Step 3: Stage project files --- echo "[3/5] Staging project files..." cp -r "$SCRIPT_DIR/airflow_monitor" "$STAGE_DIR/airflow_monitor" find "$STAGE_DIR/airflow_monitor" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true cp "$SCRIPT_DIR/config.yaml" "$STAGE_DIR/" cp "$SCRIPT_DIR/requirements.txt" "$STAGE_DIR/" cp "$SCRIPT_DIR/airflow-monitor.service" "$STAGE_DIR/" cp "$SCRIPT_DIR/README.md" "$STAGE_DIR/" cp -r "$SCRIPT_DIR/zabbix" "$STAGE_DIR/zabbix" cp -r "$WHEELS_DIR" "$STAGE_DIR/wheels" echo " Staged $(find "$STAGE_DIR" -type f | wc -l) files" # --- Step 4: Generate offline install script --- echo "[4/5] Generating install.sh..." cat > "$STAGE_DIR/install.sh" << 'INSTALL_SCRIPT' #!/usr/bin/env bash # ============================================================================= # Airflow DAG Monitor - Offline Installer # # Устанавливает сервис из автономного архива без доступа в интернет. # Если на сервере Python < 3.10 — автоматически собирает Python 3.11 # из включённых исходников в каталог приложения (без влияния на систему). # # Использование: # bash install.sh # установка в /opt/airflow-monitor # bash install.sh --prefix /srv/monitor # установка в указанный каталог # bash install.sh --user airflow # пользователь для systemd # ============================================================================= set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" INSTALL_DIR="/opt/airflow-monitor" SERVICE_USER="$(whoami)" SERVICE_NAME="airflow-monitor" PYTHON_BIN="" MIN_PYTHON_MAJOR=3 MIN_PYTHON_MINOR=10 # --- Parse arguments --- while [[ $# -gt 0 ]]; do case "$1" in --prefix) INSTALL_DIR="$2" shift 2 ;; --user) SERVICE_USER="$2" shift 2 ;; -h|--help) echo "Usage: $0 [--prefix INSTALL_DIR] [--user SERVICE_USER]" echo "" echo "Options:" echo " --prefix Installation directory (default: /opt/airflow-monitor)" echo " --user User to run the service as (default: current user)" echo "" echo "If system Python < 3.10, Python 3.11 will be built from source" echo "into INSTALL_DIR/python (no system-wide changes)." exit 0 ;; *) echo "Unknown option: $1" >&2 exit 1 ;; esac done VENV_DIR="$INSTALL_DIR/venv" LOCAL_PYTHON_DIR="$INSTALL_DIR/python" # --- Find suitable Python --- find_python() { # Check common Python binary names in order of preference for cmd in python3.14 python3.13 python3.12 python3.11 python3.10 python3; do if command -v "$cmd" >/dev/null 2>&1; then local ver ver=$("$cmd" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0") local major minor major=$(echo "$ver" | cut -d. -f1) minor=$(echo "$ver" | cut -d. -f2) if [ "$major" -ge "$MIN_PYTHON_MAJOR" ] && [ "$minor" -ge "$MIN_PYTHON_MINOR" ]; then PYTHON_BIN=$(command -v "$cmd") echo " Found: $PYTHON_BIN (Python $ver)" return 0 fi fi done # Check local Python (from previous install) if [ -x "$LOCAL_PYTHON_DIR/bin/python3" ]; then local ver ver=$("$LOCAL_PYTHON_DIR/bin/python3" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0") PYTHON_BIN="$LOCAL_PYTHON_DIR/bin/python3" echo " Found local: $PYTHON_BIN (Python $ver)" return 0 fi return 1 } # --- Build Python from source --- build_python() { local src_archive src_archive=$(find "$SCRIPT_DIR/python-src" -name 'Python-*.tar.xz' -type f 2>/dev/null | head -1) if [ -z "$src_archive" ]; then echo "ERROR: Python source archive not found in python-src/" >&2 exit 1 fi local python_version python_version=$(basename "$src_archive" | sed 's/Python-//; s/.tar.xz//') echo "" echo " Building Python ${python_version} from source..." echo " Target: ${LOCAL_PYTHON_DIR}" echo " This may take 3-10 minutes..." echo "" # Check build dependencies local missing_deps=() for cmd in gcc make; do if ! command -v "$cmd" >/dev/null 2>&1; then missing_deps+=("$cmd") fi done if [ ${#missing_deps[@]} -gt 0 ]; then echo "ERROR: Missing build tools: ${missing_deps[*]}" >&2 echo "" >&2 echo "Install them first:" >&2 echo " ALT Linux: apt-get install gcc make libssl-devel zlib-devel libffi-devel" >&2 echo " RHEL/CentOS: yum install gcc make openssl-devel zlib-devel libffi-devel" >&2 echo " Debian: apt install gcc make libssl-dev zlib1g-dev libffi-dev" >&2 exit 1 fi # Extract and build local build_dir build_dir=$(mktemp -d) trap "rm -rf '$build_dir'" EXIT echo " [1/3] Extracting source..." tar xf "$src_archive" -C "$build_dir" local src_dir src_dir=$(find "$build_dir" -maxdepth 1 -name 'Python-*' -type d | head -1) echo " [2/3] Configuring (prefix=${LOCAL_PYTHON_DIR})..." ( cd "$src_dir" ./configure \ --prefix="$LOCAL_PYTHON_DIR" \ --enable-optimizations \ --with-ensurepip=install \ --quiet 2>&1 | tail -5 ) echo " [3/3] Compiling and installing..." ( cd "$src_dir" make -j"$(nproc)" --quiet 2>&1 | tail -3 make install --quiet 2>&1 | tail -3 ) # Clean up build temp (trap handles this) trap - EXIT rm -rf "$build_dir" PYTHON_BIN="$LOCAL_PYTHON_DIR/bin/python3" if [ -x "$PYTHON_BIN" ]; then local ver ver=$("$PYTHON_BIN" --version 2>&1) echo " Built successfully: $ver" echo " Location: $PYTHON_BIN" else echo "ERROR: Python build failed" >&2 exit 1 fi } echo "=== Airflow DAG Monitor — Offline Install ===" echo "Source: $SCRIPT_DIR" echo "Install dir: $INSTALL_DIR" echo "Service user: $SERVICE_USER" echo "" # --- Step 1: Find or build Python --- echo "[1/7] Checking Python..." if ! find_python; then sys_ver=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "not found") echo " System Python: ${sys_ver} (need ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+)" echo " Building Python from included source..." mkdir -p "$INSTALL_DIR" build_python fi # --- Step 2: Copy files --- if [ "$INSTALL_DIR" != "$SCRIPT_DIR" ]; then echo "[2/7] Copying files to $INSTALL_DIR..." mkdir -p "$INSTALL_DIR" cp -r "$SCRIPT_DIR/airflow_monitor" "$INSTALL_DIR/" cp -r "$SCRIPT_DIR/wheels" "$INSTALL_DIR/" cp -r "$SCRIPT_DIR/zabbix" "$INSTALL_DIR/" cp "$SCRIPT_DIR/config.yaml" "$INSTALL_DIR/" cp "$SCRIPT_DIR/requirements.txt" "$INSTALL_DIR/" cp "$SCRIPT_DIR/airflow-monitor.service" "$INSTALL_DIR/" cp "$SCRIPT_DIR/README.md" "$INSTALL_DIR/" # Keep python-src for potential reinstalls [ -d "$SCRIPT_DIR/python-src" ] && cp -r "$SCRIPT_DIR/python-src" "$INSTALL_DIR/" echo " Done" else echo "[2/7] Installing in place, skipping copy" fi # --- Step 3: Create virtual environment --- echo "[3/7] Creating Python virtual environment..." if [ -d "$VENV_DIR" ]; then rm -rf "$VENV_DIR" fi "$PYTHON_BIN" -m venv "$VENV_DIR" echo " Created: $VENV_DIR" echo " Python: $("$VENV_DIR/bin/python3" --version)" # --- Step 4: Install dependencies from local wheels --- echo "[4/7] Installing dependencies from local wheels (offline)..." "$VENV_DIR/bin/pip" install \ --no-index \ --find-links "$INSTALL_DIR/wheels" \ -r "$INSTALL_DIR/requirements.txt" \ --quiet echo " Installed packages:" "$VENV_DIR/bin/pip" list --format=columns 2>/dev/null | grep -vE '^(Package|---)' echo "" # --- Step 5: Create required directories --- echo "[5/7] Creating directories..." STATE_DIR="/var/lib/airflow-monitor" LOG_DIR="/var/log/airflow-monitor" for dir in "$STATE_DIR" "$LOG_DIR"; do if [ ! -d "$dir" ]; then sudo mkdir -p "$dir" sudo chown "${SERVICE_USER}:$(id -gn "$SERVICE_USER")" "$dir" echo " Created: $dir" else echo " Exists: $dir" fi # Ensure zabbix agent can read data files sudo chmod 755 "$dir" done # --- Step 6: Install systemd service --- echo "[6/7] Installing systemd service..." SERVICE_FILE="$INSTALL_DIR/airflow-monitor.service" if [ -f "$SERVICE_FILE" ]; then TEMP_SERVICE=$(mktemp) sed \ -e "s|User=.*|User=${SERVICE_USER}|" \ -e "s|Group=.*|Group=$(id -gn "$SERVICE_USER")|" \ -e "s|WorkingDirectory=.*|WorkingDirectory=${INSTALL_DIR}|" \ -e "s|ExecStart=.*|ExecStart=${VENV_DIR}/bin/python -m airflow_monitor --discover --config ${INSTALL_DIR}/config.yaml|" \ "$SERVICE_FILE" > "$TEMP_SERVICE" sudo cp "$TEMP_SERVICE" /etc/systemd/system/${SERVICE_NAME}.service rm -f "$TEMP_SERVICE" sudo systemctl daemon-reload echo " Installed: /etc/systemd/system/${SERVICE_NAME}.service" else echo " WARNING: service file not found, skipping" fi # --- Step 7: Install Zabbix agent config --- echo "[7/7] Installing Zabbix agent config..." ZABBIX_CONF_SRC="$INSTALL_DIR/zabbix/airflow-monitor.conf" ZABBIX_CONF_DST="/etc/zabbix/zabbix_agentd.conf.d/airflow-monitor.conf" if [ -f "$ZABBIX_CONF_SRC" ]; then if [ -d "/etc/zabbix/zabbix_agentd.conf.d" ]; then sudo cp "$ZABBIX_CONF_SRC" "$ZABBIX_CONF_DST" sudo chmod 644 "$ZABBIX_CONF_DST" echo " Installed: $ZABBIX_CONF_DST" # Try common zabbix agent service names restarted=false for svc in zabbix_agentd zabbix-agent zabbix-agent2; do if systemctl is-active "$svc" >/dev/null 2>&1; then sudo systemctl restart "$svc" echo " Restarted: $svc" restarted=true break fi done if [ "$restarted" = false ]; then echo " WARNING: zabbix agent not found, restart manually" fi else echo " WARNING: /etc/zabbix/zabbix_agentd.conf.d/ not found" echo " Copy manually: sudo cp $ZABBIX_CONF_SRC $ZABBIX_CONF_DST" fi else echo " WARNING: zabbix config not found" fi sudo chmod 755 /var/lib/airflow-monitor 2>/dev/null || true echo "" echo "=== Install Complete ===" echo "" echo "Next steps:" echo " 1. Edit config.yaml:" echo " vim $INSTALL_DIR/config.yaml" echo "" echo " 2. Test manually:" echo " $VENV_DIR/bin/python -m airflow_monitor --discover -c $INSTALL_DIR/config.yaml" echo "" echo " 3. Enable and start:" echo " sudo systemctl enable --now $SERVICE_NAME" echo "" echo " 4. Check logs:" echo " journalctl -u $SERVICE_NAME -f" echo "" echo " 5. Verify Zabbix agent reads data:" echo " zabbix_agentd -t airflow.monitor.heartbeat" echo " zabbix_agentd -t airflow.dag.problems.count" INSTALL_SCRIPT chmod +x "$STAGE_DIR/install.sh" echo " Generated: install.sh" # --- Step 5: Create archive --- echo "[5/5] Creating tar.gz archive..." ARCHIVE_NAME="airflow-monitor-${VERSION}.tar.gz" ARCHIVE_PATH="$DIST_DIR/$ARCHIVE_NAME" (cd "$BUILD_DIR" && tar czf "$ARCHIVE_PATH" "airflow-monitor-${VERSION}") ARCHIVE_SIZE=$(du -h "$ARCHIVE_PATH" | cut -f1) echo " Archive: $ARCHIVE_PATH" echo " Size: $ARCHIVE_SIZE" # --- Cleanup build directory --- rm -rf "$BUILD_DIR" echo "" echo "=== Build Complete ===" echo "" echo "Архив для переноса в закрытый контур:" echo " $ARCHIVE_PATH" echo "" echo "Развёртывание на целевом сервере:" echo " 1. Скопировать: scp $ARCHIVE_PATH user@target:/tmp/" echo " 2. Распаковать: tar xzf /tmp/$ARCHIVE_NAME -C /tmp/" echo " 3. Установить: cd /tmp/airflow-monitor-${VERSION} && bash install.sh" echo "" echo "По умолчанию устанавливается в /opt/airflow-monitor." echo "Для смены каталога или пользователя:" echo " bash install.sh --prefix /srv/airflow-monitor --user airflow"