From ee3c7cfa2c165db5c69a66066187d2fd0a172e08 Mon Sep 17 00:00:00 2001 From: Maksim Totmin Date: Wed, 8 Apr 2026 18:13:47 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=B0=D0=B2=D1=82=D0=BE=D1=81=D0=B1=D0=BE=D1=80=D0=BA?= =?UTF-8?q?=D1=83=20Python=203.11=20=D0=B2=20install.sh=20=D0=B4=D0=BB?= =?UTF-8?q?=D1=8F=20=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80=D0=BE=D0=B2=20?= =?UTF-8?q?=D1=81=D0=BE=20=D1=81=D1=82=D0=B0=D1=80=D1=8B=D0=BC=20Python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - install.sh проверяет версию Python, если < 3.10 — собирает 3.11 из исходников - Python ставится локально в /opt/airflow-monitor/python, без влияния на систему - исходники Python 3.11.12 включены в архив (python-src/) - wheels для Python 3.9-3.14 покрывают все версии - архив 26 МБ (20 МБ — исходники Python) --- .gitignore | 1 + build.sh | 252 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 191 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index 1bcf1b0..399a187 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ __pycache__/ venv/ build/ dist/ +cache/ *.tmp .env diff --git a/build.sh b/build.sh index 95a100d..4aad502 100755 --- a/build.sh +++ b/build.sh @@ -2,9 +2,12 @@ # ============================================================================= # Airflow DAG Monitor - Offline Build Script # -# Собирает автономный ZIP-архив со всеми зависимостями (wheel-пакеты), -# исходным кодом и скриптами установки. Архив можно перенести в закрытый -# контур без доступа в интернет и развернуть через install.sh. +# Собирает автономный tar.gz архив со всеми зависимостями: +# - wheel-пакеты Python для версий 3.9-3.14 +# - исходники Python 3.11 (для сборки на серверах со старым Python) +# - исходный код, конфиги, Zabbix шаблоны +# +# Архив полностью автономен — на целевом сервере не нужен интернет. # # Использование: # ./build.sh @@ -23,6 +26,10 @@ DIST_DIR="$SCRIPT_DIR/dist" WHEELS_DIR="$BUILD_DIR/wheels" STAGE_DIR="$BUILD_DIR/airflow-monitor-${VERSION}" +PYTHON_SRC_VERSION="3.11.12" +PYTHON_SRC_URL="https://www.python.org/ftp/python/${PYTHON_SRC_VERSION}/Python-${PYTHON_SRC_VERSION}.tar.xz" +PYTHON_SRC_FILE="Python-${PYTHON_SRC_VERSION}.tar.xz" + echo "=== Airflow DAG Monitor — Offline Build ===" echo "Version: ${VERSION}" echo "Build dir: ${BUILD_DIR}" @@ -33,9 +40,8 @@ rm -rf "$BUILD_DIR" mkdir -p "$WHEELS_DIR" "$STAGE_DIR" "$DIST_DIR" # --- Step 1: Download wheel packages --- -echo "[1/4] Downloading dependencies as wheel packages..." +echo "[1/5] Downloading dependencies as wheel packages..." -# Download wheels for multiple Python versions to support different servers PYTHON_VERSIONS="39 310 311 312 313 314" echo " Downloading for Python versions: ${PYTHON_VERSIONS}" @@ -48,42 +54,49 @@ for pyver in $PYTHON_VERSIONS; do --quiet 2>/dev/null || true done -# Also download pure-python wheels (no version constraint) +# Also download pure-python wheels python3 -m pip download \ -r "$SCRIPT_DIR/requirements.txt" \ --dest "$WHEELS_DIR" \ --quiet 2>/dev/null || true -# Deduplicate (same file may be downloaded multiple times) WHEEL_COUNT=$(find "$WHEELS_DIR" -type f | wc -l) -echo " Downloaded $WHEEL_COUNT packages:" -find "$WHEELS_DIR" -type f -printf " %f\n" | sort +echo " Downloaded $WHEEL_COUNT packages" -# --- Step 2: Stage project files --- -echo "[2/4] Staging project files..." +# --- Step 2: Download Python source --- +echo "[2/5] Downloading Python ${PYTHON_SRC_VERSION} source..." + +PYTHON_SRC_PATH="$STAGE_DIR/python-src" +mkdir -p "$PYTHON_SRC_PATH" + +if [ -f "$SCRIPT_DIR/cache/$PYTHON_SRC_FILE" ]; then + cp "$SCRIPT_DIR/cache/$PYTHON_SRC_FILE" "$PYTHON_SRC_PATH/" + echo " Copied from cache" +else + curl -sL "$PYTHON_SRC_URL" -o "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" + # Cache for future builds + mkdir -p "$SCRIPT_DIR/cache" + cp "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" "$SCRIPT_DIR/cache/" + echo " Downloaded: $PYTHON_SRC_FILE ($(du -h "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" | cut -f1))" +fi + +# --- Step 3: Stage project files --- +echo "[3/5] Staging project files..." -# Application code cp -r "$SCRIPT_DIR/airflow_monitor" "$STAGE_DIR/airflow_monitor" -# Remove __pycache__ find "$STAGE_DIR/airflow_monitor" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true -# Configuration and docs cp "$SCRIPT_DIR/config.yaml" "$STAGE_DIR/" cp "$SCRIPT_DIR/requirements.txt" "$STAGE_DIR/" cp "$SCRIPT_DIR/airflow-monitor.service" "$STAGE_DIR/" cp "$SCRIPT_DIR/README.md" "$STAGE_DIR/" +cp -r "$SCRIPT_DIR/zabbix" "$STAGE_DIR/zabbix" +cp -r "$WHEELS_DIR" "$STAGE_DIR/wheels" -# Zabbix agent config -cp -r "$SCRIPT_DIR/zabbix" "$STAGE_DIR/zabbix" +echo " Staged $(find "$STAGE_DIR" -type f | wc -l) files" -# Wheels directory -cp -r "$WHEELS_DIR" "$STAGE_DIR/wheels" - -echo " Staged files:" -find "$STAGE_DIR" -type f -printf " %P\n" | head -30 - -# --- Step 3: Generate offline install script --- -echo "[3/4] Generating install.sh..." +# --- Step 4: Generate offline install script --- +echo "[4/5] Generating install.sh..." cat > "$STAGE_DIR/install.sh" << 'INSTALL_SCRIPT' #!/usr/bin/env bash @@ -91,34 +104,23 @@ cat > "$STAGE_DIR/install.sh" << 'INSTALL_SCRIPT' # Airflow DAG Monitor - Offline Installer # # Устанавливает сервис из автономного архива без доступа в интернет. -# Все зависимости включены в каталог wheels/. +# Если на сервере Python < 3.10 — автоматически собирает Python 3.11 +# из включённых исходников в каталог приложения (без влияния на систему). # # Использование: # ./install.sh # установка в /opt/airflow-monitor # ./install.sh --prefix /srv/monitor # установка в указанный каталог +# ./install.sh --user airflow # пользователь для systemd # ============================================================================= set -euo pipefail -# --- Check Python version --- -PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0") -PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d. -f1) -PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d. -f2) - -if [ "$PYTHON_MAJOR" -lt 3 ] || { [ "$PYTHON_MAJOR" -eq 3 ] && [ "$PYTHON_MINOR" -lt 10 ]; }; then - echo "ERROR: Python 3.10+ required, found Python ${PYTHON_VERSION}" >&2 - echo "" >&2 - echo "Install Python 3.10+:" >&2 - echo " RHEL/CentOS: yum install python3.11" >&2 - echo " ALT Linux: apt-get install python3" >&2 - echo " Debian: apt install python3.11" >&2 - exit 1 -fi -echo "Python version: ${PYTHON_VERSION} (OK)" - SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" INSTALL_DIR="/opt/airflow-monitor" SERVICE_USER="$(whoami)" SERVICE_NAME="airflow-monitor" +PYTHON_BIN="" +MIN_PYTHON_MAJOR=3 +MIN_PYTHON_MINOR=10 # --- Parse arguments --- while [[ $# -gt 0 ]]; do @@ -137,6 +139,9 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --prefix Installation directory (default: /opt/airflow-monitor)" echo " --user User to run the service as (default: current user)" + echo "" + echo "If system Python < 3.10, Python 3.11 will be built from source" + echo "into INSTALL_DIR/python (no system-wide changes)." exit 0 ;; *) @@ -147,16 +152,138 @@ while [[ $# -gt 0 ]]; do done VENV_DIR="$INSTALL_DIR/venv" +LOCAL_PYTHON_DIR="$INSTALL_DIR/python" + +# --- Find suitable Python --- +find_python() { + # Check common Python binary names in order of preference + for cmd in python3.14 python3.13 python3.12 python3.11 python3.10 python3; do + if command -v "$cmd" >/dev/null 2>&1; then + local ver + ver=$("$cmd" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0") + local major minor + major=$(echo "$ver" | cut -d. -f1) + minor=$(echo "$ver" | cut -d. -f2) + if [ "$major" -ge "$MIN_PYTHON_MAJOR" ] && [ "$minor" -ge "$MIN_PYTHON_MINOR" ]; then + PYTHON_BIN=$(command -v "$cmd") + echo " Found: $PYTHON_BIN (Python $ver)" + return 0 + fi + fi + done + + # Check local Python (from previous install) + if [ -x "$LOCAL_PYTHON_DIR/bin/python3" ]; then + local ver + ver=$("$LOCAL_PYTHON_DIR/bin/python3" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0") + PYTHON_BIN="$LOCAL_PYTHON_DIR/bin/python3" + echo " Found local: $PYTHON_BIN (Python $ver)" + return 0 + fi + + return 1 +} + +# --- Build Python from source --- +build_python() { + local src_archive + src_archive=$(find "$SCRIPT_DIR/python-src" -name 'Python-*.tar.xz' -type f 2>/dev/null | head -1) + + if [ -z "$src_archive" ]; then + echo "ERROR: Python source archive not found in python-src/" >&2 + exit 1 + fi + + local python_version + python_version=$(basename "$src_archive" | sed 's/Python-//; s/.tar.xz//') + echo "" + echo " Building Python ${python_version} from source..." + echo " Target: ${LOCAL_PYTHON_DIR}" + echo " This may take 3-10 minutes..." + echo "" + + # Check build dependencies + local missing_deps=() + for cmd in gcc make; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing_deps+=("$cmd") + fi + done + + if [ ${#missing_deps[@]} -gt 0 ]; then + echo "ERROR: Missing build tools: ${missing_deps[*]}" >&2 + echo "" >&2 + echo "Install them first:" >&2 + echo " ALT Linux: apt-get install gcc make libssl-devel zlib-devel libffi-devel" >&2 + echo " RHEL/CentOS: yum install gcc make openssl-devel zlib-devel libffi-devel" >&2 + echo " Debian: apt install gcc make libssl-dev zlib1g-dev libffi-dev" >&2 + exit 1 + fi + + # Extract and build + local build_dir + build_dir=$(mktemp -d) + trap "rm -rf '$build_dir'" EXIT + + echo " [1/3] Extracting source..." + tar xf "$src_archive" -C "$build_dir" + + local src_dir + src_dir=$(find "$build_dir" -maxdepth 1 -name 'Python-*' -type d | head -1) + + echo " [2/3] Configuring (prefix=${LOCAL_PYTHON_DIR})..." + ( + cd "$src_dir" + ./configure \ + --prefix="$LOCAL_PYTHON_DIR" \ + --enable-optimizations \ + --with-ensurepip=install \ + --quiet 2>&1 | tail -5 + ) + + echo " [3/3] Compiling and installing..." + ( + cd "$src_dir" + make -j"$(nproc)" --quiet 2>&1 | tail -3 + make install --quiet 2>&1 | tail -3 + ) + + # Clean up build temp (trap handles this) + trap - EXIT + rm -rf "$build_dir" + + PYTHON_BIN="$LOCAL_PYTHON_DIR/bin/python3" + + if [ -x "$PYTHON_BIN" ]; then + local ver + ver=$("$PYTHON_BIN" --version 2>&1) + echo " Built successfully: $ver" + echo " Location: $PYTHON_BIN" + else + echo "ERROR: Python build failed" >&2 + exit 1 + fi +} echo "=== Airflow DAG Monitor — Offline Install ===" -echo "Source: $SCRIPT_DIR" -echo "Install dir: $INSTALL_DIR" +echo "Source: $SCRIPT_DIR" +echo "Install dir: $INSTALL_DIR" echo "Service user: $SERVICE_USER" echo "" -# --- Step 1: Copy files if installing to a different directory --- +# --- Step 1: Find or build Python --- +echo "[1/7] Checking Python..." +if ! find_python; then + sys_ver=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "not found") + echo " System Python: ${sys_ver} (need ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+)" + echo " Building Python from included source..." + mkdir -p "$INSTALL_DIR" + build_python +fi + +# --- Step 2: Copy files --- if [ "$INSTALL_DIR" != "$SCRIPT_DIR" ]; then - echo "[1/5] Copying files to $INSTALL_DIR..." + echo "[2/7] Copying files to $INSTALL_DIR..." mkdir -p "$INSTALL_DIR" cp -r "$SCRIPT_DIR/airflow_monitor" "$INSTALL_DIR/" cp -r "$SCRIPT_DIR/wheels" "$INSTALL_DIR/" @@ -165,21 +292,24 @@ if [ "$INSTALL_DIR" != "$SCRIPT_DIR" ]; then cp "$SCRIPT_DIR/requirements.txt" "$INSTALL_DIR/" cp "$SCRIPT_DIR/airflow-monitor.service" "$INSTALL_DIR/" cp "$SCRIPT_DIR/README.md" "$INSTALL_DIR/" + # Keep python-src for potential reinstalls + [ -d "$SCRIPT_DIR/python-src" ] && cp -r "$SCRIPT_DIR/python-src" "$INSTALL_DIR/" echo " Done" else - echo "[1/5] Installing in place, skipping copy" + echo "[2/7] Installing in place, skipping copy" fi -# --- Step 2: Create virtual environment --- -echo "[2/5] Creating Python virtual environment..." +# --- Step 3: Create virtual environment --- +echo "[3/7] Creating Python virtual environment..." if [ -d "$VENV_DIR" ]; then rm -rf "$VENV_DIR" fi -python3 -m venv "$VENV_DIR" +"$PYTHON_BIN" -m venv "$VENV_DIR" echo " Created: $VENV_DIR" +echo " Python: $("$VENV_DIR/bin/python3" --version)" -# --- Step 3: Install dependencies from local wheels --- -echo "[3/5] Installing dependencies from local wheels (offline)..." +# --- Step 4: Install dependencies from local wheels --- +echo "[4/7] Installing dependencies from local wheels (offline)..." "$VENV_DIR/bin/pip" install \ --no-index \ --find-links "$INSTALL_DIR/wheels" \ @@ -189,8 +319,8 @@ echo " Installed packages:" "$VENV_DIR/bin/pip" list --format=columns 2>/dev/null | grep -vE '^(Package|---)' echo "" -# --- Step 4: Create required directories --- -echo "[4/5] Creating directories..." +# --- Step 5: Create required directories --- +echo "[5/7] Creating directories..." STATE_DIR="/var/lib/airflow-monitor" LOG_DIR="/var/log/airflow-monitor" @@ -205,10 +335,9 @@ for dir in "$STATE_DIR" "$LOG_DIR"; do fi done -# --- Step 5: Install systemd service --- -echo "[5/5] Installing systemd service..." +# --- Step 6: Install systemd service --- +echo "[6/7] Installing systemd service..." -# Patch service file with actual paths and user SERVICE_FILE="$INSTALL_DIR/airflow-monitor.service" if [ -f "$SERVICE_FILE" ]; then TEMP_SERVICE=$(mktemp) @@ -216,7 +345,7 @@ if [ -f "$SERVICE_FILE" ]; then -e "s|User=.*|User=${SERVICE_USER}|" \ -e "s|Group=.*|Group=$(id -gn "$SERVICE_USER")|" \ -e "s|WorkingDirectory=.*|WorkingDirectory=${INSTALL_DIR}|" \ - -e "s|ExecStart=.*|ExecStart=${VENV_DIR}/bin/python -m airflow_monitor --config ${INSTALL_DIR}/config.yaml|" \ + -e "s|ExecStart=.*|ExecStart=${VENV_DIR}/bin/python -m airflow_monitor --discover --config ${INSTALL_DIR}/config.yaml|" \ "$SERVICE_FILE" > "$TEMP_SERVICE" sudo cp "$TEMP_SERVICE" /etc/systemd/system/${SERVICE_NAME}.service @@ -227,8 +356,8 @@ else echo " WARNING: service file not found, skipping" fi -# --- Step 6: Install Zabbix agent config --- -echo "[6/6] Installing Zabbix agent config..." +# --- Step 7: Install Zabbix agent config --- +echo "[7/7] Installing Zabbix agent config..." ZABBIX_CONF_SRC="$INSTALL_DIR/zabbix/airflow-monitor.conf" ZABBIX_CONF_DST="/etc/zabbix/zabbix_agentd.conf.d/airflow-monitor.conf" @@ -254,7 +383,6 @@ else echo " WARNING: zabbix config not found" fi -# Ensure data dir is readable by zabbix agent sudo chmod 755 /var/lib/airflow-monitor 2>/dev/null || true echo "" @@ -281,8 +409,8 @@ INSTALL_SCRIPT chmod +x "$STAGE_DIR/install.sh" echo " Generated: install.sh" -# --- Step 4: Create archive --- -echo "[4/4] Creating tar.gz archive..." +# --- Step 5: Create archive --- +echo "[5/5] Creating tar.gz archive..." ARCHIVE_NAME="airflow-monitor-${VERSION}.tar.gz" ARCHIVE_PATH="$DIST_DIR/$ARCHIVE_NAME"