feat: добавил автосборку Python 3.11 в install.sh для серверов со старым Python

- install.sh проверяет версию Python, если < 3.10 — собирает 3.11 из исходников
- Python ставится локально в /opt/airflow-monitor/python, без влияния на систему
- исходники Python 3.11.12 включены в архив (python-src/)
- wheels для Python 3.9-3.14 покрывают все версии
- архив 26 МБ (20 МБ — исходники Python)
This commit is contained in:
Maksim Totmin 2026-04-08 18:13:47 +07:00
parent 3ec3660455
commit ee3c7cfa2c
2 changed files with 191 additions and 62 deletions

1
.gitignore vendored
View File

@ -4,5 +4,6 @@ __pycache__/
venv/ venv/
build/ build/
dist/ dist/
cache/
*.tmp *.tmp
.env .env

244
build.sh
View File

@ -2,9 +2,12 @@
# ============================================================================= # =============================================================================
# Airflow DAG Monitor - Offline Build Script # Airflow DAG Monitor - Offline Build Script
# #
# Собирает автономный ZIP-архив со всеми зависимостями (wheel-пакеты), # Собирает автономный tar.gz архив со всеми зависимостями:
# исходным кодом и скриптами установки. Архив можно перенести в закрытый # - wheel-пакеты Python для версий 3.9-3.14
# контур без доступа в интернет и развернуть через install.sh. # - исходники Python 3.11 (для сборки на серверах со старым Python)
# - исходный код, конфиги, Zabbix шаблоны
#
# Архив полностью автономен — на целевом сервере не нужен интернет.
# #
# Использование: # Использование:
# ./build.sh # ./build.sh
@ -23,6 +26,10 @@ DIST_DIR="$SCRIPT_DIR/dist"
WHEELS_DIR="$BUILD_DIR/wheels" WHEELS_DIR="$BUILD_DIR/wheels"
STAGE_DIR="$BUILD_DIR/airflow-monitor-${VERSION}" STAGE_DIR="$BUILD_DIR/airflow-monitor-${VERSION}"
PYTHON_SRC_VERSION="3.11.12"
PYTHON_SRC_URL="https://www.python.org/ftp/python/${PYTHON_SRC_VERSION}/Python-${PYTHON_SRC_VERSION}.tar.xz"
PYTHON_SRC_FILE="Python-${PYTHON_SRC_VERSION}.tar.xz"
echo "=== Airflow DAG Monitor — Offline Build ===" echo "=== Airflow DAG Monitor — Offline Build ==="
echo "Version: ${VERSION}" echo "Version: ${VERSION}"
echo "Build dir: ${BUILD_DIR}" echo "Build dir: ${BUILD_DIR}"
@ -33,9 +40,8 @@ rm -rf "$BUILD_DIR"
mkdir -p "$WHEELS_DIR" "$STAGE_DIR" "$DIST_DIR" mkdir -p "$WHEELS_DIR" "$STAGE_DIR" "$DIST_DIR"
# --- Step 1: Download wheel packages --- # --- Step 1: Download wheel packages ---
echo "[1/4] Downloading dependencies as wheel packages..." echo "[1/5] Downloading dependencies as wheel packages..."
# Download wheels for multiple Python versions to support different servers
PYTHON_VERSIONS="39 310 311 312 313 314" PYTHON_VERSIONS="39 310 311 312 313 314"
echo " Downloading for Python versions: ${PYTHON_VERSIONS}" echo " Downloading for Python versions: ${PYTHON_VERSIONS}"
@ -48,42 +54,49 @@ for pyver in $PYTHON_VERSIONS; do
--quiet 2>/dev/null || true --quiet 2>/dev/null || true
done done
# Also download pure-python wheels (no version constraint) # Also download pure-python wheels
python3 -m pip download \ python3 -m pip download \
-r "$SCRIPT_DIR/requirements.txt" \ -r "$SCRIPT_DIR/requirements.txt" \
--dest "$WHEELS_DIR" \ --dest "$WHEELS_DIR" \
--quiet 2>/dev/null || true --quiet 2>/dev/null || true
# Deduplicate (same file may be downloaded multiple times)
WHEEL_COUNT=$(find "$WHEELS_DIR" -type f | wc -l) WHEEL_COUNT=$(find "$WHEELS_DIR" -type f | wc -l)
echo " Downloaded $WHEEL_COUNT packages:" echo " Downloaded $WHEEL_COUNT packages"
find "$WHEELS_DIR" -type f -printf " %f\n" | sort
# --- Step 2: Stage project files --- # --- Step 2: Download Python source ---
echo "[2/4] Staging project files..." echo "[2/5] Downloading Python ${PYTHON_SRC_VERSION} source..."
PYTHON_SRC_PATH="$STAGE_DIR/python-src"
mkdir -p "$PYTHON_SRC_PATH"
if [ -f "$SCRIPT_DIR/cache/$PYTHON_SRC_FILE" ]; then
cp "$SCRIPT_DIR/cache/$PYTHON_SRC_FILE" "$PYTHON_SRC_PATH/"
echo " Copied from cache"
else
curl -sL "$PYTHON_SRC_URL" -o "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE"
# Cache for future builds
mkdir -p "$SCRIPT_DIR/cache"
cp "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" "$SCRIPT_DIR/cache/"
echo " Downloaded: $PYTHON_SRC_FILE ($(du -h "$PYTHON_SRC_PATH/$PYTHON_SRC_FILE" | cut -f1))"
fi
# --- Step 3: Stage project files ---
echo "[3/5] Staging project files..."
# Application code
cp -r "$SCRIPT_DIR/airflow_monitor" "$STAGE_DIR/airflow_monitor" cp -r "$SCRIPT_DIR/airflow_monitor" "$STAGE_DIR/airflow_monitor"
# Remove __pycache__
find "$STAGE_DIR/airflow_monitor" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true find "$STAGE_DIR/airflow_monitor" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
# Configuration and docs
cp "$SCRIPT_DIR/config.yaml" "$STAGE_DIR/" cp "$SCRIPT_DIR/config.yaml" "$STAGE_DIR/"
cp "$SCRIPT_DIR/requirements.txt" "$STAGE_DIR/" cp "$SCRIPT_DIR/requirements.txt" "$STAGE_DIR/"
cp "$SCRIPT_DIR/airflow-monitor.service" "$STAGE_DIR/" cp "$SCRIPT_DIR/airflow-monitor.service" "$STAGE_DIR/"
cp "$SCRIPT_DIR/README.md" "$STAGE_DIR/" cp "$SCRIPT_DIR/README.md" "$STAGE_DIR/"
# Zabbix agent config
cp -r "$SCRIPT_DIR/zabbix" "$STAGE_DIR/zabbix" cp -r "$SCRIPT_DIR/zabbix" "$STAGE_DIR/zabbix"
# Wheels directory
cp -r "$WHEELS_DIR" "$STAGE_DIR/wheels" cp -r "$WHEELS_DIR" "$STAGE_DIR/wheels"
echo " Staged files:" echo " Staged $(find "$STAGE_DIR" -type f | wc -l) files"
find "$STAGE_DIR" -type f -printf " %P\n" | head -30
# --- Step 3: Generate offline install script --- # --- Step 4: Generate offline install script ---
echo "[3/4] Generating install.sh..." echo "[4/5] Generating install.sh..."
cat > "$STAGE_DIR/install.sh" << 'INSTALL_SCRIPT' cat > "$STAGE_DIR/install.sh" << 'INSTALL_SCRIPT'
#!/usr/bin/env bash #!/usr/bin/env bash
@ -91,34 +104,23 @@ cat > "$STAGE_DIR/install.sh" << 'INSTALL_SCRIPT'
# Airflow DAG Monitor - Offline Installer # Airflow DAG Monitor - Offline Installer
# #
# Устанавливает сервис из автономного архива без доступа в интернет. # Устанавливает сервис из автономного архива без доступа в интернет.
# Все зависимости включены в каталог wheels/. # Если на сервере Python < 3.10 — автоматически собирает Python 3.11
# из включённых исходников в каталог приложения (без влияния на систему).
# #
# Использование: # Использование:
# ./install.sh # установка в /opt/airflow-monitor # ./install.sh # установка в /opt/airflow-monitor
# ./install.sh --prefix /srv/monitor # установка в указанный каталог # ./install.sh --prefix /srv/monitor # установка в указанный каталог
# ./install.sh --user airflow # пользователь для systemd
# ============================================================================= # =============================================================================
set -euo pipefail set -euo pipefail
# --- Check Python version ---
PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0")
PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d. -f1)
PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d. -f2)
if [ "$PYTHON_MAJOR" -lt 3 ] || { [ "$PYTHON_MAJOR" -eq 3 ] && [ "$PYTHON_MINOR" -lt 10 ]; }; then
echo "ERROR: Python 3.10+ required, found Python ${PYTHON_VERSION}" >&2
echo "" >&2
echo "Install Python 3.10+:" >&2
echo " RHEL/CentOS: yum install python3.11" >&2
echo " ALT Linux: apt-get install python3" >&2
echo " Debian: apt install python3.11" >&2
exit 1
fi
echo "Python version: ${PYTHON_VERSION} (OK)"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
INSTALL_DIR="/opt/airflow-monitor" INSTALL_DIR="/opt/airflow-monitor"
SERVICE_USER="$(whoami)" SERVICE_USER="$(whoami)"
SERVICE_NAME="airflow-monitor" SERVICE_NAME="airflow-monitor"
PYTHON_BIN=""
MIN_PYTHON_MAJOR=3
MIN_PYTHON_MINOR=10
# --- Parse arguments --- # --- Parse arguments ---
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
@ -137,6 +139,9 @@ while [[ $# -gt 0 ]]; do
echo "Options:" echo "Options:"
echo " --prefix Installation directory (default: /opt/airflow-monitor)" echo " --prefix Installation directory (default: /opt/airflow-monitor)"
echo " --user User to run the service as (default: current user)" echo " --user User to run the service as (default: current user)"
echo ""
echo "If system Python < 3.10, Python 3.11 will be built from source"
echo "into INSTALL_DIR/python (no system-wide changes)."
exit 0 exit 0
;; ;;
*) *)
@ -147,6 +152,118 @@ while [[ $# -gt 0 ]]; do
done done
VENV_DIR="$INSTALL_DIR/venv" VENV_DIR="$INSTALL_DIR/venv"
LOCAL_PYTHON_DIR="$INSTALL_DIR/python"
# --- Find suitable Python ---
find_python() {
# Check common Python binary names in order of preference
for cmd in python3.14 python3.13 python3.12 python3.11 python3.10 python3; do
if command -v "$cmd" >/dev/null 2>&1; then
local ver
ver=$("$cmd" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0")
local major minor
major=$(echo "$ver" | cut -d. -f1)
minor=$(echo "$ver" | cut -d. -f2)
if [ "$major" -ge "$MIN_PYTHON_MAJOR" ] && [ "$minor" -ge "$MIN_PYTHON_MINOR" ]; then
PYTHON_BIN=$(command -v "$cmd")
echo " Found: $PYTHON_BIN (Python $ver)"
return 0
fi
fi
done
# Check local Python (from previous install)
if [ -x "$LOCAL_PYTHON_DIR/bin/python3" ]; then
local ver
ver=$("$LOCAL_PYTHON_DIR/bin/python3" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "0.0")
PYTHON_BIN="$LOCAL_PYTHON_DIR/bin/python3"
echo " Found local: $PYTHON_BIN (Python $ver)"
return 0
fi
return 1
}
# --- Build Python from source ---
build_python() {
local src_archive
src_archive=$(find "$SCRIPT_DIR/python-src" -name 'Python-*.tar.xz' -type f 2>/dev/null | head -1)
if [ -z "$src_archive" ]; then
echo "ERROR: Python source archive not found in python-src/" >&2
exit 1
fi
local python_version
python_version=$(basename "$src_archive" | sed 's/Python-//; s/.tar.xz//')
echo ""
echo " Building Python ${python_version} from source..."
echo " Target: ${LOCAL_PYTHON_DIR}"
echo " This may take 3-10 minutes..."
echo ""
# Check build dependencies
local missing_deps=()
for cmd in gcc make; do
if ! command -v "$cmd" >/dev/null 2>&1; then
missing_deps+=("$cmd")
fi
done
if [ ${#missing_deps[@]} -gt 0 ]; then
echo "ERROR: Missing build tools: ${missing_deps[*]}" >&2
echo "" >&2
echo "Install them first:" >&2
echo " ALT Linux: apt-get install gcc make libssl-devel zlib-devel libffi-devel" >&2
echo " RHEL/CentOS: yum install gcc make openssl-devel zlib-devel libffi-devel" >&2
echo " Debian: apt install gcc make libssl-dev zlib1g-dev libffi-dev" >&2
exit 1
fi
# Extract and build
local build_dir
build_dir=$(mktemp -d)
trap "rm -rf '$build_dir'" EXIT
echo " [1/3] Extracting source..."
tar xf "$src_archive" -C "$build_dir"
local src_dir
src_dir=$(find "$build_dir" -maxdepth 1 -name 'Python-*' -type d | head -1)
echo " [2/3] Configuring (prefix=${LOCAL_PYTHON_DIR})..."
(
cd "$src_dir"
./configure \
--prefix="$LOCAL_PYTHON_DIR" \
--enable-optimizations \
--with-ensurepip=install \
--quiet 2>&1 | tail -5
)
echo " [3/3] Compiling and installing..."
(
cd "$src_dir"
make -j"$(nproc)" --quiet 2>&1 | tail -3
make install --quiet 2>&1 | tail -3
)
# Clean up build temp (trap handles this)
trap - EXIT
rm -rf "$build_dir"
PYTHON_BIN="$LOCAL_PYTHON_DIR/bin/python3"
if [ -x "$PYTHON_BIN" ]; then
local ver
ver=$("$PYTHON_BIN" --version 2>&1)
echo " Built successfully: $ver"
echo " Location: $PYTHON_BIN"
else
echo "ERROR: Python build failed" >&2
exit 1
fi
}
echo "=== Airflow DAG Monitor — Offline Install ===" echo "=== Airflow DAG Monitor — Offline Install ==="
echo "Source: $SCRIPT_DIR" echo "Source: $SCRIPT_DIR"
@ -154,9 +271,19 @@ echo "Install dir: $INSTALL_DIR"
echo "Service user: $SERVICE_USER" echo "Service user: $SERVICE_USER"
echo "" echo ""
# --- Step 1: Copy files if installing to a different directory --- # --- Step 1: Find or build Python ---
echo "[1/7] Checking Python..."
if ! find_python; then
sys_ver=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo "not found")
echo " System Python: ${sys_ver} (need ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+)"
echo " Building Python from included source..."
mkdir -p "$INSTALL_DIR"
build_python
fi
# --- Step 2: Copy files ---
if [ "$INSTALL_DIR" != "$SCRIPT_DIR" ]; then if [ "$INSTALL_DIR" != "$SCRIPT_DIR" ]; then
echo "[1/5] Copying files to $INSTALL_DIR..." echo "[2/7] Copying files to $INSTALL_DIR..."
mkdir -p "$INSTALL_DIR" mkdir -p "$INSTALL_DIR"
cp -r "$SCRIPT_DIR/airflow_monitor" "$INSTALL_DIR/" cp -r "$SCRIPT_DIR/airflow_monitor" "$INSTALL_DIR/"
cp -r "$SCRIPT_DIR/wheels" "$INSTALL_DIR/" cp -r "$SCRIPT_DIR/wheels" "$INSTALL_DIR/"
@ -165,21 +292,24 @@ if [ "$INSTALL_DIR" != "$SCRIPT_DIR" ]; then
cp "$SCRIPT_DIR/requirements.txt" "$INSTALL_DIR/" cp "$SCRIPT_DIR/requirements.txt" "$INSTALL_DIR/"
cp "$SCRIPT_DIR/airflow-monitor.service" "$INSTALL_DIR/" cp "$SCRIPT_DIR/airflow-monitor.service" "$INSTALL_DIR/"
cp "$SCRIPT_DIR/README.md" "$INSTALL_DIR/" cp "$SCRIPT_DIR/README.md" "$INSTALL_DIR/"
# Keep python-src for potential reinstalls
[ -d "$SCRIPT_DIR/python-src" ] && cp -r "$SCRIPT_DIR/python-src" "$INSTALL_DIR/"
echo " Done" echo " Done"
else else
echo "[1/5] Installing in place, skipping copy" echo "[2/7] Installing in place, skipping copy"
fi fi
# --- Step 2: Create virtual environment --- # --- Step 3: Create virtual environment ---
echo "[2/5] Creating Python virtual environment..." echo "[3/7] Creating Python virtual environment..."
if [ -d "$VENV_DIR" ]; then if [ -d "$VENV_DIR" ]; then
rm -rf "$VENV_DIR" rm -rf "$VENV_DIR"
fi fi
python3 -m venv "$VENV_DIR" "$PYTHON_BIN" -m venv "$VENV_DIR"
echo " Created: $VENV_DIR" echo " Created: $VENV_DIR"
echo " Python: $("$VENV_DIR/bin/python3" --version)"
# --- Step 3: Install dependencies from local wheels --- # --- Step 4: Install dependencies from local wheels ---
echo "[3/5] Installing dependencies from local wheels (offline)..." echo "[4/7] Installing dependencies from local wheels (offline)..."
"$VENV_DIR/bin/pip" install \ "$VENV_DIR/bin/pip" install \
--no-index \ --no-index \
--find-links "$INSTALL_DIR/wheels" \ --find-links "$INSTALL_DIR/wheels" \
@ -189,8 +319,8 @@ echo " Installed packages:"
"$VENV_DIR/bin/pip" list --format=columns 2>/dev/null | grep -vE '^(Package|---)' "$VENV_DIR/bin/pip" list --format=columns 2>/dev/null | grep -vE '^(Package|---)'
echo "" echo ""
# --- Step 4: Create required directories --- # --- Step 5: Create required directories ---
echo "[4/5] Creating directories..." echo "[5/7] Creating directories..."
STATE_DIR="/var/lib/airflow-monitor" STATE_DIR="/var/lib/airflow-monitor"
LOG_DIR="/var/log/airflow-monitor" LOG_DIR="/var/log/airflow-monitor"
@ -205,10 +335,9 @@ for dir in "$STATE_DIR" "$LOG_DIR"; do
fi fi
done done
# --- Step 5: Install systemd service --- # --- Step 6: Install systemd service ---
echo "[5/5] Installing systemd service..." echo "[6/7] Installing systemd service..."
# Patch service file with actual paths and user
SERVICE_FILE="$INSTALL_DIR/airflow-monitor.service" SERVICE_FILE="$INSTALL_DIR/airflow-monitor.service"
if [ -f "$SERVICE_FILE" ]; then if [ -f "$SERVICE_FILE" ]; then
TEMP_SERVICE=$(mktemp) TEMP_SERVICE=$(mktemp)
@ -216,7 +345,7 @@ if [ -f "$SERVICE_FILE" ]; then
-e "s|User=.*|User=${SERVICE_USER}|" \ -e "s|User=.*|User=${SERVICE_USER}|" \
-e "s|Group=.*|Group=$(id -gn "$SERVICE_USER")|" \ -e "s|Group=.*|Group=$(id -gn "$SERVICE_USER")|" \
-e "s|WorkingDirectory=.*|WorkingDirectory=${INSTALL_DIR}|" \ -e "s|WorkingDirectory=.*|WorkingDirectory=${INSTALL_DIR}|" \
-e "s|ExecStart=.*|ExecStart=${VENV_DIR}/bin/python -m airflow_monitor --config ${INSTALL_DIR}/config.yaml|" \ -e "s|ExecStart=.*|ExecStart=${VENV_DIR}/bin/python -m airflow_monitor --discover --config ${INSTALL_DIR}/config.yaml|" \
"$SERVICE_FILE" > "$TEMP_SERVICE" "$SERVICE_FILE" > "$TEMP_SERVICE"
sudo cp "$TEMP_SERVICE" /etc/systemd/system/${SERVICE_NAME}.service sudo cp "$TEMP_SERVICE" /etc/systemd/system/${SERVICE_NAME}.service
@ -227,8 +356,8 @@ else
echo " WARNING: service file not found, skipping" echo " WARNING: service file not found, skipping"
fi fi
# --- Step 6: Install Zabbix agent config --- # --- Step 7: Install Zabbix agent config ---
echo "[6/6] Installing Zabbix agent config..." echo "[7/7] Installing Zabbix agent config..."
ZABBIX_CONF_SRC="$INSTALL_DIR/zabbix/airflow-monitor.conf" ZABBIX_CONF_SRC="$INSTALL_DIR/zabbix/airflow-monitor.conf"
ZABBIX_CONF_DST="/etc/zabbix/zabbix_agentd.conf.d/airflow-monitor.conf" ZABBIX_CONF_DST="/etc/zabbix/zabbix_agentd.conf.d/airflow-monitor.conf"
@ -254,7 +383,6 @@ else
echo " WARNING: zabbix config not found" echo " WARNING: zabbix config not found"
fi fi
# Ensure data dir is readable by zabbix agent
sudo chmod 755 /var/lib/airflow-monitor 2>/dev/null || true sudo chmod 755 /var/lib/airflow-monitor 2>/dev/null || true
echo "" echo ""
@ -281,8 +409,8 @@ INSTALL_SCRIPT
chmod +x "$STAGE_DIR/install.sh" chmod +x "$STAGE_DIR/install.sh"
echo " Generated: install.sh" echo " Generated: install.sh"
# --- Step 4: Create archive --- # --- Step 5: Create archive ---
echo "[4/4] Creating tar.gz archive..." echo "[5/5] Creating tar.gz archive..."
ARCHIVE_NAME="airflow-monitor-${VERSION}.tar.gz" ARCHIVE_NAME="airflow-monitor-${VERSION}.tar.gz"
ARCHIVE_PATH="$DIST_DIR/$ARCHIVE_NAME" ARCHIVE_PATH="$DIST_DIR/$ARCHIVE_NAME"