feat: add route_fallbacks metric to Prometheus and Zabbix monitoring

- Prometheus: new pulse_route_fallbacks_total counter - Zabbix: extend response with route_requests_total, route_fallbacks_total, uptime_seconds - Docs: update Prometheus metrics table and Zabbix response example
docs: complete API reference — add routing rules, health, node structures, monitoring details
2026-06-25 23:27:12 +07:00 · 2026-06-25 23:14:19 +07:00
4 changed files with 218 additions and 16 deletions
@@ -17,9 +17,36 @@ Roles: `admin` (full access), `viewer` (read-only nodes/metrics).
 |--------|------|------|-------------|
 | `GET` | `/api/route` | – | Route a call. Query: `caller_id`, `dest`, `ingress_trunk` |
 | `PUT` | `/api/route/mode` | admin | Set balancing mode: `{"mode":"best\|weighted_random"}` — см. [Routing](routing.md#balancing-modes) |
-| `GET` | `/api/health` | – | Stats: total nodes, healthy, requests, uptime |
-| `GET` | `/api/health/live` | – | Liveness probe (always 200) |
-| `GET` | `/api/health/ready` | – | Readiness probe (NATS + metrics) |
+| `GET` | `/api/health` | – | Full health status with connections and stats |
+| `GET` | `/api/health/live` | – | Liveness probe — always 200 `{"status":"alive"}` |
+| `GET` | `/api/health/ready` | – | Readiness — 200 `{"status":"ready"}` или 503 `{"reason":"nats_disconnected"}` |
+
+### Health Response
+
+```json
+{
+  "status": "ok",
+  "version": "1.0.0",
+  "connections": {
+    "nats": "connected|disconnected",
+    "esl": {
+      "status": "disabled|connected|disconnected",
+      "host": "127.0.0.1",
+      "uptime_seconds": 3600,
+      "reconnects": 0,
+      "gateway_ops": 42,
+      "events_recv": 15000
+    }
+  },
+  "stats": {
+    "total_nodes": 3,
+    "healthy_nodes": 2,
+    "route_requests": 5000,
+    "route_fallbacks": 10,
+    "uptime_seconds": 7200
+  }
+}
+```

 ### Route Response (success)

@@ -29,6 +56,13 @@ Roles: `admin` (full access), `viewer` (read-only nodes/metrics).

 `matched_rule` — имя сработавшего routing rule, либо `"none"` если применился глобальный scoring.

+### Route Mode Response
+
+`PUT /api/route/mode` возвращает текущий режим:
+```json
+{ "mode": "best" }
+```
+
 ### Route Response (fallback — all nodes unhealthy)

 ```json
@@ -36,14 +70,19 @@ Roles: `admin` (full access), `viewer` (read-only nodes/metrics).
  "fallback": true,
  "sip_gateway": "sip:operator.lan:5060",
  "reason": "all_nodes_unhealthy",
-  "nodes": [...]
+  "matched_rule": "my-pool-rule",
+  "nodes": [
+    { "id": "pbx-01", "score": -100, "reason": "disabled", "status": "disabled" },
+    { "id": "pbx-02", "score": -1, "reason": "stale", "status": "stale" },
+    { "id": "pbx-03", "score": 70, "reason": "", "status": "healthy" }
+  ]
 }
 ```

 ### Route Response (no nodes registered)

 ```json
-{ "error": "no_nodes_registered" }
+{ "error": "no_nodes_registered", "matched_rule": "none" }
 ```

 ## Nodes
@@ -52,7 +91,42 @@ Roles: `admin` (full access), `viewer` (read-only nodes/metrics).
 |--------|------|------|-------------|
 | `GET` | `/api/nodes` | admin/viewer | List all nodes with scores and state |
 | `GET` | `/api/nodes/{id}/metrics` | admin/viewer | Metrics history (ring buffer, ~30 min) |
-| `PUT` | `/api/nodes/{id}/toggle` | admin | `{"disabled":true, "reason":"..."}` |
+| `PUT` | `/api/nodes/{id}/toggle` | admin | `{"disabled":true, "reason":"..."}` — returns updated `NodeInfo` |
+
+### NodeInfo (GET /api/nodes)
+
+```json
+{
+  "node_id": "pbx-01",
+  "ts": 1719000000,
+  "status": "ok",
+  "active_calls": 42,
+  "max_calls": 250,
+  "idle_cpu": 35.0,
+  "load_avg": 0.85,
+  "call_failure_rate": 0.5,
+  "disabled": false,
+  "disabled_reason": "",
+  "score": 88.0,
+  "lethal_reason": "",
+  "is_stale": false,
+  "seconds_ago": 3.0
+}
+```
+
+### Node Metrics (GET /api/nodes/{id}/metrics)
+
+История метрик ноды из ring buffer (~30 мин, шаг 5 с):
+
+```json
+{
+  "node_id": "pbx-01",
+  "snapshots": [
+    { "active_calls": 42, "load_avg": 0.85, "score": 88.0, ... }
+  ],
+  "count": 360
+}
+```

 ## Trunks

@@ -88,16 +162,32 @@ Trunk types:
 | `balance` | Destination trunk (bound to a Node) | 1..N |
 | `fallback` | Operator fallback (when all balance score < 0) | exactly 1 |

+## Routing Rules
+
+| Method | Path | Role | Description |
+|--------|------|------|-------------|
+| `GET` | `/api/routing/rules` | admin/viewer | Список правил с счётчиками hits |
+| `POST` | `/api/routing/rules` | admin | Создать правило |
+| `PUT` | `/api/routing/rules/{id}` | admin | Обновить правило |
+| `DELETE` | `/api/routing/rules/{id}` | admin | Удалить правило |
+| `PUT` | `/api/routing/rules/{id}/toggle` | admin | Вкл/выкл: `{"enabled":true}` |
+| `PUT` | `/api/routing/rules/reorder` | admin | Новый порядок: `["rr-abc", "rr-def", ...]` |
+
+Поля правила — см. [docs/routing.md](routing.md#routing-rules). Ответы содержат полную модель правила.
+
 ## Users (admin only)

 | Method | Path | Description |
 |--------|------|-------------|
-| `GET` | `/api/users` | List users |
-| `POST` | `/api/users` | Create user |
-| `PUT` | `/api/users/{id}` | Update user |
-| `DELETE` | `/api/users/{id}` | Delete user |
+| `GET` | `/api/users` | List users (без password_hash) |
+| `POST` | `/api/users` | `{"username":"...","password":"...","role":"admin\|viewer"}` |
+| `PUT` | `/api/users/{id}` | Partial update: `{"username":"...","role":"viewer"}` |
+| `DELETE` | `/api/users/{id}` | Удалить (204 No Content) |

-Note: `user-01` (primary admin) cannot be deleted or demoted to viewer.
+Защиты:
+- `user-01` нельзя удалить или понизить до viewer
+- Нельзя удалить последнего администратора
+- GET-ответ исключает поле `password_hash`

 ## Monitoring (API-key auth)

@@ -105,14 +195,67 @@ Use `X-API-Key` header (not JWT).

 | Method | Path | Description |
 |--------|------|-------------|
-| `GET` | `/api/monitoring/zabbix` | JSON for Zabbix LLD + items |
-| `GET` | `/api/monitoring/prometheus` | `text/plain` metrics for Prometheus |
+| `GET` | `/api/monitoring/zabbix` | JSON для Zabbix LLD |
+| `GET` | `/api/monitoring/prometheus` | `text/plain` метрики для Prometheus |
+
+### Zabbix Response
+
+```json
+{
+  "data": [
+    {
+      "{#NODE_ID}": "pbx-01",
+      "{#STATUS}": "ok",
+      "{#ACTIVE_CALLS}": 42,
+      "{#MAX_CALLS}": 250,
+      "{#IDLE_CPU}": 35.0,
+      "{#LOAD_AVG}": 0.85,
+      "{#CALL_FAILURE_RATE}": 0.5,
+      "{#SCORE}": 88.0,
+      "{#DISABLED}": false,
+      "{#IS_STALE}": false,
+      "{#SECONDS_AGO}": 3.0
+    }
+  ],
+  "route_requests_total": 5000,
+  "route_fallbacks_total": 42,
+  "uptime_seconds": 7200
+}
+```
+
+### Prometheus Metrics
+
+| Метрика | Type | Labels |
+|---------|------|--------|
+| `pulse_route_requests_total` | counter | – |
+| `pulse_route_fallbacks_total` | counter | – |
+| `pulse_nodes_total` | gauge | – |
+| `pulse_nodes_healthy` | gauge | – |
+| `pulse_uptime_seconds` | gauge | – |
+| `pulse_node_active_calls` | gauge | `node` |
+| `pulse_node_max_calls` | gauge | `node` |
+| `pulse_node_idle_cpu` | gauge | `node` |
+| `pulse_node_load_avg` | gauge | `node` |
+| `pulse_node_call_failure_rate` | gauge | `node` |
+| `pulse_node_score` | gauge | `node` |
+| `pulse_node_seconds_ago` | gauge | `node` |
+| `pulse_node_disabled` | gauge | `node` (0/1) |
+| `pulse_node_stale` | gauge | `node` (0/1) |
+| `pulse_node_status_ok` | gauge | `node` (0/1) |

 ## WebSocket

 | Method | Path | Description |
 |--------|------|-------------|
-| `WS` | `/ws/metrics` | Real-time metrics stream for frontend (JWT in query `?token=`) |
+| `WS` | `/ws/metrics` | Real-time metrics (JWT в query `?token=`) |
+
+Типы сообщений (поле `type`):
+
+| Тип | Payload | Описание |
+|-----|---------|----------|
+| `nodes_update` | `[]NodeInfo` | Обновление метрик всех нод (каждые 5 с) |
+| `balancer_update` | `BalancerInfo` | Состояние балансировщика (uptime, mode, ESL, NATS) |
+| `gateway_update` | `{"gateway","trunk_id","status"}` | Статус FS-gateway (up/down) |

 ## Examples

@@ -135,6 +278,37 @@ curl -X PUT http://localhost:8080/api/nodes/pbx-01/toggle \
  -H 'Content-Type: application/json' \
  -d '{"disabled":true,"reason":"maintenance"}'

+# Route mode (admin)
+curl -X PUT http://localhost:8080/api/route/mode \
+  -H "Authorization: Bearer $TOKEN" \
+  -H 'Content-Type: application/json' \
+  -d '{"mode":"weighted_random"}'
+# → {"mode":"weighted_random"}
+
+# Routing rules (admin)
+curl -X POST http://localhost:8080/api/routing/rules \
+  -H "Authorization: Bearer $TOKEN" \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name":"VIP pool",
+    "priority":10,
+    "match_caller_id":"^7495",
+    "action_type":"pool",
+    "action_pool_ids":["pbx-01","pbx-02"]
+  }'
+
+# Toggle rule (admin)
+curl -X PUT http://localhost:8080/api/routing/rules/rr-abc123/toggle \
+  -H "Authorization: Bearer $TOKEN" \
+  -H 'Content-Type: application/json' \
+  -d '{"enabled":false}'
+
+# Reorder rules (admin)
+curl -X PUT http://localhost:8080/api/routing/rules/reorder \
+  -H "Authorization: Bearer $TOKEN" \
+  -H 'Content-Type: application/json' \
+  -d '["rr-def456", "rr-abc123"]'
+
 # Prometheus metrics (API-key)
 curl -H "X-API-Key: $(jq -r '.monitoring_api_key' data/config.json)" \
  http://localhost:8080/api/monitoring/prometheus
@@ -90,6 +90,21 @@ Authorization: Bearer <admin-jwt>

 Допустимые значения: `"best"`, `"weighted_random"`. API-эндпоинт сразу применяет режим и сохраняет в `config.json`.

+## Routing Rules API
+
+Правила маршрутизации CRUD — `/api/routing/rules*`. Все эндпоинты требуют JWT (admin).
+
+| Метод | Путь | Описание |
+|-------|------|----------|
+| `GET` | `/api/routing/rules` | Список всех правил (с hits) |
+| `POST` | `/api/routing/rules` | Создать новое правило |
+| `PUT` | `/api/routing/rules/{id}` | Обновить правило (частично) |
+| `DELETE` | `/api/routing/rules/{id}` | Удалить правило |
+| `PUT` | `/api/routing/rules/{id}/toggle` | Вкл/выкл: `{"enabled":true}` |
+| `PUT` | `/api/routing/rules/reorder` | Новый порядок: `["rr-abc", "rr-def"]` — переназначает Priority |
+
+Правила хранятся в `data/routes.json` и применяются в порядке `Priority` (first-match-wins).
+
 ## Ответ /api/route

 В ответе маршрутизации появилось поле `matched_rule` — имя сработавшего правила (или `"none"` если правило не применилось):
@@ -29,7 +29,13 @@ func (a *API) handleZabbix(w http.ResponseWriter, r *http.Request) {
 		})
 	}

-	writeJSON(w, http.StatusOK, models.ZabbixResponse{Data: data})
+	stats := a.engine.GetHealthStats()
+	writeJSON(w, http.StatusOK, models.ZabbixResponse{
+		Data:           data,
+		RouteRequests:  stats.RouteRequests,
+		RouteFallbacks: stats.RouteFallbacks,
+		UptimeSeconds:  stats.UptimeSeconds,
+	})
 }

 // handlePrometheus — GET /api/monitoring/prometheus — text/plain метрики для Prometheus.
@@ -52,6 +58,10 @@ func (a *API) handlePrometheus(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprintf(&sb, "# TYPE pulse_nodes_healthy gauge\n")
 	fmt.Fprintf(&sb, "pulse_nodes_healthy %d\n", stats.HealthyNodes)

+	fmt.Fprintf(&sb, "# HELP pulse_route_fallbacks_total Total number of fallback route requests.\n")
+	fmt.Fprintf(&sb, "# TYPE pulse_route_fallbacks_total counter\n")
+	fmt.Fprintf(&sb, "pulse_route_fallbacks_total %d\n", stats.RouteFallbacks)
+
 	fmt.Fprintf(&sb, "# HELP pulse_uptime_seconds Uptime in seconds.\n")
 	fmt.Fprintf(&sb, "# TYPE pulse_uptime_seconds gauge\n")
 	fmt.Fprintf(&sb, "pulse_uptime_seconds %d\n", stats.UptimeSeconds)
@@ -242,6 +242,9 @@ type ZabbixNode struct {
 // ZabbixResponse — ответ на запрос Zabbix LLD.
 type ZabbixResponse struct {
 	Data             []ZabbixNode `json:"data"`
+	RouteRequests    int64        `json:"route_requests_total"`
+	RouteFallbacks   int64        `json:"route_fallbacks_total"`
+	UptimeSeconds    int64        `json:"uptime_seconds"`
 }

 // --- Balancer ---
Author	SHA1	Message	Date
Maksim Totmin	727fd1195a	feat: add route_fallbacks metric to Prometheus and Zabbix monitoring - Prometheus: new pulse_route_fallbacks_total counter - Zabbix: extend response with route_requests_total, route_fallbacks_total, uptime_seconds - Docs: update Prometheus metrics table and Zabbix response example	2026-06-25 23:27:12 +07:00
Maksim Totmin	2df40d5dbe	docs: complete API reference — add routing rules, health, node structures, monitoring details	2026-06-25 23:14:19 +07:00