ops: docker prune timer, kanboard backup/healthcheck, backup script fixes
docker-maintenance/ adds a systemd timer + prune.sh for the root LV that holds Docker's data-root and has filled to 100% before, risking ENOSPC corruption. The script sticks to the safe reclaim set (builder cache, dangling images, stopped containers) and deliberately avoids `-a` and volume pruning, which can destroy live data when run unattended. kanboard/backup.sh and healthcheck.sh bring Kanboard in line with the other services. seafile/ and vaultwarden/ backup scripts get fixes carried from the stability audit. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
92
kanboard/healthcheck.sh
Executable file
92
kanboard/healthcheck.sh
Executable file
@@ -0,0 +1,92 @@
|
||||
#!/bin/bash
|
||||
# Kanboard service + JSON-RPC API health -- tier-0 hardening (kb#158, A2A-26,
|
||||
# DESIGN-a2a-agents.md v2.1 §6c: "Zabbix monitoring of the service and API").
|
||||
#
|
||||
# Runs every 2 minutes via alvis's user crontab (no root needed -- alvis is in the
|
||||
# `docker` group). Pushes two Zabbix trapper items on host AgapHost:
|
||||
# kanboard.service.up -- 1 if the `kanboard` container is running AND its own
|
||||
# Docker healthcheck reports "healthy", else 0
|
||||
# kanboard.jsonrpc.up -- 1 if an authenticated JSON-RPC call round-trips
|
||||
# correctly, else 0
|
||||
# Both are pushed every run (unlike backup.sh, which only pushes on success) so a
|
||||
# transition to "down" is reported immediately rather than waiting for nodata() --
|
||||
# the triggers additionally use nodata(...,10m) as a backstop in case this script
|
||||
# itself stops running.
|
||||
|
||||
set -uo pipefail # no -e: we want to push a "0" and continue, not abort, on failure
|
||||
|
||||
ZABBIX_TOKEN_FILE="/home/alvis/.zabbix_token"
|
||||
KANBOARD_TOKEN_FILE="/home/alvis/.kanboard_token"
|
||||
ZABBIX_URL="http://192.168.1.4:81/api_jsonrpc.php"
|
||||
SERVICE_ITEM_ID="70606" # kanboard.service.up
|
||||
JSONRPC_ITEM_ID="70607" # kanboard.jsonrpc.up
|
||||
|
||||
# kb#188: tier-0 fabric container up/down. Piggybacks on this same 2-minute cron
|
||||
# slot (no new cron entry) -- container:itemid map, plain `docker inspect
|
||||
# .State.Running` since most of these have no HEALTHCHECK defined (only `adolf`
|
||||
# does; checking bare Running is what's available uniformly here). Pushed
|
||||
# unconditionally like the two items above, with nodata(...,10m) as trigger backstop.
|
||||
declare -A FABRIC_ITEMS=(
|
||||
["litellm"]="70625"
|
||||
["litellm-db"]="70626"
|
||||
["adolf"]="70627"
|
||||
["adolf-llm"]="70628"
|
||||
["hindsight"]="70629"
|
||||
["hindsight-llm"]="70630"
|
||||
["tei-reranker"]="70631"
|
||||
["ollama"]="70632"
|
||||
["kanboard-mcp-kanboard-mcp-1"]="70633"
|
||||
["kanboard-mcp-adolf"]="70634"
|
||||
["agap-mcp-agap-mcp-1"]="70635"
|
||||
["fabric-keeper"]="70636"
|
||||
)
|
||||
|
||||
# --- 1. Container health ---
|
||||
HEALTH=$(docker inspect --format '{{.State.Health.Status}}' kanboard 2>/dev/null)
|
||||
if [[ "$HEALTH" == "healthy" ]]; then
|
||||
SERVICE_UP=1
|
||||
else
|
||||
SERVICE_UP=0
|
||||
fi
|
||||
|
||||
# --- 2. JSON-RPC API health (authenticated round-trip, not just a TCP/HTTP check) ---
|
||||
JSONRPC_UP=0
|
||||
if [[ -f "$KANBOARD_TOKEN_FILE" ]]; then
|
||||
KANBOARD_TOKEN=$(cat "$KANBOARD_TOKEN_FILE")
|
||||
RESP=$(env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
||||
curl -s --max-time 10 -u "jsonrpc:$KANBOARD_TOKEN" -X POST http://127.0.0.1:4800/jsonrpc.php \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"jsonrpc":"2.0","method":"getVersion","id":1}' 2>/dev/null)
|
||||
if echo "$RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if 'result' in d else 1)" 2>/dev/null; then
|
||||
JSONRPC_UP=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- 3. Fabric tier-0 container up/down (kb#188) ---
|
||||
FABRIC_PUSH_PARAMS="{\"itemid\":\"$SERVICE_ITEM_ID\",\"value\":$SERVICE_UP},{\"itemid\":\"$JSONRPC_ITEM_ID\",\"value\":$JSONRPC_UP}"
|
||||
FABRIC_SUMMARY=""
|
||||
for container in "${!FABRIC_ITEMS[@]}"; do
|
||||
itemid="${FABRIC_ITEMS[$container]}"
|
||||
running=$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$running" == "true" ]]; then
|
||||
up=1
|
||||
else
|
||||
up=0
|
||||
fi
|
||||
FABRIC_PUSH_PARAMS="$FABRIC_PUSH_PARAMS,{\"itemid\":\"$itemid\",\"value\":$up}"
|
||||
FABRIC_SUMMARY="$FABRIC_SUMMARY $container=$up"
|
||||
done
|
||||
|
||||
# --- 4. Push everything to Zabbix in one batch, unconditionally (down is a real value, not a gap) ---
|
||||
if [[ -f "$ZABBIX_TOKEN_FILE" ]]; then
|
||||
ZABBIX_TOKEN=$(cat "$ZABBIX_TOKEN_FILE")
|
||||
env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
||||
curl -s --max-time 10 -X POST "$ZABBIX_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $ZABBIX_TOKEN" \
|
||||
-d "{\"jsonrpc\":\"2.0\",\"method\":\"history.push\",\"id\":1,\"params\":[$FABRIC_PUSH_PARAMS]}" > /dev/null
|
||||
else
|
||||
echo "WARNING: $ZABBIX_TOKEN_FILE not found -- skipped Zabbix push." >&2
|
||||
fi
|
||||
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') service_up=$SERVICE_UP jsonrpc_up=$JSONRPC_UP$FABRIC_SUMMARY"
|
||||
Reference in New Issue
Block a user