ops: docker prune timer, kanboard backup/healthcheck, backup script fixes
docker-maintenance/ adds a systemd timer + prune.sh for the root LV that holds Docker's data-root and has filled to 100% before, risking ENOSPC corruption. The script sticks to the safe reclaim set (builder cache, dangling images, stopped containers) and deliberately avoids `-a` and volume pruning, which can destroy live data when run unattended. kanboard/backup.sh and healthcheck.sh bring Kanboard in line with the other services. seafile/ and vaultwarden/ backup scripts get fixes carried from the stability audit. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
69
kanboard/backup.sh
Executable file
69
kanboard/backup.sh
Executable file
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
# Kanboard backup — tier-0 hardening (kb#158, A2A-26, DESIGN-a2a-agents.md v2.1 §6c).
|
||||
# Mirrors the vaultwarden backup.sh pattern (same repo, ~/agap_git/vaultwarden/backup.sh):
|
||||
# scheduled dump -> /mnt/backups, retention of last 5, Zabbix freshness trapper.
|
||||
#
|
||||
# Runs every 3 days via alvis's user crontab (NOT root crontab like vaultwarden's --
|
||||
# /mnt/backups/kanboard was bootstrapped chown'd to alvis specifically so this backup,
|
||||
# like the rest of the kanboard tooling, needs no root/sudo at all. alvis is in the
|
||||
# `docker` group so `docker exec`/`docker cp` need no privilege escalation either).
|
||||
#
|
||||
# DB dump method: kanboard's container has no sqlite3 CLI and no PHP `sqlite3`
|
||||
# extension (only pdo_sqlite) -- checked directly (kb#158). Instead we run SQLite's
|
||||
# own `VACUUM INTO` via PDO, which is SQLite's supported way to take an atomic,
|
||||
# consistent online snapshot of a live database (safe against concurrent writers,
|
||||
# same safety property `vaultwarden backup` gives us for that service).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BACKUP_DIR="/mnt/backups/kanboard"
|
||||
ZABBIX_TOKEN_FILE="/home/alvis/.zabbix_token"
|
||||
ZABBIX_URL="http://192.168.1.4:81/api_jsonrpc.php"
|
||||
ZABBIX_ITEM_ID="70605" # kanboard.backup.ts on host AgapHost (10776)
|
||||
|
||||
DATE=$(date '+%Y%m%d-%H%M')
|
||||
DEST="$BACKUP_DIR/$DATE"
|
||||
TMP_NAME="backup_${DATE}.sqlite"
|
||||
|
||||
mkdir -p "$DEST"
|
||||
|
||||
# Online, consistent snapshot via SQLite's VACUUM INTO (PDO sqlite driver is present
|
||||
# in the image; the sqlite3 CLI/extension is not, so this replaces the vaultwarden
|
||||
# `docker exec vaultwarden /vaultwarden backup` equivalent for this service).
|
||||
docker exec kanboard php -r '
|
||||
$db = new PDO("sqlite:/var/www/app/data/db.sqlite");
|
||||
$db->exec("VACUUM INTO \"/var/www/app/data/'"$TMP_NAME"'\"");
|
||||
'
|
||||
|
||||
# Pull the snapshot out to the host, then remove the temp copy from the live data dir
|
||||
# (mirrors vaultwarden's "move the file out of DATA_DIR" step).
|
||||
docker cp "kanboard:/var/www/app/data/$TMP_NAME" "$DEST/db.sqlite"
|
||||
docker exec kanboard rm -f "/var/www/app/data/$TMP_NAME"
|
||||
|
||||
# Plugins volume (PLUGIN_INSTALLER=true means plugins can be installed at runtime,
|
||||
# not just baked into the image) -- back it up too so a restore doesn't silently
|
||||
# drop installed plugins.
|
||||
docker run --rm --user 1000:1000 -v kanboard_plugins:/plugins:ro -v "$DEST":/dest alpine \
|
||||
sh -c 'cd /plugins && tar -czf /dest/plugins.tar.gz . 2>/dev/null || true'
|
||||
|
||||
echo "$(date): Backup complete: $DEST"
|
||||
ls -la "$DEST/"
|
||||
|
||||
# Notify Zabbix (trapper item kanboard.backup.ts, unixtime) -- pushes a real epoch
|
||||
# timestamp, unlike vaultwarden.backup.ts which (kb#158 finding) pushes a formatted
|
||||
# date STRING into a numeric item and has therefore never recorded a valid value.
|
||||
if [[ -f "$ZABBIX_TOKEN_FILE" ]]; then
|
||||
ZABBIX_TOKEN=$(cat "$ZABBIX_TOKEN_FILE")
|
||||
NOW_EPOCH=$(date '+%s')
|
||||
env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
||||
curl -s -X POST "$ZABBIX_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $ZABBIX_TOKEN" \
|
||||
-d "{\"jsonrpc\":\"2.0\",\"method\":\"history.push\",\"id\":1,\"params\":{\"itemid\":\"$ZABBIX_ITEM_ID\",\"value\":$NOW_EPOCH}}" > /dev/null \
|
||||
&& echo "Zabbix notified (kanboard.backup.ts=$NOW_EPOCH)."
|
||||
else
|
||||
echo "WARNING: $ZABBIX_TOKEN_FILE not found -- skipped Zabbix freshness push." >&2
|
||||
fi
|
||||
|
||||
# Rotate: keep last 5 backups
|
||||
ls -1dt "$BACKUP_DIR"/[0-9]*-[0-9]* 2>/dev/null | tail -n +6 | xargs -r rm -rf
|
||||
92
kanboard/healthcheck.sh
Executable file
92
kanboard/healthcheck.sh
Executable file
@@ -0,0 +1,92 @@
|
||||
#!/bin/bash
|
||||
# Kanboard service + JSON-RPC API health -- tier-0 hardening (kb#158, A2A-26,
|
||||
# DESIGN-a2a-agents.md v2.1 §6c: "Zabbix monitoring of the service and API").
|
||||
#
|
||||
# Runs every 2 minutes via alvis's user crontab (no root needed -- alvis is in the
|
||||
# `docker` group). Pushes two Zabbix trapper items on host AgapHost:
|
||||
# kanboard.service.up -- 1 if the `kanboard` container is running AND its own
|
||||
# Docker healthcheck reports "healthy", else 0
|
||||
# kanboard.jsonrpc.up -- 1 if an authenticated JSON-RPC call round-trips
|
||||
# correctly, else 0
|
||||
# Both are pushed every run (unlike backup.sh, which only pushes on success) so a
|
||||
# transition to "down" is reported immediately rather than waiting for nodata() --
|
||||
# the triggers additionally use nodata(...,10m) as a backstop in case this script
|
||||
# itself stops running.
|
||||
|
||||
set -uo pipefail # no -e: we want to push a "0" and continue, not abort, on failure
|
||||
|
||||
ZABBIX_TOKEN_FILE="/home/alvis/.zabbix_token"
|
||||
KANBOARD_TOKEN_FILE="/home/alvis/.kanboard_token"
|
||||
ZABBIX_URL="http://192.168.1.4:81/api_jsonrpc.php"
|
||||
SERVICE_ITEM_ID="70606" # kanboard.service.up
|
||||
JSONRPC_ITEM_ID="70607" # kanboard.jsonrpc.up
|
||||
|
||||
# kb#188: tier-0 fabric container up/down. Piggybacks on this same 2-minute cron
|
||||
# slot (no new cron entry) -- container:itemid map, plain `docker inspect
|
||||
# .State.Running` since most of these have no HEALTHCHECK defined (only `adolf`
|
||||
# does; checking bare Running is what's available uniformly here). Pushed
|
||||
# unconditionally like the two items above, with nodata(...,10m) as trigger backstop.
|
||||
declare -A FABRIC_ITEMS=(
|
||||
["litellm"]="70625"
|
||||
["litellm-db"]="70626"
|
||||
["adolf"]="70627"
|
||||
["adolf-llm"]="70628"
|
||||
["hindsight"]="70629"
|
||||
["hindsight-llm"]="70630"
|
||||
["tei-reranker"]="70631"
|
||||
["ollama"]="70632"
|
||||
["kanboard-mcp-kanboard-mcp-1"]="70633"
|
||||
["kanboard-mcp-adolf"]="70634"
|
||||
["agap-mcp-agap-mcp-1"]="70635"
|
||||
["fabric-keeper"]="70636"
|
||||
)
|
||||
|
||||
# --- 1. Container health ---
|
||||
HEALTH=$(docker inspect --format '{{.State.Health.Status}}' kanboard 2>/dev/null)
|
||||
if [[ "$HEALTH" == "healthy" ]]; then
|
||||
SERVICE_UP=1
|
||||
else
|
||||
SERVICE_UP=0
|
||||
fi
|
||||
|
||||
# --- 2. JSON-RPC API health (authenticated round-trip, not just a TCP/HTTP check) ---
|
||||
JSONRPC_UP=0
|
||||
if [[ -f "$KANBOARD_TOKEN_FILE" ]]; then
|
||||
KANBOARD_TOKEN=$(cat "$KANBOARD_TOKEN_FILE")
|
||||
RESP=$(env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
||||
curl -s --max-time 10 -u "jsonrpc:$KANBOARD_TOKEN" -X POST http://127.0.0.1:4800/jsonrpc.php \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"jsonrpc":"2.0","method":"getVersion","id":1}' 2>/dev/null)
|
||||
if echo "$RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if 'result' in d else 1)" 2>/dev/null; then
|
||||
JSONRPC_UP=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- 3. Fabric tier-0 container up/down (kb#188) ---
|
||||
FABRIC_PUSH_PARAMS="{\"itemid\":\"$SERVICE_ITEM_ID\",\"value\":$SERVICE_UP},{\"itemid\":\"$JSONRPC_ITEM_ID\",\"value\":$JSONRPC_UP}"
|
||||
FABRIC_SUMMARY=""
|
||||
for container in "${!FABRIC_ITEMS[@]}"; do
|
||||
itemid="${FABRIC_ITEMS[$container]}"
|
||||
running=$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$running" == "true" ]]; then
|
||||
up=1
|
||||
else
|
||||
up=0
|
||||
fi
|
||||
FABRIC_PUSH_PARAMS="$FABRIC_PUSH_PARAMS,{\"itemid\":\"$itemid\",\"value\":$up}"
|
||||
FABRIC_SUMMARY="$FABRIC_SUMMARY $container=$up"
|
||||
done
|
||||
|
||||
# --- 4. Push everything to Zabbix in one batch, unconditionally (down is a real value, not a gap) ---
|
||||
if [[ -f "$ZABBIX_TOKEN_FILE" ]]; then
|
||||
ZABBIX_TOKEN=$(cat "$ZABBIX_TOKEN_FILE")
|
||||
env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
||||
curl -s --max-time 10 -X POST "$ZABBIX_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $ZABBIX_TOKEN" \
|
||||
-d "{\"jsonrpc\":\"2.0\",\"method\":\"history.push\",\"id\":1,\"params\":[$FABRIC_PUSH_PARAMS]}" > /dev/null
|
||||
else
|
||||
echo "WARNING: $ZABBIX_TOKEN_FILE not found -- skipped Zabbix push." >&2
|
||||
fi
|
||||
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') service_up=$SERVICE_UP jsonrpc_up=$JSONRPC_UP$FABRIC_SUMMARY"
|
||||
Reference in New Issue
Block a user