ops: docker prune timer, kanboard backup/healthcheck, backup script fixes
docker-maintenance/ adds a systemd timer + prune.sh for the root LV that holds Docker's data-root and has filled to 100% before, risking ENOSPC corruption. The script sticks to the safe reclaim set (builder cache, dangling images, stopped containers) and deliberately avoids `-a` and volume pruning, which can destroy live data when run unattended. kanboard/backup.sh and healthcheck.sh bring Kanboard in line with the other services. seafile/ and vaultwarden/ backup scripts get fixes carried from the stability audit. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
111
docker-maintenance/prune.sh
Executable file
111
docker-maintenance/prune.sh
Executable file
@@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
# Docker growth guard — recurring safe prune (kb#184, stability audit 2026-07-24).
|
||||
#
|
||||
# BACKGROUND: root LV (/) holds Docker's data-root. On 2026-07-24 it filled to
|
||||
# 1.5G free / 100% used, risking ENOSPC corruption across every service on Agap.
|
||||
# A one-off safe reclaim (builder prune + dangling images + stopped containers)
|
||||
# took it from 1.5G to 25G free. This script is the recurring guard so it can't
|
||||
# silently refill between now and the next manual audit.
|
||||
#
|
||||
# SAFE SCOPE ONLY (learned from the 2026-07-24 incident — do not expand without
|
||||
# a human decision, see README.md in this directory):
|
||||
# - docker builder prune -f (build cache; always safe, fully rebuildable)
|
||||
# - docker container prune -f (stopped/exited containers only)
|
||||
# - docker image prune -f (DANGLING images only — untagged, no -a)
|
||||
#
|
||||
# EXPLICITLY NOT DONE HERE (human judgment call, see README.md):
|
||||
# - docker image prune -a (removes TAGGED but currently-unused images —
|
||||
# could remove images a human wants kept)
|
||||
# - docker volume prune (destroys live data if a volume is unmounted
|
||||
# but still wanted — never run unattended)
|
||||
#
|
||||
# Pushes freed-space and post-prune build-cache-reclaimable metrics to Zabbix
|
||||
# (trapper items on host AgapHost, hostid 10776). Zabbix lives on lizacer
|
||||
# (192.168.1.4:81), not Agap.
|
||||
#
|
||||
# Run manually to test: bash prune.sh --dry-run
|
||||
# Scheduled via cron (see README.md for the exact line — NOT installed by this
|
||||
# script; installation is a human/handoff step, not something this script does).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_FILE="/home/alvis/agap_git/docker-maintenance/prune.log"
|
||||
ZABBIX_TOKEN_FILE="/home/alvis/.zabbix_token"
|
||||
ZABBIX_URL="http://192.168.1.4:81/api_jsonrpc.php"
|
||||
ZABBIX_ITEM_ID="70624" # docker.buildcache.reclaimable.bytes on host AgapHost (10776)
|
||||
|
||||
DRY_RUN=0
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
DRY_RUN=1
|
||||
fi
|
||||
|
||||
log() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') $*"
|
||||
}
|
||||
|
||||
{
|
||||
log "=== docker-maintenance prune.sh start (dry_run=$DRY_RUN) ==="
|
||||
|
||||
log "--- before ---"
|
||||
df -h / | tail -n +2
|
||||
docker system df
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
log "DRY RUN: would run: docker builder prune -f"
|
||||
log "DRY RUN: would run: docker container prune -f"
|
||||
log "DRY RUN: would run: docker image prune -f (dangling only)"
|
||||
else
|
||||
log "Running: docker builder prune -f"
|
||||
docker builder prune -f || true
|
||||
|
||||
log "Running: docker container prune -f"
|
||||
docker container prune -f || true
|
||||
|
||||
log "Running: docker image prune -f (dangling only, no -a)"
|
||||
docker image prune -f || true
|
||||
fi
|
||||
|
||||
log "--- after ---"
|
||||
df -h / | tail -n +2
|
||||
DF_JSON=$(docker system df --format '{{json .}}')
|
||||
echo "$DF_JSON"
|
||||
|
||||
# Extract build-cache reclaimable bytes for Zabbix. `docker system df` doesn't
|
||||
# give bytes directly (only human-readable strings like "14.49GB"), so use
|
||||
# `docker system df -v` reclaimable percentage isn't reliable either; instead
|
||||
# compute it from `docker builder du`-equivalent: sum of build cache records
|
||||
# not marked Shared/in-use is nontrivial to script robustly, so fall back to
|
||||
# parsing the "Build Cache" line's RECLAIMABLE column via docker system df.
|
||||
RECLAIM_STR=$(docker system df | awk '/^Build Cache/ {print $NF}')
|
||||
# RECLAIM_STR looks like "14.49GB" or "0B" -- convert to bytes (approx, GB/MB/KB = *1000^n, matching docker's own decimal convention)
|
||||
RECLAIM_BYTES=$(python3 -c "
|
||||
import re, sys
|
||||
s = '$RECLAIM_STR'
|
||||
m = re.match(r'([0-9.]+)\s*([KMGTP]?B)', s)
|
||||
if not m:
|
||||
print(0)
|
||||
else:
|
||||
val, unit = float(m.group(1)), m.group(2)
|
||||
mult = {'B':1,'KB':1000,'MB':1000**2,'GB':1000**3,'TB':1000**4,'PB':1000**5}[unit]
|
||||
print(int(val*mult))
|
||||
" 2>/dev/null || echo 0)
|
||||
|
||||
log "Build cache reclaimable: $RECLAIM_STR (~$RECLAIM_BYTES bytes)"
|
||||
|
||||
if [[ -f "$ZABBIX_TOKEN_FILE" && "$DRY_RUN" -eq 0 ]]; then
|
||||
ZABBIX_TOKEN=$(cat "$ZABBIX_TOKEN_FILE")
|
||||
env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
||||
curl -s -X POST "$ZABBIX_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $ZABBIX_TOKEN" \
|
||||
-d "{\"jsonrpc\":\"2.0\",\"method\":\"history.push\",\"id\":1,\"params\":{\"itemid\":\"$ZABBIX_ITEM_ID\",\"value\":$RECLAIM_BYTES}}" > /dev/null \
|
||||
&& log "Zabbix notified (docker.buildcache.reclaimable.bytes=$RECLAIM_BYTES)"
|
||||
else
|
||||
log "Skipped Zabbix push (dry-run or token file missing)"
|
||||
fi
|
||||
|
||||
AVAIL=$(df / | tail -1 | awk '{print $4}')
|
||||
log "Root free: ${AVAIL}KB"
|
||||
|
||||
log "=== docker-maintenance prune.sh end ==="
|
||||
} | tee -a "$LOG_FILE"
|
||||
Reference in New Issue
Block a user