alvis
41f3f15d27
docker-maintenance/ adds a systemd timer + prune.sh for the root LV that holds Docker's data-root and has filled to 100% before, risking ENOSPC corruption. The script sticks to the safe reclaim set (builder cache, dangling images, stopped containers) and deliberately avoids `-a` and volume pruning, which can destroy live data when run unattended. kanboard/backup.sh and healthcheck.sh bring Kanboard in line with the other services. seafile/ and vaultwarden/ backup scripts get fixes carried from the stability audit. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
112 lines
4.5 KiB
Bash
Executable File
112 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Docker growth guard — recurring safe prune (kb#184, stability audit 2026-07-24).
|
|
#
|
|
# BACKGROUND: root LV (/) holds Docker's data-root. On 2026-07-24 it filled to
|
|
# 1.5G free / 100% used, risking ENOSPC corruption across every service on Agap.
|
|
# A one-off safe reclaim (builder prune + dangling images + stopped containers)
|
|
# took it from 1.5G to 25G free. This script is the recurring guard so it can't
|
|
# silently refill between now and the next manual audit.
|
|
#
|
|
# SAFE SCOPE ONLY (learned from the 2026-07-24 incident — do not expand without
|
|
# a human decision, see README.md in this directory):
|
|
# - docker builder prune -f (build cache; always safe, fully rebuildable)
|
|
# - docker container prune -f (stopped/exited containers only)
|
|
# - docker image prune -f (DANGLING images only — untagged, no -a)
|
|
#
|
|
# EXPLICITLY NOT DONE HERE (human judgment call, see README.md):
|
|
# - docker image prune -a (removes TAGGED but currently-unused images —
|
|
# could remove images a human wants kept)
|
|
# - docker volume prune (destroys live data if a volume is unmounted
|
|
# but still wanted — never run unattended)
|
|
#
|
|
# Pushes freed-space and post-prune build-cache-reclaimable metrics to Zabbix
|
|
# (trapper items on host AgapHost, hostid 10776). Zabbix lives on lizacer
|
|
# (192.168.1.4:81), not Agap.
|
|
#
|
|
# Run manually to test: bash prune.sh --dry-run
|
|
# Scheduled via cron (see README.md for the exact line — NOT installed by this
|
|
# script; installation is a human/handoff step, not something this script does).
|
|
|
|
set -euo pipefail
|
|
|
|
LOG_FILE="/home/alvis/agap_git/docker-maintenance/prune.log"
|
|
ZABBIX_TOKEN_FILE="/home/alvis/.zabbix_token"
|
|
ZABBIX_URL="http://192.168.1.4:81/api_jsonrpc.php"
|
|
ZABBIX_ITEM_ID="70624" # docker.buildcache.reclaimable.bytes on host AgapHost (10776)
|
|
|
|
DRY_RUN=0
|
|
if [[ "${1:-}" == "--dry-run" ]]; then
|
|
DRY_RUN=1
|
|
fi
|
|
|
|
log() {
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') $*"
|
|
}
|
|
|
|
{
|
|
log "=== docker-maintenance prune.sh start (dry_run=$DRY_RUN) ==="
|
|
|
|
log "--- before ---"
|
|
df -h / | tail -n +2
|
|
docker system df
|
|
|
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
log "DRY RUN: would run: docker builder prune -f"
|
|
log "DRY RUN: would run: docker container prune -f"
|
|
log "DRY RUN: would run: docker image prune -f (dangling only)"
|
|
else
|
|
log "Running: docker builder prune -f"
|
|
docker builder prune -f || true
|
|
|
|
log "Running: docker container prune -f"
|
|
docker container prune -f || true
|
|
|
|
log "Running: docker image prune -f (dangling only, no -a)"
|
|
docker image prune -f || true
|
|
fi
|
|
|
|
log "--- after ---"
|
|
df -h / | tail -n +2
|
|
DF_JSON=$(docker system df --format '{{json .}}')
|
|
echo "$DF_JSON"
|
|
|
|
# Extract build-cache reclaimable bytes for Zabbix. `docker system df` doesn't
|
|
# give bytes directly (only human-readable strings like "14.49GB"), so use
|
|
# `docker system df -v` reclaimable percentage isn't reliable either; instead
|
|
# compute it from `docker builder du`-equivalent: sum of build cache records
|
|
# not marked Shared/in-use is nontrivial to script robustly, so fall back to
|
|
# parsing the "Build Cache" line's RECLAIMABLE column via docker system df.
|
|
RECLAIM_STR=$(docker system df | awk '/^Build Cache/ {print $NF}')
|
|
# RECLAIM_STR looks like "14.49GB" or "0B" -- convert to bytes (approx, GB/MB/KB = *1000^n, matching docker's own decimal convention)
|
|
RECLAIM_BYTES=$(python3 -c "
|
|
import re, sys
|
|
s = '$RECLAIM_STR'
|
|
m = re.match(r'([0-9.]+)\s*([KMGTP]?B)', s)
|
|
if not m:
|
|
print(0)
|
|
else:
|
|
val, unit = float(m.group(1)), m.group(2)
|
|
mult = {'B':1,'KB':1000,'MB':1000**2,'GB':1000**3,'TB':1000**4,'PB':1000**5}[unit]
|
|
print(int(val*mult))
|
|
" 2>/dev/null || echo 0)
|
|
|
|
log "Build cache reclaimable: $RECLAIM_STR (~$RECLAIM_BYTES bytes)"
|
|
|
|
if [[ -f "$ZABBIX_TOKEN_FILE" && "$DRY_RUN" -eq 0 ]]; then
|
|
ZABBIX_TOKEN=$(cat "$ZABBIX_TOKEN_FILE")
|
|
env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \
|
|
curl -s -X POST "$ZABBIX_URL" \
|
|
-H "Content-Type: application/json" \
|
|
-H "Authorization: Bearer $ZABBIX_TOKEN" \
|
|
-d "{\"jsonrpc\":\"2.0\",\"method\":\"history.push\",\"id\":1,\"params\":{\"itemid\":\"$ZABBIX_ITEM_ID\",\"value\":$RECLAIM_BYTES}}" > /dev/null \
|
|
&& log "Zabbix notified (docker.buildcache.reclaimable.bytes=$RECLAIM_BYTES)"
|
|
else
|
|
log "Skipped Zabbix push (dry-run or token file missing)"
|
|
fi
|
|
|
|
AVAIL=$(df / | tail -1 | awk '{print $4}')
|
|
log "Root free: ${AVAIL}KB"
|
|
|
|
log "=== docker-maintenance prune.sh end ==="
|
|
} | tee -a "$LOG_FILE"
|