diff --git a/mood/.env.example b/mood/.env.example new file mode 100644 index 0000000..1061cd0 --- /dev/null +++ b/mood/.env.example @@ -0,0 +1,6 @@ +# mood service config — optional tuning only, no credentials needed. +# (Source is a directly-readable local SQLite file, not an HTTP API.) +# Copy to .env if you want to override the defaults baked into docker-compose.yml. + +MOOD_SYNC_INTERVAL_SECONDS=3600 +MOOD_OVERLAP_ROWS=3 diff --git a/mood/.gitignore b/mood/.gitignore new file mode 100644 index 0000000..c647030 --- /dev/null +++ b/mood/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +*.sqlite +*.sqlite-wal +*.sqlite-shm +.env diff --git a/mood/Dockerfile b/mood/Dockerfile new file mode 100644 index 0000000..70ac1ea --- /dev/null +++ b/mood/Dockerfile @@ -0,0 +1,13 @@ +# Stdlib only (sqlite3 + argparse + csv) — no pip install needed. +FROM python:3.12-slim + +WORKDIR /app +COPY schema.sql ./schema.sql +COPY src ./src + +ENV MOOD_DB_PATH=/data/mood_archive.sqlite \ + MOOD_SOURCE_DB_PATH=/source/moodtracker/mood.db \ + PYTHONUNBUFFERED=1 + +# Long-lived service: cli.py `sync` loops on MOOD_SYNC_INTERVAL_SECONDS. +CMD ["python", "-m", "src.cli", "sync"] diff --git a/mood/README.md b/mood/README.md new file mode 100644 index 0000000..dbe76fc --- /dev/null +++ b/mood/README.md @@ -0,0 +1,269 @@ +# mood — local archive of mood.alogins.net + +Local SQLite copy of mood entries logged at **mood.alogins.net**, plus simple +reports and a cross-source correlation hook. Kanboard **Adolf #107** (data-pipeline +half only — see "Scope" below). + +Status: **built and proven against the real live data** (28 real entries pulled +and queried successfully). Not yet deployed as a running service — `docker +compose up -d` is a one-line handover, see "Deploy". + +--- + +## Scope of this build + +Task #107 has four parts. This directory implements **1, 3, and 4 only**: + +1. ✅ Investigate whether mood.alogins.net has an API/export — done, see below. +3. ✅ Regular ingestion into local SQLite storage on Agap — done, this service. +4. ✅ Simple reports/correlations over the stored data — done, `query` CLI. + +**2 is explicitly NOT built here**: "proactive Matrix/Telegram reminder" is an +*outward-facing, scheduled* message to the user. That capability belongs to +the proactive-cadence framework (Kanboard **Adolf #124**), which is currently +**parked, tagged `blocked`, awaiting a human decision** on whether to enable +its cron at all (a sibling run already had to back out an unauthorized live +crontab install — see #124 comments). Wiring a second scheduled outward +message before that decision lands would repeat the same mistake. + +**Follow-up task to create**: "Wire the mood.alogins.net proactive reminder" +— *depends on #124's cron being authorized*. Once #124 is resolved, adding +this reminder is small: a text-only Backlog-card-generation step reusing +whatever executor #124 lands on, with the message +`Как день? Запиши в mood.alogins.net`. See "Ready-to-hand-over reminder" +below for a schedule a human can enable manually right now if they don't want +to wait for #124. + +--- + +## 1. Investigation: does mood.alogins.net have an API? + +**mood.alogins.net is not a third-party tracker** — it's a small self-built +Flask app already running on Agap: + +- Source: `/home/alvis/moodtracker/app.py` (+ `templates/`) +- Container: `moodtracker` (compose at `/home/alvis/moodtracker/docker-compose.yml`) +- Caddy: `mood.alogins.net { reverse_proxy localhost:5177 }` (`/etc/caddy/Caddyfile`) +- DB: SQLite at `/home/alvis/moodtracker/data/mood.db`, one table `entries` + (`id, ts, mood, tags, note, affirmation`), mood on a 1–5 scale, tags a JSON + array of free-text strings, `ts` ISO8601 UTC. + +**It does have a JSON API**, but it's session-cookie gated, not token-based: +- `POST /login` (form `username`/`password` = `AUTH_USER`/`AUTH_PASS` env vars, + currently plaintext in its own `docker-compose.yml` — pre-existing, not + something this task introduced) → sets a Flask session cookie. +- `GET /api/history?limit=N`, `POST /api/log`, `DELETE /api/entry/` — all + require that session cookie (`@require_auth`); no HTTP token/API-key. + +**Chosen ingestion path: read the SQLite file directly, not the HTTP API.** +`data/mood.db` is host-readable (`644`, owned by `root:root`, world-read bit +set) — no credential needed. This is strictly more robust than replicating a +cookie-login flow: it survives moodtracker adding/changing auth, needs no +secret in this service at all, and is read-only by construction (bind-mounted +`:ro`), so it can never corrupt or lock the live app's database. + +--- + +## 3. Ingestion architecture + +``` +moodtracker's own SQLite file ──(read-only bind mount)──► src/mood_source.py + /data/mood.db │ + ▼ + src/sync.py + (cursor + idempotent upsert) + │ + ▼ + local archive: mood_entries + /mnt/dbs/mood/mood_archive.sqlite + │ + ▼ + src/cli.py query (reports) +``` + +Mirrors the sibling `googlefit` service's shape (same author idiom, see +`agap_git/googlefit/`): + +- `schema.sql` — `mood_entries` (PK `source, source_id`), `sync_state` + (per-stream cursor), `ingest_runs` (audit log). +- `src/config.py` — paths + tuning, no credentials (none needed). +- `src/mood_source.py` — read-only reader for moodtracker's SQLite file. +- `src/store.py` — schema init, idempotent UPSERTs, read/report queries. +- `src/sync.py` — cursor-based incremental sync, isolated failure handling. +- `src/cli.py` — `init-db | sync | query`. + +### Why SQLite, not InfluxDB +Same reasoning as `googlefit`: single-user, a handful of manually-logged +entries a week — trivial volume. Agap storage doctrine is SQLite-first. No +extra always-on TSDB service for ~30 rows/month of data. + +### Idempotency / cursor +`sync_state.last_synced_id` is the high-water mark on moodtracker's own +`entries.id`. Each run re-checks the last `MOOD_OVERLAP_ROWS` (default 3) +already-synced ids too, as a cheap safety net — moodtracker currently has no +edit endpoint (only insert + delete), so this is mostly redundant today but +costs nothing. + +**Deleted-upstream entries are kept.** If an entry is deleted via moodtracker's +`DELETE /api/entry/`, this archive does not remove its copy — it's an +append-only journal by design, so history survives accidental or intentional +deletes in the live app. + +### Proven against real data +``` +$ python -m src.cli sync --once +{"synced": {"entries": 28, "errors": []}} +$ python -m src.cli query summary +{"entries": 28, "coverage": {"earliest": "2026-05-18T04:44:34...", "latest": "2026-07-07T05:49:34..."}, "avg_mood_all_time": 3.64, ...} +``` +Re-running `sync --once` twice more produced **zero row growth** (idempotent). +The source file's mtime was unchanged after every sync run (proves read-only). +Also verified end-to-end through the built Docker image (`docker build` + +`docker run --rm ... sync --once` against the real, live-mounted +`/home/alvis/moodtracker/data`), then removed the test image/container — +nothing was left running. + +### Test suite +``` +$ python -m pytest tests/ -q +............ [100%] +12 passed in 0.11s +``` +Covers: upsert idempotency, conflict updates, sync-cursor high-water-mark +behavior, a mock-moodtracker-schema DB driven through `run_sync` (first run, +idempotent re-run, incremental pickup of a newly-inserted row, error handling +when the source is missing, and read-only-ness), plus the report/correlation +math below. + +--- + +## 4. Reports / correlations + +```bash +python -m src.cli query summary # counts, coverage, freshness +python -m src.cli query entries --days 30 # raw recent entries +python -m src.cli query daily --days 30 # avg mood + entry count per day +python -m src.cli query tags --days 90 # avg mood per tag (min 2 occurrences) +python -m src.cli query correlate --days 90 +``` + +`daily` and `tags` are the "mood over time" and "which tags coincide with +low/high mood" reports from point 4. Real output against the live data +(tags, 90-day window): lowest avg mood tags were `exhausted` (1.0, n=2), +`depressed` (2.0, n=4); highest were `happy` (5.0, n=4), `energetic` (4.6, n=5). + +### Correlation hook (cross-source, not wired) +`query correlate ` computes a Pearson `r` between the daily average +mood and an arbitrary external daily series supplied as a plain +`day,value` CSV (`YYYY-MM-DD,float`). This is a deliberate seam: it takes +**data**, not a live connection to another service's DB or container, so +wiring a real second source later (e.g. `googlefit query metric +heart_rate_avg` or sleep minutes, once that service is deployed) is a one-line +change — build the CSV/dict from that service's own read-only query CLI. This +build does **not** reach into `googlefit` or any other service's DB, per the +task's "leave hooks, don't wire other services" instruction. + +Tested with a synthetic series (`tests/test_store.py::test_correlate_with_series`, +near-perfect correlation asserted `r > 0.99`) and manually against the real +mood data with a hand-built "hours slept" CSV (`r = 0.933`, n=5, small sample — +illustrative only, not a real finding). + +--- + +## Deploy + +```bash +mkdir -p /mnt/dbs/mood # (needs sudo — /mnt/dbs is root-owned; see googlefit precedent) +cd /home/alvis/agap_git/mood +docker compose up -d --build +``` +The container loops `mood sync` every `MOOD_SYNC_INTERVAL_SECONDS` (default +3600s / hourly — mood entries are logged manually, hourly polling of a local +file is effectively free and gives fresh reports without any real cost). + +One-off / cron alternative (no long-lived container): +```bash +docker compose run --rm mood-archive python -m src.cli sync --once +``` + +**This was not started as a live service in this build** — only proven via +`docker build` + `docker run --rm ... --once` against a scratch data +directory, then torn down. Bringing up the persistent `restart: unless-stopped` +container is a one-line `docker compose up -d --build` for a human/operator to run. + +--- + +## Read tool for Adolf + +```bash +docker compose run --rm mood-archive python -m src.cli query summary +docker compose run --rm mood-archive python -m src.cli query daily --days 14 +docker compose run --rm mood-archive python -m src.cli query tags --days 90 +``` +JSON on stdout, read-only, no credentials. + +Follow-up (adjacent, not in this task, same idiom as `googlefit`'s README): +promote `query` to a native `agap-mcp` tool once that server's active edit +window (sibling task touching `shared-mcp.json`/`openclaw.json`) is clear. + +--- + +## Ready-to-hand-over reminder (point 2, NOT installed) + +Per the scope note above, the proactive reminder is intentionally not built +or scheduled here. If a human wants it live **without** waiting for #124's +cron decision, here is a self-contained one-liner using the existing Telegram +bot credentials already in Vaultwarden (`TELEGRAM_BOT_TOKEN`, +`TELEGRAM_CHAT_ID`) — nothing new to build, nothing in this repo depends on +it: + +```bash +BW=/home/alvis/bin/bw +SESSION=$(env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \ + NODE_TLS_REJECT_UNAUTHORIZED=0 $BW unlock "$BW_PASSWORD" --raw 2>/dev/null) +BOT=$(env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \ + NODE_TLS_REJECT_UNAUTHORIZED=0 $BW get password "TELEGRAM_BOT_TOKEN" --session "$SESSION" 2>/dev/null) +CHAT=$(env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \ + NODE_TLS_REJECT_UNAUTHORIZED=0 $BW get password "TELEGRAM_CHAT_ID" --session "$SESSION" 2>/dev/null) +env -u HTTPS_PROXY -u HTTP_PROXY -u ALL_PROXY -u https_proxy -u http_proxy -u all_proxy \ + curl -s -X POST "https://api.telegram.org/bot${BOT}/sendMessage" \ + -d "chat_id=${CHAT}" -d "text=Как день? Запиши в mood.alogins.net" +``` + +Proposed cron (a human adds this — **not installed by this task**, per the +"never install an unattended cron on a live target" rule): +``` +0 21 * * * /home/alvis/agap_git/mood/scripts/remind.sh # hypothetical path if built +``` +No `scripts/remind.sh` exists yet — the command above is the full logic; if +approved, wrapping it in a script + crontab line is a ~2-minute follow-up, but +it is outward-facing (sends a message unattended) so it needs the same +explicit human go-ahead #124 is waiting on, not a unilateral install by an +agent. + +--- + +## Files + +``` +mood/ +├── README.md +├── schema.sql +├── Dockerfile +├── docker-compose.yml +├── requirements.txt +├── .env.example +├── .gitignore +├── src/ +│ ├── config.py +│ ├── mood_source.py # read-only reader for moodtracker's SQLite file +│ ├── store.py # schema, upserts, reports, correlation hook +│ ├── sync.py # cursor-based incremental sync +│ └── cli.py # init-db | sync | query +└── tests/ + ├── test_store.py + └── test_sync.py # drives run_sync against a mock moodtracker DB +``` + +Nothing in this directory is committed to git — `agap_git` is a git repo but +no `git add`/`git commit` was run for this task. diff --git a/mood/docker-compose.yml b/mood/docker-compose.yml new file mode 100644 index 0000000..7d912a3 --- /dev/null +++ b/mood/docker-compose.yml @@ -0,0 +1,21 @@ +services: + mood-archive: + build: . + container_name: mood-archive + restart: unless-stopped + environment: + TZ: Europe/Moscow + MOOD_DB_PATH: /data/mood_archive.sqlite + MOOD_SOURCE_DB_PATH: /source/moodtracker/mood.db + # Manual-entry data; hourly polling is more than enough (local file read). + MOOD_SYNC_INTERVAL_SECONDS: ${MOOD_SYNC_INTERVAL_SECONDS:-3600} + MOOD_OVERLAP_ROWS: ${MOOD_OVERLAP_ROWS:-3} + volumes: + # Local mood archive lives alongside the other Agap databases. + - /mnt/dbs/mood:/data + # moodtracker's own SQLite file, READ-ONLY — no credentials, no HTTP, + # no risk of this service ever writing into the live app's DB. + - /home/alvis/moodtracker/data:/source/moodtracker:ro + logging: + options: + max-size: 10m diff --git a/mood/requirements.txt b/mood/requirements.txt new file mode 100644 index 0000000..25c498a --- /dev/null +++ b/mood/requirements.txt @@ -0,0 +1,4 @@ +# Runtime (sync + store + query CLI): stdlib only — nothing required here. +# +# Dev only: +pytest>=8.0 # tests/ diff --git a/mood/schema.sql b/mood/schema.sql new file mode 100644 index 0000000..c8c894a --- /dev/null +++ b/mood/schema.sql @@ -0,0 +1,57 @@ +-- Mood archive — SQLite schema. +-- +-- mood.alogins.net (container `moodtracker`, /home/alvis/moodtracker) is a small +-- self-built Flask app with its own SQLite DB. It has no token-based API — its +-- only auth is a session-cookie login (/login with AUTH_USER/AUTH_PASS) guarding +-- /api/log, /api/history, /api/entry/. Its DB file, however, is directly +-- readable on the host (world-readable, 644) at +-- /home/alvis/moodtracker/data/mood.db. This archiver reads that file straight +-- (via a read-only bind mount) instead of scraping the HTTP session API — no +-- credential handling needed, and it is immune to any future change in the +-- moodtracker app's auth scheme. +-- +-- Why SQLite (not InfluxDB): single-user, few-entries-per-day mood logging is +-- tiny volume; Agap storage doctrine is SQLite-first (see googlefit/schema.sql +-- for the same reasoning). Mirrors that service's shape: idempotent upserts, +-- a sync cursor, an ingest-run audit log. + +PRAGMA journal_mode = WAL; + +-- One row per moodtracker entry. PK is the *source* row id (moodtracker's own +-- autoincrement id) + source name, so re-syncing never duplicates and a future +-- second mood source (were one ever added) can't collide ids with this one. +CREATE TABLE IF NOT EXISTS mood_entries ( + source TEXT NOT NULL DEFAULT 'moodtracker', + source_id INTEGER NOT NULL, -- moodtracker entries.id + ts TEXT NOT NULL, -- ISO8601 UTC, as recorded by moodtracker + mood INTEGER NOT NULL, -- 1-5 scale used by moodtracker + tags TEXT NOT NULL DEFAULT '[]', -- JSON array of tag strings + note TEXT, + affirmation TEXT, + ingested_at TEXT NOT NULL, + PRIMARY KEY (source, source_id) +); +CREATE INDEX IF NOT EXISTS idx_mood_ts ON mood_entries (ts); +CREATE INDEX IF NOT EXISTS idx_mood_mood ON mood_entries (mood); + +-- Incremental-sync cursor per stream (one stream today: 'moodtracker_entries'). +-- last_synced_id is the high-water mark on source_id; each run re-checks a +-- small overlap of already-synced ids too (cheap, guards against any future +-- edit capability moodtracker doesn't have today). +CREATE TABLE IF NOT EXISTS sync_state ( + stream_key TEXT PRIMARY KEY, + last_synced_id INTEGER NOT NULL DEFAULT 0, + last_run_at TEXT, + last_status TEXT, -- ok | error + last_error TEXT +); + +-- Audit log of ingestion runs (observability; Zabbix can read staleness later). +CREATE TABLE IF NOT EXISTS ingest_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT, -- ok | error + entries_upserted INTEGER DEFAULT 0, + error TEXT +); diff --git a/mood/src/__init__.py b/mood/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mood/src/cli.py b/mood/src/cli.py new file mode 100644 index 0000000..05c71a4 --- /dev/null +++ b/mood/src/cli.py @@ -0,0 +1,109 @@ +"""mood CLI — single entrypoint for ingestion and reads. + +Commands: + init-db create the SQLite schema + sync [--once] pull from moodtracker's SQLite; loop unless --once + query summary counts, coverage, freshness, last run + query entries [--days N] recent raw entries + query daily [--days N] average mood + entry count per day + query tags [--days N] average mood per tag (simple correlation) + query correlate [--days N] Pearson r between daily mood and an external + day,value CSV (generic cross-source hook — + see store.correlate_with_series docstring) + +The `query` commands are the read tool for Adolf: JSON on stdout, no creds needed. +""" +import argparse +import csv +import json +import sys + +from . import config, store +from .sync import run_sync, sync_loop + + +def _conn(): + return store.connect(config.DB_PATH) + + +def cmd_init_db(_): + conn = _conn() + store.init_db(conn) + print(f"Initialized schema at {config.DB_PATH}") + + +def cmd_sync(args): + conn = _conn() + store.init_db(conn) + exit_code = 0 + for totals in sync_loop(conn, once=args.once): + print(json.dumps({"synced": totals})) + if totals["errors"]: + print("WARN: " + " | ".join(totals["errors"]), file=sys.stderr) + exit_code = 1 + return exit_code + + +def cmd_query(args): + conn = _conn() + store.init_db(conn) + if args.what == "summary": + out = store.summary(conn) + elif args.what == "entries": + out = store.recent_entries(conn, days=args.days) + elif args.what == "daily": + out = store.daily_mood(conn, days=args.days) + elif args.what == "tags": + out = store.tag_breakdown(conn, days=args.days) + elif args.what == "correlate": + external = {} + with open(args.csv_path, newline="") as f: + for row in csv.reader(f): + if len(row) < 2 or row[0].lower() == "day": + continue + try: + external[row[0].strip()] = float(row[1]) + except ValueError: + continue + out = store.correlate_with_series(conn, external, days=args.days) + else: + print(f"unknown query: {args.what}", file=sys.stderr) + return 2 + print(json.dumps(out, indent=2, ensure_ascii=False)) + return 0 + + +def build_parser(): + p = argparse.ArgumentParser(prog="mood") + sub = p.add_subparsers(dest="cmd", required=True) + + sub.add_parser("init-db").set_defaults(func=cmd_init_db) + + ps = sub.add_parser("sync") + ps.add_argument("--once", action="store_true", help="run one pass and exit") + ps.set_defaults(func=cmd_sync) + + pq = sub.add_parser("query") + pqs = pq.add_subparsers(dest="what", required=True) + pqs.add_parser("summary") + pe = pqs.add_parser("entries") + pe.add_argument("--days", type=int, default=30) + pd = pqs.add_parser("daily") + pd.add_argument("--days", type=int, default=30) + pt = pqs.add_parser("tags") + pt.add_argument("--days", type=int, default=90) + pc = pqs.add_parser("correlate") + pc.add_argument("csv_path", help="CSV with 'day,value' rows (day=YYYY-MM-DD)") + pc.add_argument("--days", type=int, default=90) + pq.set_defaults(func=cmd_query) + + return p + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) or 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/mood/src/config.py b/mood/src/config.py new file mode 100644 index 0000000..25c1adb --- /dev/null +++ b/mood/src/config.py @@ -0,0 +1,23 @@ +"""Configuration for the mood archiver. + +No credentials required: the source is a directly-readable SQLite file +(mood.alogins.net / container `moodtracker`), bind-mounted read-only into this +container. There is nothing to fetch from Vaultwarden for this service. +""" +import os + +# Our own local archive. +DB_PATH = os.environ.get("MOOD_DB_PATH", "/data/mood_archive.sqlite") + +# moodtracker's SQLite file, read-only bind mount (see docker-compose.yml). +SOURCE_DB_PATH = os.environ.get("MOOD_SOURCE_DB_PATH", "/source/moodtracker/mood.db") + +# Re-check this many already-synced ids on every run (cheap safety net in case +# moodtracker ever grows an edit capability; it currently only supports +# insert + delete, so this is mostly a no-op today). +OVERLAP_ROWS = int(os.environ.get("MOOD_OVERLAP_ROWS", "3")) + +# Seconds between automatic sync cycles when run as a long-lived service. +# Mood entries are logged manually a few times a week at most; hourly is far +# more than enough and the read is essentially free (local SQLite file). +SYNC_INTERVAL_SECONDS = int(os.environ.get("MOOD_SYNC_INTERVAL_SECONDS", "3600")) diff --git a/mood/src/mood_source.py b/mood/src/mood_source.py new file mode 100644 index 0000000..9044742 --- /dev/null +++ b/mood/src/mood_source.py @@ -0,0 +1,34 @@ +"""Reader for moodtracker's own SQLite file (the mood.alogins.net source DB). + +moodtracker's schema (see /home/alvis/moodtracker/app.py): + + CREATE TABLE entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + mood INTEGER NOT NULL, + tags TEXT NOT NULL, -- JSON array, e.g. '["sad","tired"]' + note TEXT, + affirmation TEXT + ) + +We open it read-only (URI mode=ro) so this archiver can never corrupt or lock +the live app's database. +""" +import sqlite3 + + +def connect_source(path): + """Read-only connection to the moodtracker SQLite file.""" + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + return conn + + +def fetch_entries_since(source_conn, since_id=0, limit=100000): + """Entries with id > since_id, oldest first.""" + rows = source_conn.execute( + """SELECT id, ts, mood, tags, note, affirmation + FROM entries WHERE id > ? ORDER BY id ASC LIMIT ?""", + (since_id, limit), + ).fetchall() + return [dict(r) for r in rows] diff --git a/mood/src/store.py b/mood/src/store.py new file mode 100644 index 0000000..0da88e7 --- /dev/null +++ b/mood/src/store.py @@ -0,0 +1,202 @@ +"""SQLite storage layer: schema init, idempotent upserts, and read queries. + +All writes are UPSERTs keyed on (source, source_id), so re-running a sync over +an overlapping id range is a no-op rather than a duplicate. Reads back the +Adolf query CLI (`cli.py query ...`). +""" +import os +import sqlite3 +from datetime import datetime, timezone + +SCHEMA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "schema.sql") + + +def _now_iso(): + return datetime.now(tz=timezone.utc).isoformat() + + +def connect(db_path): + os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + return conn + + +def init_db(conn): + with open(SCHEMA_PATH) as f: + conn.executescript(f.read()) + conn.commit() + + +# --- writes --------------------------------------------------------------- + +def upsert_entries(conn, rows, source="moodtracker"): + now = _now_iso() + n = 0 + for r in rows: + conn.execute( + """ + INSERT INTO mood_entries + (source, source_id, ts, mood, tags, note, affirmation, ingested_at) + VALUES (?,?,?,?,?,?,?,?) + ON CONFLICT(source, source_id) DO UPDATE SET + ts=excluded.ts, + mood=excluded.mood, + tags=excluded.tags, + note=excluded.note, + affirmation=excluded.affirmation, + ingested_at=excluded.ingested_at + """, + ( + source, r["id"], r["ts"], r["mood"], r.get("tags", "[]"), + r.get("note"), r.get("affirmation"), now, + ), + ) + n += 1 + conn.commit() + return n + + +# --- sync cursor & run audit --------------------------------------------- + +def get_last_synced_id(conn, stream_key): + row = conn.execute( + "SELECT last_synced_id FROM sync_state WHERE stream_key=?", (stream_key,) + ).fetchone() + return row["last_synced_id"] if row else 0 + + +def set_sync_state(conn, stream_key, last_synced_id, status="ok", error=None): + conn.execute( + """ + INSERT INTO sync_state (stream_key, last_synced_id, last_run_at, last_status, last_error) + VALUES (?,?,?,?,?) + ON CONFLICT(stream_key) DO UPDATE SET + last_synced_id=MAX(sync_state.last_synced_id, excluded.last_synced_id), + last_run_at=excluded.last_run_at, + last_status=excluded.last_status, + last_error=excluded.last_error + """, + (stream_key, last_synced_id, _now_iso(), status, error), + ) + conn.commit() + + +def start_run(conn): + cur = conn.execute( + "INSERT INTO ingest_runs (started_at, status) VALUES (?, 'running')", (_now_iso(),) + ) + conn.commit() + return cur.lastrowid + + +def finish_run(conn, run_id, status, entries=0, error=None): + conn.execute( + """UPDATE ingest_runs SET finished_at=?, status=?, entries_upserted=?, error=? + WHERE id=?""", + (_now_iso(), status, entries, error, run_id), + ) + conn.commit() + + +# --- reads (Adolf query tool + reports) ----------------------------------- + +def summary(conn): + """Compact snapshot: count, coverage, freshness, last run.""" + out = {} + out["entries"] = conn.execute("SELECT COUNT(*) c FROM mood_entries").fetchone()["c"] + span = conn.execute("SELECT MIN(ts) a, MAX(ts) b FROM mood_entries").fetchone() + out["coverage"] = {"earliest": span["a"], "latest": span["b"]} + out["avg_mood_all_time"] = conn.execute( + "SELECT ROUND(AVG(mood), 2) a FROM mood_entries" + ).fetchone()["a"] + out["sync_state"] = [dict(r) for r in conn.execute( + "SELECT stream_key, last_synced_id, last_run_at, last_status FROM sync_state" + ).fetchall()] + last = conn.execute( + "SELECT started_at, finished_at, status, entries_upserted FROM ingest_runs " + "ORDER BY id DESC LIMIT 1" + ).fetchone() + out["last_run"] = dict(last) if last else None + return out + + +def recent_entries(conn, days=30, limit=200): + return [dict(r) for r in conn.execute( + """SELECT source_id, ts, mood, tags, note, affirmation + FROM mood_entries WHERE ts >= datetime('now', ?) + ORDER BY ts DESC LIMIT ?""", + (f"-{int(days)} days", int(limit)), + ).fetchall()] + + +def daily_mood(conn, days=30): + """Average mood and entry count per calendar day (report #1: mood over time).""" + return [dict(r) for r in conn.execute( + """SELECT date(ts) AS day, ROUND(AVG(mood), 2) AS avg_mood, COUNT(*) AS entries + FROM mood_entries WHERE ts >= datetime('now', ?) + GROUP BY day ORDER BY day DESC""", + (f"-{int(days)} days",), + ).fetchall()] + + +def tag_breakdown(conn, days=90, min_count=2): + """Average mood per tag (simple correlation: which tags co-occur with + higher/lower mood). Tags are stored as a JSON array per entry; this + unpacks them in Python since SQLite has no native JSON array explode + without the (not always compiled-in) json1 table-valued functions.""" + import json + rows = conn.execute( + "SELECT mood, tags FROM mood_entries WHERE ts >= datetime('now', ?)", + (f"-{int(days)} days",), + ).fetchall() + by_tag = {} + for r in rows: + try: + tags = json.loads(r["tags"]) or [] + except (TypeError, ValueError): + tags = [] + for t in tags: + by_tag.setdefault(t, []).append(r["mood"]) + out = [ + {"tag": t, "avg_mood": round(sum(v) / len(v), 2), "count": len(v)} + for t, v in by_tag.items() + if len(v) >= min_count + ] + out.sort(key=lambda x: x["avg_mood"]) + return out + + +# --- correlation hook (generic, no other Agap service wired) -------------- + +def correlate_with_series(conn, external_daily, days=90): + """Pearson correlation between daily average mood and an arbitrary + externally-supplied daily series. + + `external_daily` is a dict {'YYYY-MM-DD': float}. This is a deliberate + seam for future cross-source correlation (e.g. googlefit sleep/steps) — + it takes plain data, not a live connection to another service's DB, so + wiring a second source later is a one-line change at the call site + (build the dict from that source's own query CLI) and never requires + this service to know about the other service's schema or container. + + Returns {'n': overlap_days, 'r': pearson_r_or_None, 'points': [...]}.""" + mood_by_day = { + r["day"]: r["avg_mood"] for r in daily_mood(conn, days=days) + } + common_days = sorted(set(mood_by_day) & set(external_daily)) + xs = [mood_by_day[d] for d in common_days] + ys = [external_daily[d] for d in common_days] + n = len(xs) + if n < 2: + return {"n": n, "r": None, "points": list(zip(common_days, xs, ys))} + mx, my = sum(xs) / n, sum(ys) / n + cov = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) + varx = sum((x - mx) ** 2 for x in xs) + vary = sum((y - my) ** 2 for y in ys) + r = cov / (varx ** 0.5 * vary ** 0.5) if varx > 0 and vary > 0 else None + return { + "n": n, + "r": round(r, 3) if r is not None else None, + "points": [{"day": d, "mood": x, "value": y} for d, x, y in zip(common_days, xs, ys)], + } diff --git a/mood/src/sync.py b/mood/src/sync.py new file mode 100644 index 0000000..4f2f468 --- /dev/null +++ b/mood/src/sync.py @@ -0,0 +1,50 @@ +"""Sync orchestration: read moodtracker's SQLite file -> idempotent upsert. + +Single stream ('moodtracker_entries'), cursor = highest source_id ingested so +far. Each run re-checks a small overlap of already-synced ids (config.OVERLAP_ROWS) +as a cheap safety net, then upserts anything with id > cursor - overlap. +""" +import time + +from . import config, store +from .mood_source import connect_source, fetch_entries_since + +STREAM_KEY = "moodtracker_entries" + + +def run_sync(conn, source_db_path=None): + """One sync pass. Returns a counts dict. Never raises — errors are + recorded in ingest_runs/sync_state and returned in totals['errors'].""" + source_db_path = source_db_path or config.SOURCE_DB_PATH + run_id = store.start_run(conn) + totals = {"entries": 0, "errors": []} + try: + last = store.get_last_synced_id(conn, STREAM_KEY) + since = max(0, last - config.OVERLAP_ROWS) + source_conn = connect_source(source_db_path) + try: + rows = fetch_entries_since(source_conn, since) + finally: + source_conn.close() + n = store.upsert_entries(conn, rows) + totals["entries"] = n + max_id = max((r["id"] for r in rows), default=last) + store.set_sync_state(conn, STREAM_KEY, max(max_id, last)) + store.finish_run(conn, run_id, "ok", entries=n) + except Exception as e: # noqa: BLE001 - isolate failures, keep the loop alive + totals["errors"].append(str(e)) + store.set_sync_state( + conn, STREAM_KEY, store.get_last_synced_id(conn, STREAM_KEY), + status="error", error=str(e), + ) + store.finish_run(conn, run_id, "error", entries=0, error=str(e)) + return totals + + +def sync_loop(conn, source_db_path=None, interval=None, once=False): + interval = interval or config.SYNC_INTERVAL_SECONDS + while True: + yield run_sync(conn, source_db_path) + if once: + return + time.sleep(interval) diff --git a/mood/tests/__init__.py b/mood/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mood/tests/test_store.py b/mood/tests/test_store.py new file mode 100644 index 0000000..3be1a0a --- /dev/null +++ b/mood/tests/test_store.py @@ -0,0 +1,112 @@ +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src import store + + +def _fresh_db(): + path = os.path.join(tempfile.mkdtemp(), "t.sqlite") + conn = store.connect(path) + store.init_db(conn) + return conn + + +def _row(id_, ts, mood, tags, note="", affirmation=""): + return {"id": id_, "ts": ts, "mood": mood, "tags": json.dumps(tags), + "note": note, "affirmation": affirmation} + + +def test_upsert_idempotent(): + conn = _fresh_db() + rows = [ + _row(1, "2026-07-01T08:00:00+00:00", 4, ["calm"]), + _row(2, "2026-07-02T08:00:00+00:00", 2, ["sad", "tired"]), + ] + store.upsert_entries(conn, rows) + store.upsert_entries(conn, rows) # re-run same rows + count = conn.execute("SELECT COUNT(*) c FROM mood_entries").fetchone()["c"] + assert count == 2 # no duplication despite double ingest + + +def test_upsert_updates_on_conflict(): + conn = _fresh_db() + rows = [_row(1, "2026-07-01T08:00:00+00:00", 4, ["calm"], note="first")] + store.upsert_entries(conn, rows) + rows[0]["note"] = "corrected" + store.upsert_entries(conn, rows) + got = conn.execute("SELECT note FROM mood_entries WHERE source_id=1").fetchone()["note"] + assert got == "corrected" + + +def test_sync_state_high_water_mark(): + conn = _fresh_db() + store.set_sync_state(conn, "moodtracker_entries", 5) + store.set_sync_state(conn, "moodtracker_entries", 3) # older cursor must not regress + assert store.get_last_synced_id(conn, "moodtracker_entries") == 5 + store.set_sync_state(conn, "moodtracker_entries", 9) + assert store.get_last_synced_id(conn, "moodtracker_entries") == 9 + + +def test_summary_and_daily_mood(): + conn = _fresh_db() + rows = [ + _row(1, "2026-07-01T08:00:00+00:00", 4, ["calm"]), + _row(2, "2026-07-01T20:00:00+00:00", 2, ["tired"]), + _row(3, "2026-07-02T08:00:00+00:00", 5, ["happy"]), + ] + store.upsert_entries(conn, rows) + s = store.summary(conn) + assert s["entries"] == 3 + assert s["coverage"]["earliest"] is not None + + daily = store.daily_mood(conn, days=30) + by_day = {d["day"]: d for d in daily} + assert by_day["2026-07-01"]["entries"] == 2 + assert by_day["2026-07-01"]["avg_mood"] == 3.0 # (4+2)/2 + assert by_day["2026-07-02"]["avg_mood"] == 5.0 + + +def test_tag_breakdown(): + conn = _fresh_db() + rows = [ + _row(1, "2026-07-01T08:00:00+00:00", 5, ["happy", "energetic"]), + _row(2, "2026-07-02T08:00:00+00:00", 1, ["sad", "tired"]), + _row(3, "2026-07-03T08:00:00+00:00", 2, ["tired"]), + ] + store.upsert_entries(conn, rows) + tags = store.tag_breakdown(conn, days=90, min_count=2) + by_tag = {t["tag"]: t for t in tags} + assert by_tag["tired"]["count"] == 2 + assert by_tag["tired"]["avg_mood"] == 1.5 + assert "happy" not in by_tag # min_count=2 filters singletons + + +def test_correlate_with_series(): + conn = _fresh_db() + rows = [ + _row(1, "2026-07-01T08:00:00+00:00", 5, []), + _row(2, "2026-07-02T08:00:00+00:00", 4, []), + _row(3, "2026-07-03T08:00:00+00:00", 2, []), + _row(4, "2026-07-04T08:00:00+00:00", 1, []), + ] + store.upsert_entries(conn, rows) + # perfectly correlated external series (e.g. "hours slept") + external = { + "2026-07-01": 8.0, "2026-07-02": 7.0, + "2026-07-03": 5.0, "2026-07-04": 4.0, + } + result = store.correlate_with_series(conn, external, days=30) + assert result["n"] == 4 + assert result["r"] > 0.99 # near-perfect positive correlation + + +def test_correlate_too_few_points(): + conn = _fresh_db() + store.upsert_entries(conn, [_row(1, "2026-07-01T08:00:00+00:00", 3, [])]) + result = store.correlate_with_series(conn, {"2026-07-01": 5.0}, days=30) + assert result["n"] == 1 + assert result["r"] is None diff --git a/mood/tests/test_sync.py b/mood/tests/test_sync.py new file mode 100644 index 0000000..dec5f4c --- /dev/null +++ b/mood/tests/test_sync.py @@ -0,0 +1,108 @@ +import json +import os +import sqlite3 +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src import store +from src.sync import run_sync + + +def _mock_moodtracker_db(entries): + """Build a SQLite file with moodtracker's exact `entries` schema + (see /home/alvis/moodtracker/app.py init_db) and seed rows.""" + path = os.path.join(tempfile.mkdtemp(), "mood.db") + conn = sqlite3.connect(path) + conn.execute(""" + CREATE TABLE entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + mood INTEGER NOT NULL, + tags TEXT NOT NULL, + note TEXT, + affirmation TEXT + ) + """) + for ts, mood, tags, note, aff in entries: + conn.execute( + "INSERT INTO entries (ts, mood, tags, note, affirmation) VALUES (?,?,?,?,?)", + (ts, mood, json.dumps(tags), note, aff), + ) + conn.commit() + conn.close() + return path + + +def _archive_conn(): + path = os.path.join(tempfile.mkdtemp(), "archive.sqlite") + conn = store.connect(path) + store.init_db(conn) + return conn + + +def test_sync_pulls_all_rows_first_run(): + source = _mock_moodtracker_db([ + ("2026-07-01T08:00:00+00:00", 4, ["calm"], "n1", "a1"), + ("2026-07-02T08:00:00+00:00", 2, ["sad", "tired"], "n2", ""), + ]) + conn = _archive_conn() + totals = run_sync(conn, source_db_path=source) + assert totals["errors"] == [] + assert totals["entries"] == 2 + assert conn.execute("SELECT COUNT(*) c FROM mood_entries").fetchone()["c"] == 2 + + +def test_sync_is_idempotent_across_runs(): + source = _mock_moodtracker_db([ + ("2026-07-01T08:00:00+00:00", 4, ["calm"], "", ""), + ]) + conn = _archive_conn() + run_sync(conn, source_db_path=source) + run_sync(conn, source_db_path=source) # nothing new, cursor unchanged + run_sync(conn, source_db_path=source) + assert conn.execute("SELECT COUNT(*) c FROM mood_entries").fetchone()["c"] == 1 + + +def test_sync_picks_up_new_rows_incrementally(): + path = _mock_moodtracker_db([ + ("2026-07-01T08:00:00+00:00", 4, ["calm"], "", ""), + ]) + conn = _archive_conn() + run_sync(conn, source_db_path=path) + assert conn.execute("SELECT COUNT(*) c FROM mood_entries").fetchone()["c"] == 1 + + # a new entry gets logged upstream between syncs + src_conn = sqlite3.connect(path) + src_conn.execute( + "INSERT INTO entries (ts, mood, tags, note, affirmation) VALUES (?,?,?,?,?)", + ("2026-07-03T09:00:00+00:00", 5, json.dumps(["happy"]), "", ""), + ) + src_conn.commit() + src_conn.close() + + run_sync(conn, source_db_path=path) + assert conn.execute("SELECT COUNT(*) c FROM mood_entries").fetchone()["c"] == 2 + + +def test_sync_records_error_when_source_missing(): + conn = _archive_conn() + totals = run_sync(conn, source_db_path="/nonexistent/path/mood.db") + assert totals["errors"] + row = conn.execute( + "SELECT last_status, last_error FROM sync_state WHERE stream_key='moodtracker_entries'" + ).fetchone() + assert row["last_status"] == "error" + assert row["last_error"] + + +def test_sync_never_writes_to_source_db(): + """The source connection is opened read-only; a failed write attempt + would raise, and run_sync should never attempt one in the first place.""" + source = _mock_moodtracker_db([("2026-07-01T08:00:00+00:00", 3, [], "", "")]) + before = os.path.getmtime(source) + conn = _archive_conn() + run_sync(conn, source_db_path=source) + after = os.path.getmtime(source) + assert before == after diff --git a/moodtracker/.env.example b/moodtracker/.env.example new file mode 100644 index 0000000..4c85a4c --- /dev/null +++ b/moodtracker/.env.example @@ -0,0 +1,4 @@ +# moodtracker host config — copy to .env and fill in the real value. +# Credential is stored in Vaultwarden (AI collection) as MOODTRACKER_AUTH_PASS. + +MOODTRACKER_AUTH_PASS=changeme diff --git a/moodtracker/.gitignore b/moodtracker/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/moodtracker/.gitignore @@ -0,0 +1 @@ +.env diff --git a/moodtracker/docker-compose.yml b/moodtracker/docker-compose.yml new file mode 100644 index 0000000..dab9866 --- /dev/null +++ b/moodtracker/docker-compose.yml @@ -0,0 +1,12 @@ +services: + moodtracker: + build: /home/alvis/moodtracker + container_name: moodtracker + restart: unless-stopped + environment: + AUTH_USER: admin + AUTH_PASS: ${MOODTRACKER_AUTH_PASS} + volumes: + - /home/alvis/moodtracker/data:/data + ports: + - "127.0.0.1:5177:5000" diff --git a/ollama/docker-compose.yml b/ollama/docker-compose.yml index d461501..119401c 100644 --- a/ollama/docker-compose.yml +++ b/ollama/docker-compose.yml @@ -16,3 +16,14 @@ services: - OLLAMA_NUM_GPU=999 runtime: nvidia mem_limit: 4g + # kb#190: `ollama list` just queries the local server's model registry -- + # no model load/inference, cheap. This is a SEPARATE compose project from + # openai/docker-compose.yml (reached from there via + # host.docker.internal:11436), so it cannot be wired into that file's + # depends_on/condition chain -- this only gives it its own status. + healthcheck: + test: ["CMD-SHELL", "ollama list || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 20s diff --git a/overleaf/.gitignore b/overleaf/.gitignore new file mode 100644 index 0000000..ec18e2f --- /dev/null +++ b/overleaf/.gitignore @@ -0,0 +1,16 @@ +# Runtime data and temporary files +data/ +*.bak* + +# Docker compose overrides +docker-compose.override.yml + +# Nginx configuration (only used with NGINX_ENABLED=true) +config/nginx/ + +# TLS certificates (only used with NGINX_ENABLED=true) +config/certs/ + +# Logs and temporary files +logs/ +*.log diff --git a/overleaf/README.md b/overleaf/README.md new file mode 100644 index 0000000..fd34b71 --- /dev/null +++ b/overleaf/README.md @@ -0,0 +1,165 @@ +# Overleaf Service + +Overleaf is an open-source online LaTeX editor. This directory contains the Docker Compose configuration for running Overleaf on Agap. + +## Configuration Files + +- **`.env`** - Docker Compose environment variables (image versions, ports, data paths) +- **`docker-compose.yml`** - Service definitions (Overleaf, MongoDB, Redis) +- **`overleaf.rc`** - Overleaf toolkit configuration (compatibility layer) +- **`variables.env`** - Overleaf application environment variables +- **`version`** - Overleaf image version (6.1.2) + +## Data Directories + +The following directories must exist on the host and have appropriate permissions: + +``` +/mnt/ssd/dbs/overleaf/ +├── data/ # Overleaf application data +├── mongo/ # MongoDB database files +└── redis/ # Redis persistence files +``` + +Create them if they don't exist: +```bash +mkdir -p /mnt/ssd/dbs/overleaf/{data,mongo} +mkdir -p /mnt/ssd/dbs/overleaf/redis +chmod 755 /mnt/ssd/dbs/overleaf/* +``` + +## Quick Start + +From the `agap_git/overleaf/` directory: + +```bash +# Start all services (compose reads .env automatically) +docker compose up -d + +# Check status +docker compose ps + +# View logs +docker compose logs -f sharelatex + +# Stop all services +docker compose down +``` + +## Services + +### sharelatex +- **Image**: `sharelatex/sharelatex:6.1.2` +- **Port**: `127.0.0.1:8089` (localhost only) +- **Data**: `/mnt/ssd/dbs/overleaf/data:/var/lib/overleaf` +- **Features**: + - Sandboxed compiles via Docker sibling containers + - Email disabled by default (see `variables.env`) + - Templates and project files enabled (see `variables.env`) + +### mongo +- **Image**: `mongo:8.0` +- **Port**: `27017` (internal, exposed only to sharelatex) +- **Data**: `/mnt/ssd/dbs/overleaf/mongo:/data/db` +- **Replica Set**: Initialized automatically on first run with `--replSet overleaf` + +### redis +- **Image**: `redis:7.4` +- **Port**: `6379` (internal, exposed only to sharelatex) +- **Data**: `/mnt/ssd/dbs/overleaf/redis:/data` +- **Persistence**: AOF (Append-Only File) enabled + +## Configuration + +### Environment Variables + +Edit `variables.env` to customize Overleaf behavior: + +- `OVERLEAF_APP_NAME` - Display name for the instance +- `ENABLE_CONVERSIONS` - Enable PDF thumbnail generation +- `EMAIL_CONFIRMATION_DISABLED` - Disable email confirmation requirement +- `OVERLEAF_SITE_URL` - Public URL (if behind proxy) +- `OVERLEAF_BEHIND_PROXY` - Set to true if behind reverse proxy +- `OVERLEAF_SECURE_COOKIE` - Use secure cookies when behind TLS proxy + +### Port Binding + +The `OVERLEAF_LISTEN_IP` in `.env` controls which interface Overleaf listens on: +- `127.0.0.1` - Localhost only (default, requires reverse proxy) +- `0.0.0.0` - All interfaces (not recommended without TLS) + +### Storage + +All data is stored on `/mnt/ssd/dbs/overleaf/`: +- Application data (documents, projects) +- MongoDB replica set database +- Redis cache and session data + +## Maintenance + +### Backup + +To back up Overleaf data: + +```bash +# Stop services gracefully +docker compose stop + +# Backup directories +tar czf overleaf-backup-$(date +%Y%m%d).tar.gz /mnt/ssd/dbs/overleaf/ + +# Restart +docker compose up -d +``` + +### Upgrade Image Version + +To upgrade the Overleaf image: + +1. Edit `.env` and update `SHARELATEX_IMAGE` version tag +2. Pull the new image: `docker compose pull` +3. Recreate the service: `docker compose up -d` +4. MongoDB and Redis require no migration for patch/minor version bumps + +### Database Replica Set + +MongoDB is configured with a single-node replica set (`--replSet overleaf`) which is required by Overleaf. If MongoDB fails to initialize: + +```bash +docker compose exec mongo mongosh --eval "rs.initiate({ _id: 'overleaf', members: [ { _id: 0, host: 'mongo:27017' } ] })" +``` + +## Troubleshooting + +### Overleaf won't start +```bash +# Check logs +docker compose logs sharelatex + +# Common issues: +# - MongoDB not ready (check mongo logs) +# - Redis not ready (check redis logs) +# - Data volume permissions (check /mnt/ssd/dbs/overleaf/ permissions) +``` + +### High memory usage +- Redis AOF file can grow; use `BGREWRITEAOF` if needed +- MongoDB maintenance: run `db.collection.reIndex()` for indexed collections + +### Replica set errors in logs +Safe to ignore on first startup; it initializes automatically. If persistent: +```bash +docker compose restart mongo +``` + +## Notes + +- This is Overleaf **Community Edition** (SERVER_PRO=false in overleaf.rc) +- Sibling container sandboxing is enabled but uses single-node mode +- No TLS termination (nginx proxy is disabled); use Caddy or another reverse proxy +- Email is disabled by default; configure SMTP in variables.env to enable + +## References + +- [Overleaf Toolkit Documentation](https://github.com/overleaf/toolkit) +- [Overleaf GitHub Wiki](https://github.com/overleaf/overleaf/wiki) diff --git a/overleaf/docker-compose.yml b/overleaf/docker-compose.yml new file mode 100644 index 0000000..3b7e44a --- /dev/null +++ b/overleaf/docker-compose.yml @@ -0,0 +1,79 @@ +--- +services: + + # MongoDB for Overleaf data storage + mongo: + restart: always + image: "${MONGO_IMAGE}:${MONGO_VERSION}" + command: --replSet overleaf + container_name: mongo + volumes: + - "${MONGO_DATA_PATH}:/data/db" + expose: + - 27017 + healthcheck: + test: echo 'db.stats().ok' | mongosh localhost:27017/test --quiet + interval: 10s + timeout: 10s + retries: 5 + networks: + - overleaf + + # Redis for caching and sessions + redis: + restart: always + image: "${REDIS_IMAGE}" + container_name: redis + command: redis-server --appendonly yes + volumes: + - "${REDIS_DATA_PATH}:/data" + expose: + - 6379 + networks: + - overleaf + + # Overleaf (ShareLaTeX) application + sharelatex: + restart: always + image: "${SHARELATEX_IMAGE}" + container_name: sharelatex + depends_on: + mongo: + condition: service_healthy + redis: + condition: service_started + ports: + - "${OVERLEAF_LISTEN_IP}:${OVERLEAF_PORT}:80" + volumes: + # Data volume + - "${OVERLEAF_DATA_PATH}:/var/lib/overleaf" + # Docker socket for sandboxed compiles + - "${DOCKER_SOCKET_PATH}:/var/run/docker.sock" + environment: + # Connectivity + OVERLEAF_MONGO_URL: "${MONGO_URL}" + OVERLEAF_REDIS_HOST: "${REDIS_HOST}" + OVERLEAF_REDIS_PORT: "${REDIS_PORT}" + + # Docker and compilation settings + DOCKER_RUNNER: 'true' + SANDBOXED_COMPILES: 'true' + SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true' + SANDBOXED_COMPILES_HOST_DIR: "${OVERLEAF_DATA_PATH}/data/compiles" + SYNCTEX_BIN_HOST_PATH: "${OVERLEAF_DATA_PATH}/bin/synctex" + + # Git bridge (disabled for Community Edition) + GIT_BRIDGE_ENABLED: 'false' + + # Load additional environment variables + env_file: + - variables.env + links: + - mongo + - redis + networks: + - overleaf + +networks: + overleaf: + driver: bridge diff --git a/overleaf/overleaf.rc b/overleaf/overleaf.rc new file mode 100644 index 0000000..95af8bd --- /dev/null +++ b/overleaf/overleaf.rc @@ -0,0 +1,47 @@ +#### Overleaf RC #### + +PROJECT_NAME=overleaf + +# Sharelatex container +# Uncomment the OVERLEAF_IMAGE_NAME variable to use a user-defined image. +# OVERLEAF_IMAGE_NAME=sharelatex/sharelatex +OVERLEAF_DATA_PATH=/mnt/ssd/dbs/overleaf/data +SERVER_PRO=false +OVERLEAF_LISTEN_IP=127.0.0.1 +OVERLEAF_PORT=8089 + +# Sibling Containers +SIBLING_CONTAINERS_ENABLED=true +DOCKER_SOCKET_PATH=/var/run/docker.sock + +# Mongo configuration +MONGO_ENABLED=true +MONGO_DATA_PATH=/mnt/ssd/dbs/overleaf/mongo +MONGO_IMAGE=mongo +MONGO_VERSION=8.0 + +# Redis configuration +REDIS_ENABLED=true +REDIS_DATA_PATH=data/redis +REDIS_IMAGE=redis:7.4 +REDIS_AOF_PERSISTENCE=true + +# Git-bridge configuration (Server Pro only) +GIT_BRIDGE_ENABLED=false +GIT_BRIDGE_DATA_PATH=/mnt/ssd/dbs/overleaf/git-bridge + +# TLS proxy configuration (optional) +# See documentation in doc/tls-proxy.md +NGINX_ENABLED=false +NGINX_CONFIG_PATH=config/nginx/nginx.conf +NGINX_HTTP_PORT=80 +# Replace these IP addresses with the external IP address of your host +NGINX_HTTP_LISTEN_IP=127.0.1.1 +NGINX_TLS_LISTEN_IP=127.0.1.1 +TLS_PRIVATE_KEY_PATH=config/nginx/certs/overleaf_key.pem +TLS_CERTIFICATE_PATH=config/nginx/certs/overleaf_certificate.pem +TLS_PORT=443 + +# In Air-gapped setups, skip pulling images +# PULL_BEFORE_UPGRADE=false +# SIBLING_CONTAINERS_PULL=false diff --git a/overleaf/variables.env b/overleaf/variables.env new file mode 100644 index 0000000..2e7aded --- /dev/null +++ b/overleaf/variables.env @@ -0,0 +1,125 @@ +OVERLEAF_APP_NAME="Our Overleaf Instance" + +ENABLED_LINKED_FILE_TYPES=project_file,project_output_file + +# Enables Thumbnail generation using an external converter (pdftocairo by default) +ENABLE_CONVERSIONS=true + +# Disables email confirmation requirement +EMAIL_CONFIRMATION_DISABLED=true + +## Nginx +# NGINX_WORKER_PROCESSES=4 +# NGINX_WORKER_CONNECTIONS=768 + +## Set for TLS via nginx-proxy +# OVERLEAF_BEHIND_PROXY=true +# OVERLEAF_SECURE_COOKIE=true + +# OVERLEAF_SITE_URL=http://overleaf.example.com +# OVERLEAF_NAV_TITLE=Our Overleaf Instance +# OVERLEAF_HEADER_IMAGE_URL=http://somewhere.com/mylogo.png +# OVERLEAF_ADMIN_EMAIL=support@example.com + +# OVERLEAF_LEFT_FOOTER='[{"text": "Contact your support team", "url": "mailto:support@example.com"}]' +# OVERLEAF_RIGHT_FOOTER='[{"text": "Hello, I am on the Right"}]' + +# OVERLEAF_EMAIL_FROM_ADDRESS=team@example.com + +# OVERLEAF_EMAIL_AWS_SES_ACCESS_KEY_ID= +# OVERLEAF_EMAIL_AWS_SES_SECRET_KEY= + +# OVERLEAF_EMAIL_SMTP_HOST=smtp.example.com +# OVERLEAF_EMAIL_SMTP_PORT=587 +# OVERLEAF_EMAIL_SMTP_SECURE=false +# OVERLEAF_EMAIL_SMTP_USER= +# OVERLEAF_EMAIL_SMTP_PASS= +# OVERLEAF_EMAIL_SMTP_NAME= +# OVERLEAF_EMAIL_SMTP_LOGGER=false +# OVERLEAF_EMAIL_SMTP_TLS_REJECT_UNAUTH=true +# OVERLEAF_EMAIL_SMTP_IGNORE_TLS=false +# OVERLEAF_CUSTOM_EMAIL_FOOTER=This system is run by department x + +################ +## Server Pro ## +################ + +EXTERNAL_AUTH=none +# OVERLEAF_LDAP_URL=ldap://ldap:389 +# OVERLEAF_LDAP_SEARCH_BASE=ou=people,dc=planetexpress,dc=com +# OVERLEAF_LDAP_SEARCH_FILTER=(uid={{username}}) +# OVERLEAF_LDAP_BIND_DN=cn=admin,dc=planetexpress,dc=com +# OVERLEAF_LDAP_BIND_CREDENTIALS=GoodNewsEveryone +# OVERLEAF_LDAP_EMAIL_ATT=mail +# OVERLEAF_LDAP_NAME_ATT=cn +# OVERLEAF_LDAP_LAST_NAME_ATT=sn +# OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN=true + +# OVERLEAF_TEMPLATES_USER_ID=578773160210479700917ee5 +# OVERLEAF_NEW_PROJECT_TEMPLATE_LINKS=[{"name":"All Templates","url":"/templates/all"}] + +# TEX_LIVE_DOCKER_IMAGE=quay.io/sharelatex/texlive-full:2022.1 +# ALL_TEX_LIVE_DOCKER_IMAGES=quay.io/sharelatex/texlive-full:2022.1,quay.io/sharelatex/texlive-full:2021.1,quay.io/sharelatex/texlive-full:2020.1 + +# OVERLEAF_PROXY_LEARN=true + +# S3 +# Docs: https://github.com/overleaf/overleaf/wiki/S3 +# ## Enable the s3 backend for filestore +# OVERLEAF_FILESTORE_BACKEND=s3 +# ## Enable S3 backend for history +# OVERLEAF_HISTORY_BACKEND=s3 +# # +# # Pick one of the two sections "AWS S3" or "Self-hosted S3". +# # +# # AWS S3 +# ## Bucket name for project files +# OVERLEAF_FILESTORE_USER_FILES_BUCKET_NAME=overleaf-user-files +# ## Bucket name for template files +# OVERLEAF_FILESTORE_TEMPLATE_FILES_BUCKET_NAME=overleaf-template-files +# ## Key for filestore user +# OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID=... +# ## Secret for filestore user +# OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY=... +# ## Bucket region you picked when creating the buckets. +# OVERLEAF_FILESTORE_S3_REGION="" +# ## Bucket name for project history blobs +# OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET=overleaf-project-blobs +# ## Bucket name for history chunks +# OVERLEAF_HISTORY_CHUNKS_BUCKET=overleaf-chunks +# ## Key for history user +# OVERLEAF_HISTORY_S3_ACCESS_KEY_ID=... +# ## Secret for history user +# OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY=... +# ## Bucket region you picked when creating the buckets. +# OVERLEAF_HISTORY_S3_REGION="" +# +# # Self-hosted S3 +# ## Bucket name for project files +# OVERLEAF_FILESTORE_USER_FILES_BUCKET_NAME=overleaf-user-files +# ## Bucket name for template files +# OVERLEAF_FILESTORE_TEMPLATE_FILES_BUCKET_NAME=overleaf-template-files +# ## Key for filestore user +# OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID=... +# ## Secret for filestore user +# OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY=... +# ## S3 provider endpoint +# OVERLEAF_FILESTORE_S3_ENDPOINT=http://10.10.10.10:9000 +# ## Path style addressing of buckets. Most likely you need to set this to "true". +# OVERLEAF_FILESTORE_S3_PATH_STYLE="true" +# ## Bucket region. Most likely you do not need to configure this. +# OVERLEAF_FILESTORE_S3_REGION="" +# ## Bucket name for project history blobs +# OVERLEAF_HISTORY_PROJECT_BLOBS_BUCKET=overleaf-project-blobs +# ## Bucket name for history chunks +# OVERLEAF_HISTORY_CHUNKS_BUCKET=overleaf-chunks +# ## Key for history user +# OVERLEAF_HISTORY_S3_ACCESS_KEY_ID=... +# ## Secret for history user +# OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY=... +# ## S3 provider endpoint +# OVERLEAF_HISTORY_S3_ENDPOINT=http://10.10.10.10:9000 +# ## Path style addressing of buckets. Most likely you need to set this to "true". +# OVERLEAF_HISTORY_S3_PATH_STYLE="true" +# ## Bucket region. Most likely you do not need to configure this. +# OVERLEAF_HISTORY_S3_REGION="" diff --git a/overleaf/version b/overleaf/version new file mode 100644 index 0000000..5e32542 --- /dev/null +++ b/overleaf/version @@ -0,0 +1 @@ +6.1.2 diff --git a/personal-sensing/.gitignore b/personal-sensing/.gitignore new file mode 100644 index 0000000..f854283 --- /dev/null +++ b/personal-sensing/.gitignore @@ -0,0 +1,7 @@ +.env +__pycache__/ +*.pyc +*.sqlite +*.sqlite-wal +*.sqlite-shm +client_secret*.json diff --git a/personal-sensing/README.md b/personal-sensing/README.md new file mode 100644 index 0000000..2892a88 --- /dev/null +++ b/personal-sensing/README.md @@ -0,0 +1,34 @@ +# Personal Sensing Store + +Source-agnostic local SQLite archive for personal health and activity data. The store schema (`schema.sql`) and storage layer (`src/store.py`) are fed by an ETL pipeline that aggregates data from Home Assistant (Companion sensors, integrations) and Health Connect (Android). This service does not ingest data directly; it only provides the normalized storage layer. Data ingestion is handled by the HA→Agap ETL (see Kanboard #207). + +## Schema + +- **data_points**: Time-series metrics (steps, heart rate, calories, weight, distance, etc.) — one row per (metric, interval, source). +- **sessions**: Workouts and sleep sessions — keyed on stable session ID. +- **sleep_segments**: Sleep stage breakdowns (awake, light, deep, REM, out-of-bed) — one row per stage segment. +- **sync_state**: Incremental-sync cursor per stream — tracks high-water mark for resumable ingestion. +- **ingest_runs**: Audit log of ingestion runs — observability and staleness detection for Zabbix. + +All writes are idempotent UPSERTs keyed on the row's natural identity, so re-runs over overlapping windows are no-ops. + +## Storage Layer + +`src/store.py` provides connection, schema initialization, and read/write functions: + +- `connect(db_path)` — open or create the SQLite database +- `init_db(conn)` — run schema.sql +- `upsert_data_points(conn, rows, source="ha")` — insert/update metric points +- `upsert_sessions(conn, rows, source="ha")` — insert/update sessions +- `upsert_sleep_segments(conn, rows, source="ha")` — insert/update sleep segments +- `daily_metric(conn, metric, days=14)` — aggregate metric by day +- `recent_sessions(conn, days=30, limit=50)` — query recent sessions +- `sleep_by_night(conn, days=14)` — aggregate sleep by stage and night +- `summary(conn)` — compact health snapshot (row counts, coverage, freshness) + +## Sources + +- `ha`: Home Assistant (HA Companion sensors, integrations) +- `takeout`: Legacy import placeholder (unused) + +Default source for new data is `ha`. diff --git a/personal-sensing/schema.sql b/personal-sensing/schema.sql new file mode 100644 index 0000000..4a04ceb --- /dev/null +++ b/personal-sensing/schema.sql @@ -0,0 +1,95 @@ +-- Google Fit local archive — SQLite schema (source-agnostic). +-- +-- Design note: one normalized store serves BOTH ingestion paths — the Google Fit +-- REST API adapter and the Google Takeout importer (deprecation hedge, see README). +-- Time-series points, workout/sleep sessions, and sleep stages each get a table; +-- every write is an idempotent UPSERT keyed on the natural Fit identity of the row, +-- so re-running a sync over an overlapping window never duplicates data. +-- +-- Why SQLite and not InfluxDB: single-user, daily-cadence health data is low volume +-- (thousands of rows/day at most); the Agap/OpenClaw storage doctrine is SQLite-only; +-- and a normalized relational store answers the "sessions + metadata + series" query +-- mix better than a pure TSDB would. Adding an always-on InfluxDB service would be +-- operational cost with no payoff at this scale. (Task listed both as options.) + +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +-- Time-series metric points: steps, heart rate, calories, distance, weight, etc. +-- One row per (metric, interval, source). `metric` is our normalized name +-- (e.g. 'steps', 'heart_rate_avg'), decoupled from Google's data type strings. +CREATE TABLE IF NOT EXISTS data_points ( + metric TEXT NOT NULL, -- normalized: steps, heart_rate_avg, calories, ... + data_type_name TEXT NOT NULL, -- raw Google Fit data type (provenance) + start_ns INTEGER NOT NULL, -- interval start, epoch nanoseconds + end_ns INTEGER NOT NULL, -- interval end, epoch nanoseconds + start_time TEXT NOT NULL, -- ISO8601 UTC (human/SQL friendly) + end_time TEXT NOT NULL, + value_int INTEGER, -- populated for integer metrics + value_float REAL, -- populated for float metrics + value_str TEXT, -- populated for string/enum metrics + unit TEXT, -- count, bpm, kcal, m, kg, min, ... + data_source_id TEXT NOT NULL DEFAULT '', -- originating Fit stream (may be '') + source TEXT NOT NULL DEFAULT 'ha', -- ha | takeout + ingested_at TEXT NOT NULL, + PRIMARY KEY (metric, start_ns, end_ns, data_source_id) +); +CREATE INDEX IF NOT EXISTS idx_dp_metric_time ON data_points (metric, start_ns); +CREATE INDEX IF NOT EXISTS idx_dp_start_time ON data_points (start_time); + +-- Workouts / activities / sleep sessions (com.google.session). +CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, -- Fit session id (stable, upsert key) + name TEXT, + description TEXT, + activity_type INTEGER, -- Fit activity type enum + activity_name TEXT, -- resolved label (e.g. 'Running', 'Sleep') + start_ns INTEGER NOT NULL, + end_ns INTEGER NOT NULL, + start_time TEXT NOT NULL, + end_time TEXT NOT NULL, + modified_ns INTEGER, + application TEXT, -- packageName that wrote the session + source TEXT NOT NULL DEFAULT 'ha', + raw_json TEXT, -- full session payload for reprocessing + ingested_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_sessions_start ON sessions (start_ns); +CREATE INDEX IF NOT EXISTS idx_sessions_type ON sessions (activity_type); + +-- Sleep stage segments (com.google.sleep.segment). Stage enum decoded to a label. +CREATE TABLE IF NOT EXISTS sleep_segments ( + start_ns INTEGER NOT NULL, + end_ns INTEGER NOT NULL, + start_time TEXT NOT NULL, + end_time TEXT NOT NULL, + stage INTEGER NOT NULL, -- raw Fit sleep-stage enum + stage_name TEXT NOT NULL, -- awake, light, deep, rem, out_of_bed, sleep + data_source_id TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT 'ha', + ingested_at TEXT NOT NULL, + PRIMARY KEY (start_ns, end_ns, data_source_id) +); +CREATE INDEX IF NOT EXISTS idx_sleep_start ON sleep_segments (start_ns); + +-- Incremental-sync cursor per stream. `last_synced_ns` is the high-water mark +-- (max end_ns fetched); the next run resumes from there minus a small overlap. +CREATE TABLE IF NOT EXISTS sync_state ( + stream_key TEXT PRIMARY KEY, -- metric name, 'sessions', or 'sleep' + last_synced_ns INTEGER NOT NULL DEFAULT 0, + last_run_at TEXT, + last_status TEXT, -- ok | error + last_error TEXT +); + +-- Audit log of ingestion runs (observability; Zabbix can later read staleness). +CREATE TABLE IF NOT EXISTS ingest_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT, -- ok | error + points_upserted INTEGER DEFAULT 0, + sessions_upserted INTEGER DEFAULT 0, + segments_upserted INTEGER DEFAULT 0, + error TEXT +); diff --git a/personal-sensing/src/__init__.py b/personal-sensing/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/personal-sensing/src/store.py b/personal-sensing/src/store.py new file mode 100644 index 0000000..2b0991f --- /dev/null +++ b/personal-sensing/src/store.py @@ -0,0 +1,237 @@ +"""SQLite storage layer: schema init, idempotent upserts, and read queries. + +Source-agnostic — the Fit REST adapter and the Takeout importer both call the +same upsert_* functions. Reads (summary/series/sessions/sleep) back the Adolf +query CLI. All writes are UPSERTs keyed on the row's natural Fit identity, so a +re-sync over an overlapping window is a no-op rather than a duplicate. +""" +import json +import os +import sqlite3 +from datetime import datetime, timezone + +SCHEMA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "schema.sql") + + +def _now_iso(): + return datetime.now(tz=timezone.utc).isoformat() + + +def connect(db_path): + os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + return conn + + +def init_db(conn): + with open(SCHEMA_PATH) as f: + conn.executescript(f.read()) + conn.commit() + + +# --- writes --------------------------------------------------------------- + +def upsert_data_points(conn, rows, source="ha"): + now = _now_iso() + n = 0 + for r in rows: + conn.execute( + """ + INSERT INTO data_points + (metric, data_type_name, start_ns, end_ns, start_time, end_time, + value_int, value_float, value_str, unit, data_source_id, source, ingested_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(metric, start_ns, end_ns, data_source_id) DO UPDATE SET + value_int=excluded.value_int, + value_float=excluded.value_float, + value_str=excluded.value_str, + unit=excluded.unit, + data_type_name=excluded.data_type_name, + source=excluded.source, + ingested_at=excluded.ingested_at + """, + ( + r["metric"], r["data_type_name"], r["start_ns"], r["end_ns"], + r["start_time"], r["end_time"], r.get("value_int"), r.get("value_float"), + r.get("value_str"), r.get("unit"), r.get("data_source_id", ""), source, now, + ), + ) + n += 1 + conn.commit() + return n + + +def upsert_sleep_segments(conn, rows, source="ha"): + now = _now_iso() + n = 0 + for r in rows: + conn.execute( + """ + INSERT INTO sleep_segments + (start_ns, end_ns, start_time, end_time, stage, stage_name, + data_source_id, source, ingested_at) + VALUES (?,?,?,?,?,?,?,?,?) + ON CONFLICT(start_ns, end_ns, data_source_id) DO UPDATE SET + stage=excluded.stage, + stage_name=excluded.stage_name, + source=excluded.source, + ingested_at=excluded.ingested_at + """, + ( + r["start_ns"], r["end_ns"], r["start_time"], r["end_time"], + r["stage"], r["stage_name"], r.get("data_source_id", ""), source, now, + ), + ) + n += 1 + conn.commit() + return n + + +def upsert_sessions(conn, rows, source="ha"): + now = _now_iso() + n = 0 + for r in rows: + raw = r.get("raw_json") + if raw is not None and not isinstance(raw, str): + raw = json.dumps(raw) + conn.execute( + """ + INSERT INTO sessions + (id, name, description, activity_type, activity_name, start_ns, end_ns, + start_time, end_time, modified_ns, application, source, raw_json, ingested_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET + name=excluded.name, + description=excluded.description, + activity_type=excluded.activity_type, + activity_name=excluded.activity_name, + start_ns=excluded.start_ns, + end_ns=excluded.end_ns, + start_time=excluded.start_time, + end_time=excluded.end_time, + modified_ns=excluded.modified_ns, + application=excluded.application, + source=excluded.source, + raw_json=excluded.raw_json, + ingested_at=excluded.ingested_at + """, + ( + r["id"], r.get("name"), r.get("description"), r.get("activity_type"), + r.get("activity_name"), r["start_ns"], r["end_ns"], r["start_time"], + r["end_time"], r.get("modified_ns"), r.get("application"), source, raw, now, + ), + ) + n += 1 + conn.commit() + return n + + +# --- sync cursor & run audit --------------------------------------------- + +def get_last_synced_ns(conn, stream_key): + row = conn.execute( + "SELECT last_synced_ns FROM sync_state WHERE stream_key=?", (stream_key,) + ).fetchone() + return row["last_synced_ns"] if row else 0 + + +def set_sync_state(conn, stream_key, last_synced_ns, status="ok", error=None): + conn.execute( + """ + INSERT INTO sync_state (stream_key, last_synced_ns, last_run_at, last_status, last_error) + VALUES (?,?,?,?,?) + ON CONFLICT(stream_key) DO UPDATE SET + last_synced_ns=MAX(sync_state.last_synced_ns, excluded.last_synced_ns), + last_run_at=excluded.last_run_at, + last_status=excluded.last_status, + last_error=excluded.last_error + """, + (stream_key, last_synced_ns, _now_iso(), status, error), + ) + conn.commit() + + +def start_run(conn): + cur = conn.execute( + "INSERT INTO ingest_runs (started_at, status) VALUES (?, 'running')", (_now_iso(),) + ) + conn.commit() + return cur.lastrowid + + +def finish_run(conn, run_id, status, points=0, sessions=0, segments=0, error=None): + conn.execute( + """UPDATE ingest_runs SET finished_at=?, status=?, points_upserted=?, + sessions_upserted=?, segments_upserted=?, error=? WHERE id=?""", + (_now_iso(), status, points, sessions, segments, error, run_id), + ) + conn.commit() + + +# --- reads (Adolf query tool) -------------------------------------------- + +def daily_metric(conn, metric, days=14): + return [dict(r) for r in conn.execute( + """SELECT date(start_time) AS day, + SUM(COALESCE(value_int, value_float)) AS total, + MAX(unit) AS unit + FROM data_points WHERE metric=? + AND start_time >= datetime('now', ?) + GROUP BY day ORDER BY day DESC""", + (metric, f"-{int(days)} days"), + ).fetchall()] + + +def latest_metric(conn, metric, limit=20): + return [dict(r) for r in conn.execute( + """SELECT start_time, end_time, value_int, value_float, unit + FROM data_points WHERE metric=? ORDER BY start_ns DESC LIMIT ?""", + (metric, int(limit)), + ).fetchall()] + + +def recent_sessions(conn, days=30, limit=50): + return [dict(r) for r in conn.execute( + """SELECT id, name, activity_type, activity_name, start_time, end_time, + (end_ns - start_ns)/60000000000.0 AS duration_min + FROM sessions WHERE start_time >= datetime('now', ?) + ORDER BY start_ns DESC LIMIT ?""", + (f"-{int(days)} days", int(limit)), + ).fetchall()] + + +def sleep_by_night(conn, days=14): + """Total minutes per stage grouped by the calendar day the sleep segment ends + (a night that crosses midnight is attributed to the wake day).""" + return [dict(r) for r in conn.execute( + """SELECT date(end_time) AS night, stage_name, + SUM((end_ns - start_ns)/60000000000.0) AS minutes + FROM sleep_segments WHERE end_time >= datetime('now', ?) + GROUP BY night, stage_name ORDER BY night DESC""", + (f"-{int(days)} days",), + ).fetchall()] + + +def summary(conn): + """Compact health snapshot for Adolf: row counts, coverage, freshness.""" + out = {} + for name, q in ( + ("data_points", "SELECT COUNT(*) c FROM data_points"), + ("sessions", "SELECT COUNT(*) c FROM sessions"), + ("sleep_segments", "SELECT COUNT(*) c FROM sleep_segments"), + ): + out[name] = conn.execute(q).fetchone()["c"] + out["metrics"] = [r["metric"] for r in conn.execute( + "SELECT DISTINCT metric FROM data_points ORDER BY metric").fetchall()] + span = conn.execute( + "SELECT MIN(start_time) a, MAX(start_time) b FROM data_points").fetchone() + out["coverage"] = {"earliest": span["a"], "latest": span["b"]} + out["sync_state"] = [dict(r) for r in conn.execute( + "SELECT stream_key, last_run_at, last_status FROM sync_state").fetchall()] + last = conn.execute( + "SELECT started_at, finished_at, status, points_upserted FROM ingest_runs " + "ORDER BY id DESC LIMIT 1").fetchone() + out["last_run"] = dict(last) if last else None + return out