Sync infra config: HA/Zabbix relocation, Immich storage move, new services
Accumulated uncommitted infra changes: - Caddyfile: repoint HA/Zabbix to 192.168.1.4/.3, add ~20 new site routes - Immich: move media to /mnt/smsg, enable CUDA ML, mem limits, rewrite backup.sh - Add service stacks: agap-mcp, anki, family, freshrss, iperf3, kanboard, linkwarden, qbittorrent, radicale, syncthing, vikunja, windows - openwebui: enable API keys; ollama: drop CPU fallback - seafile/zabbix: extra_hosts entries; matrix: add user juris - Remove pihole stack and stale wiki/migrate.py - Ignore marketplace-mcp (standalone repo) and linkwarden runtime data Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
Alvis
458
family/migrate.py
Normal file
458
family/migrate.py
Normal file
@@ -0,0 +1,458 @@
|
||||
#!/usr/bin/env python3
|
||||
"""OtterWiki → MediaWiki migration script."""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
REPO = Path('/mnt/ssd/dbs/otter/app-data/repository')
|
||||
API = 'http://localhost:8099/api.php'
|
||||
|
||||
_FN_DEF = re.compile(r'^\[(\^[^\]]+)\]:\s*(.*)')
|
||||
|
||||
# Cached pandoc availability (None = not yet checked)
|
||||
_PANDOC_AVAILABLE: bool | None = None
|
||||
|
||||
|
||||
def _cap(s: str) -> str:
|
||||
"""Title-case: capitalize first letter of each word."""
|
||||
return s.title() if s else s
|
||||
|
||||
|
||||
def _pandoc_available() -> bool:
|
||||
global _PANDOC_AVAILABLE
|
||||
if _PANDOC_AVAILABLE is None:
|
||||
try:
|
||||
_PANDOC_AVAILABLE = subprocess.run(
|
||||
['pandoc', '--version'], capture_output=True
|
||||
).returncode == 0
|
||||
except FileNotFoundError:
|
||||
_PANDOC_AVAILABLE = False
|
||||
return _PANDOC_AVAILABLE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MediaWiki session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def mw_login(user: str, password: str):
|
||||
s = requests.Session()
|
||||
r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'type': 'login', 'format': 'json'})
|
||||
token = r.json()['query']['tokens']['logintoken']
|
||||
s.post(API, data={'action': 'login', 'lgname': user, 'lgpassword': password,
|
||||
'lgtoken': token, 'format': 'json'})
|
||||
r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'format': 'json'})
|
||||
csrf = r.json()['query']['tokens']['csrftoken']
|
||||
return s, csrf
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Title determination
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def page_title(md_path: Path) -> str:
|
||||
parts = md_path.relative_to(REPO).parts
|
||||
if md_path.name == 'home.md' and len(parts) == 1:
|
||||
return 'Заглавная страница'
|
||||
return _cap(md_path.stem)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown → wikitext conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def convert_pandoc(text: str) -> str:
|
||||
return subprocess.run(
|
||||
['pandoc', '-f', 'markdown', '-t', 'mediawiki'],
|
||||
input=text, capture_output=True, text=True
|
||||
).stdout
|
||||
|
||||
|
||||
def convert_python(text: str, skip_info: bool = False) -> str:
|
||||
lines = text.split('\n')
|
||||
|
||||
# Collect footnote definitions in one pass
|
||||
footnotes: dict[str, str] = {}
|
||||
for line in lines:
|
||||
m = _FN_DEF.match(line)
|
||||
if m:
|
||||
footnotes[m.group(1)] = m.group(2)
|
||||
|
||||
def replace_fn(m):
|
||||
key = m.group(0)[1:-1] # strip outer [ ] to match footnotes dict keys
|
||||
content = footnotes.get(key, m.group(0))
|
||||
content = re.sub(r'\[([^\]^][^\]]*)\]\(([^)]+)\)', r'[\2 \1]', content)
|
||||
return f'<ref>{content}</ref>'
|
||||
|
||||
out = []
|
||||
for line in lines:
|
||||
if _FN_DEF.match(line):
|
||||
continue
|
||||
|
||||
m = re.match(r'^(#{1,6})\s+(.*)', line)
|
||||
if m:
|
||||
eq = '=' * len(m.group(1))
|
||||
out.append(f'{eq} {m.group(2)} {eq}')
|
||||
continue
|
||||
|
||||
if re.match(r'^---+$', line.strip()):
|
||||
out.append('----')
|
||||
continue
|
||||
|
||||
line = re.sub(r'^(\s*)-(\s)', r'\1*\2', line)
|
||||
line = re.sub(r'\*\*\*(.+?)\*\*\*', r"'''''\1'''''", line)
|
||||
line = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", line)
|
||||
line = re.sub(r'\*(.+?)\*', r"''\1''", line)
|
||||
line = re.sub(r'(?<!!)\[([^\]^!][^\]]*)\]\(([^)]+)\)', r'[\2 \1]', line)
|
||||
line = re.sub(r'\[\^\S+?\]', replace_fn, line)
|
||||
out.append(line)
|
||||
|
||||
result = convert_tables('\n'.join(out), skip_info=skip_info)
|
||||
if footnotes:
|
||||
result += '\n<references />'
|
||||
return result
|
||||
|
||||
|
||||
def _split_cells(line: str) -> list[str]:
|
||||
"""Split a markdown table row on | but not inside [[ ]]."""
|
||||
cells = []
|
||||
depth = 0
|
||||
current = []
|
||||
i = 0
|
||||
# Strip leading/trailing |
|
||||
line = line.strip()
|
||||
if line.startswith('|'):
|
||||
line = line[1:]
|
||||
if line.endswith('|'):
|
||||
line = line[:-1]
|
||||
while i < len(line):
|
||||
if line[i:i+2] == '[[':
|
||||
depth += 1
|
||||
current.append('[[')
|
||||
i += 2
|
||||
elif line[i:i+2] == ']]':
|
||||
depth -= 1
|
||||
current.append(']]')
|
||||
i += 2
|
||||
elif line[i] == '|' and depth == 0:
|
||||
cells.append(''.join(current).strip())
|
||||
current = []
|
||||
i += 1
|
||||
else:
|
||||
current.append(line[i])
|
||||
i += 1
|
||||
cells.append(''.join(current).strip())
|
||||
return cells
|
||||
|
||||
|
||||
def convert_tables(text: str, skip_info: bool = False) -> str:
|
||||
lines = text.split('\n')
|
||||
out = []
|
||||
in_table = False
|
||||
skip_table = False
|
||||
|
||||
for line in lines:
|
||||
if re.match(r'^\|', line):
|
||||
cells = _split_cells(line)
|
||||
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
||||
# Separator row: start or continue table
|
||||
if not in_table:
|
||||
header_line = out.pop() if out else ''
|
||||
hcells = _split_cells(header_line)
|
||||
# Blank-header table (all-empty header cells) = OtterWiki info table
|
||||
if skip_info and all(c == '' for c in hcells):
|
||||
skip_table = True
|
||||
in_table = True
|
||||
else:
|
||||
out += ['{| class="wikitable"', '|-', '! ' + ' !! '.join(hcells)]
|
||||
in_table = True
|
||||
skip_table = False
|
||||
if not skip_table:
|
||||
out.append('|-')
|
||||
else:
|
||||
if in_table:
|
||||
if not skip_table:
|
||||
out.append('|-')
|
||||
out.append('| ' + ' || '.join(cells))
|
||||
# else: skip info table row
|
||||
else:
|
||||
out.append(line)
|
||||
else:
|
||||
if in_table:
|
||||
if not skip_table:
|
||||
out.append('|}')
|
||||
in_table = False
|
||||
skip_table = False
|
||||
out.append(line)
|
||||
|
||||
if in_table and not skip_table:
|
||||
out.append('|}')
|
||||
|
||||
return '\n'.join(out)
|
||||
|
||||
|
||||
_PERSON_FIELD_MAP = {
|
||||
'родился': 'родился', 'родилась': 'родился',
|
||||
'умер': 'умер', 'умерла': 'умер',
|
||||
'отец': 'отец', 'мать': 'мать',
|
||||
'супруг': 'супруг', 'супруга': 'супруг', 'муж': 'супруг', 'жена': 'супруг',
|
||||
'дети': 'дети', 'ребёнок': 'дети',
|
||||
'братья': 'братья', 'брат': 'братья', 'сестра': 'братья',
|
||||
'сёстры': 'братья', 'сестры': 'братья',
|
||||
'место рождения': 'место_рождения', 'место_рождения': 'место_рождения',
|
||||
'прочее': 'прочее',
|
||||
}
|
||||
_PERSONA_PARAM_ORDER = ['родился', 'место_рождения', 'умер', 'отец', 'мать', 'супруг', 'дети', 'братья', 'прочее']
|
||||
|
||||
_PLACE_FIELD_MAP = {
|
||||
'тип': 'тип',
|
||||
'статус': 'статус',
|
||||
'страна': 'страна',
|
||||
'регион': 'регион', 'область': 'регион',
|
||||
'район': 'район', 'расположение': 'район', 'самоуправление': 'район',
|
||||
'река': 'река',
|
||||
'основана': 'основана', 'основан': 'основана',
|
||||
'население': 'население',
|
||||
'адрес': 'адрес',
|
||||
'период': 'период', 'годы': 'период',
|
||||
'жильцы': 'жильцы', 'жильцы/семья': 'жильцы', 'семейное имя': 'прочее',
|
||||
'латв. название': 'назв_латыш',
|
||||
'белор. название': 'назв_белор',
|
||||
'координаты': 'координаты',
|
||||
'сайт': 'сайт',
|
||||
'телефон': 'телефон', 'email': 'телефон',
|
||||
'полное название': 'прочее', 'классы': 'прочее',
|
||||
'штаб-квартира': 'прочее', 'сотрудников': 'прочее',
|
||||
}
|
||||
_PLACE_PARAM_ORDER = ['тип', 'статус', 'страна', 'регион', 'район', 'река', 'основана',
|
||||
'население', 'адрес', 'период', 'жильцы', 'назв_латыш', 'назв_белор',
|
||||
'координаты', 'сайт', 'телефон', 'прочее']
|
||||
|
||||
|
||||
def _extract_infobox(text: str, title: str, name_param: str, template: str,
|
||||
field_map: dict, param_order: list) -> tuple[str, str]:
|
||||
"""Extract photo + blank-header info table; return ({{Template|...}}, cleaned_text)."""
|
||||
lines = text.split('\n')
|
||||
photo = None
|
||||
fields: dict[str, str] = {}
|
||||
remove: set[int] = set()
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if re.match(r'^\|', line):
|
||||
break # reached info table — stop looking for photo
|
||||
m = re.match(r'^\s*\[!\[[^\]]*\]\(\./(?:[^/)]+/)?([^)?]+?)(?:\?[^)]*)?\)\]', line)
|
||||
if m:
|
||||
photo = m.group(1)
|
||||
remove.add(i)
|
||||
break
|
||||
|
||||
in_table = False
|
||||
for i, line in enumerate(lines):
|
||||
if re.match(r'^\|', line):
|
||||
cells = _split_cells(line)
|
||||
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
||||
if not in_table:
|
||||
prev = i - 1
|
||||
if prev >= 0 and re.match(r'^\|', lines[prev]):
|
||||
hcells = _split_cells(lines[prev])
|
||||
if all(c == '' for c in hcells):
|
||||
in_table = True
|
||||
remove.add(prev)
|
||||
if in_table:
|
||||
remove.add(i)
|
||||
elif in_table:
|
||||
remove.add(i)
|
||||
if len(cells) >= 2:
|
||||
key = re.sub(r'\*\*(.+?)\*\*', r'\1', cells[0]).strip().lower()
|
||||
val = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", cells[1].strip())
|
||||
param = field_map.get(key)
|
||||
if param and param not in fields:
|
||||
fields[param] = val
|
||||
elif in_table:
|
||||
in_table = False
|
||||
|
||||
if not photo and not fields:
|
||||
return '', text
|
||||
|
||||
parts = ['{{' + template, f'| {name_param:<16} = {title}']
|
||||
if photo:
|
||||
parts.append(f'| фото = {photo}')
|
||||
for param in param_order:
|
||||
if param in fields:
|
||||
parts.append(f'| {param:<16} = {fields[param]}')
|
||||
infobox = '\n'.join(parts) + '\n}}'
|
||||
cleaned = '\n'.join(line for i, line in enumerate(lines) if i not in remove)
|
||||
return infobox, cleaned
|
||||
|
||||
|
||||
def extract_person_infobox(text: str, title: str) -> tuple[str, str]:
|
||||
return _extract_infobox(text, title, 'имя', 'Персона', _PERSON_FIELD_MAP, _PERSONA_PARAM_ORDER)
|
||||
|
||||
|
||||
def extract_place_infobox(text: str, title: str) -> tuple[str, str]:
|
||||
return _extract_infobox(text, title, 'название', 'Место', _PLACE_FIELD_MAP, _PLACE_PARAM_ORDER)
|
||||
|
||||
|
||||
def strip_first_heading(text: str) -> str:
|
||||
"""Remove the first H1 line — MW displays the page title itself."""
|
||||
return re.sub(r'^#[^#][^\n]*\n?', '', text, count=1)
|
||||
|
||||
|
||||
def convert(text: str, skip_info: bool = False, is_place: bool = False, title: str = '') -> str:
|
||||
text = strip_first_heading(text)
|
||||
infobox = ''
|
||||
if skip_info:
|
||||
infobox, text = extract_person_infobox(text, title)
|
||||
elif is_place:
|
||||
infobox, text = extract_place_infobox(text, title)
|
||||
result = convert_pandoc(text) if _pandoc_available() else convert_python(text, skip_info=skip_info)
|
||||
if infobox:
|
||||
result = infobox + '\n' + result.lstrip('\n')
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Post-processing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def fix_links(text: str) -> str:
|
||||
pattern = (r'\[\[([^\]|]+)\|'
|
||||
r'(?:Люди|Места|Воспоминания|люди|места|воспоминания|Место)'
|
||||
r'/([^\]]+)\]\]')
|
||||
|
||||
def replace_link(m):
|
||||
display = m.group(1).strip()
|
||||
page = _cap(m.group(2).strip().lower())
|
||||
if display.lower() == page.lower():
|
||||
return f'[[{page}]]'
|
||||
return f'[[{page}|{display}]]'
|
||||
|
||||
text = re.sub(pattern, replace_link, text)
|
||||
# Also handle bare section paths: [[Section/PageName]] → [[PageName]]
|
||||
text = re.sub(
|
||||
r'\[\[(?:Люди|Места|Воспоминания|люди|места|воспоминания|Место)/([^\]|]+)\]\]',
|
||||
lambda m: f'[[{m.group(1).strip().lower().title()}]]',
|
||||
text
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def fix_images(text: str) -> str:
|
||||
# Handle linked images: [](./file.jpg)
|
||||
# and plain images: 
|
||||
pattern = r'(?:\[)?!\[[^\]]*\]\(\./(?:[^/)]+/)?([^)?]+?)(\?[^)]*)?\)(?:\]\([^)]*\))?'
|
||||
|
||||
def replace_img(m):
|
||||
filename = m.group(1)
|
||||
size_m = re.search(r'thumbnail=(\d+)', m.group(2) or '')
|
||||
return f'[[File:{filename}|{size_m.group(1)}px]]' if size_m else f'[[File:{filename}]]'
|
||||
|
||||
return re.sub(pattern, replace_img, text)
|
||||
|
||||
|
||||
_CATEGORY_MAP = {'люди': 'Люди', 'места': 'Места', 'воспоминания': 'Воспоминания'}
|
||||
|
||||
|
||||
def category_suffix(md_path: Path) -> str:
|
||||
parts = md_path.relative_to(REPO).parts
|
||||
if len(parts) == 1:
|
||||
return '' if md_path.name == 'home.md' else '\n\n[[Category:Статьи]]'
|
||||
cat = _CATEGORY_MAP.get(parts[0].lower())
|
||||
return f'\n\n[[Category:{cat}]]' if cat else ''
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MW operations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def post_page(session, csrf: str, title: str, text: str, dry_run: bool) -> bool:
|
||||
if dry_run:
|
||||
print(f'[DRY] {title}')
|
||||
return True
|
||||
data = session.post(API, data={
|
||||
'action': 'edit', 'title': title, 'text': text,
|
||||
'token': csrf, 'format': 'json'
|
||||
}).json()
|
||||
if 'error' in data:
|
||||
print(f'[ERR] {title}: {data["error"].get("info", data["error"])}')
|
||||
return False
|
||||
print(f'[OK] {title}')
|
||||
return True
|
||||
|
||||
|
||||
def upload_image(session, csrf: str, image_path: Path, dry_run: bool) -> bool:
|
||||
basename = image_path.name
|
||||
if dry_run:
|
||||
print(f'[DRY] File:{basename}')
|
||||
return True
|
||||
with open(image_path, 'rb') as f:
|
||||
data = session.post(API, data={
|
||||
'action': 'upload', 'filename': basename,
|
||||
'token': csrf, 'format': 'json', 'ignorewarnings': '1'
|
||||
}, files={'file': f}).json()
|
||||
if 'error' in data:
|
||||
print(f'[ERR] File:{basename}: {data["error"].get("info", data["error"])}')
|
||||
return False
|
||||
if data.get('upload', {}).get('result') == 'Success':
|
||||
print(f'[OK] File:{basename}')
|
||||
else:
|
||||
print(f'[SKIP] File:{basename} (already exists or no change)')
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collect_images() -> list[Path]:
|
||||
images = []
|
||||
for folder in ('люди', 'места'):
|
||||
p = REPO / folder
|
||||
if p.exists():
|
||||
for ext in ('*.jpg', '*.jpeg', '*.JPG', '*.JPEG', '*.png', '*.PNG'):
|
||||
images.extend(p.rglob(ext))
|
||||
return images
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Migrate OtterWiki to MediaWiki')
|
||||
parser.add_argument('--user', required=True)
|
||||
parser.add_argument('--password', required=True)
|
||||
parser.add_argument('--dry-run', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
session = csrf = None
|
||||
if not args.dry_run:
|
||||
session, csrf = mw_login(args.user, args.password)
|
||||
|
||||
pages_ok = pages_err = images_ok = images_err = 0
|
||||
|
||||
for md_path in sorted(REPO.rglob('*.md')):
|
||||
title = page_title(md_path)
|
||||
raw = md_path.read_text(encoding='utf-8')
|
||||
parts = md_path.relative_to(REPO).parts
|
||||
is_people = len(parts) > 0 and parts[0].lower() == 'люди'
|
||||
is_place = len(parts) > 0 and parts[0].lower() == 'места'
|
||||
wikitext = convert(raw, skip_info=is_people, is_place=is_place, title=title)
|
||||
wikitext = fix_links(wikitext)
|
||||
wikitext = fix_images(wikitext)
|
||||
wikitext += category_suffix(md_path)
|
||||
if post_page(session, csrf, title, wikitext, args.dry_run):
|
||||
pages_ok += 1
|
||||
else:
|
||||
pages_err += 1
|
||||
|
||||
for img in collect_images():
|
||||
if upload_image(session, csrf, img, args.dry_run):
|
||||
images_ok += 1
|
||||
else:
|
||||
images_err += 1
|
||||
|
||||
print(f'\nDone: {pages_ok} pages, {images_ok} images, {pages_err + images_err} errors')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user