b8efe4732d
Accumulated uncommitted infra changes: - Caddyfile: repoint HA/Zabbix to 192.168.1.4/.3, add ~20 new site routes - Immich: move media to /mnt/smsg, enable CUDA ML, mem limits, rewrite backup.sh - Add service stacks: agap-mcp, anki, family, freshrss, iperf3, kanboard, linkwarden, qbittorrent, radicale, syncthing, vikunja, windows - openwebui: enable API keys; ollama: drop CPU fallback - seafile/zabbix: extra_hosts entries; matrix: add user juris - Remove pihole stack and stale wiki/migrate.py - Ignore marketplace-mcp (standalone repo) and linkwarden runtime data Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
459 lines
17 KiB
Python
459 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""OtterWiki → MediaWiki migration script."""
|
||
|
||
import argparse
|
||
import re
|
||
import subprocess
|
||
from pathlib import Path
|
||
|
||
import requests
|
||
|
||
REPO = Path('/mnt/ssd/dbs/otter/app-data/repository')
|
||
API = 'http://localhost:8099/api.php'
|
||
|
||
_FN_DEF = re.compile(r'^\[(\^[^\]]+)\]:\s*(.*)')
|
||
|
||
# Cached pandoc availability (None = not yet checked)
|
||
_PANDOC_AVAILABLE: bool | None = None
|
||
|
||
|
||
def _cap(s: str) -> str:
|
||
"""Title-case: capitalize first letter of each word."""
|
||
return s.title() if s else s
|
||
|
||
|
||
def _pandoc_available() -> bool:
|
||
global _PANDOC_AVAILABLE
|
||
if _PANDOC_AVAILABLE is None:
|
||
try:
|
||
_PANDOC_AVAILABLE = subprocess.run(
|
||
['pandoc', '--version'], capture_output=True
|
||
).returncode == 0
|
||
except FileNotFoundError:
|
||
_PANDOC_AVAILABLE = False
|
||
return _PANDOC_AVAILABLE
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MediaWiki session
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def mw_login(user: str, password: str):
|
||
s = requests.Session()
|
||
r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'type': 'login', 'format': 'json'})
|
||
token = r.json()['query']['tokens']['logintoken']
|
||
s.post(API, data={'action': 'login', 'lgname': user, 'lgpassword': password,
|
||
'lgtoken': token, 'format': 'json'})
|
||
r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'format': 'json'})
|
||
csrf = r.json()['query']['tokens']['csrftoken']
|
||
return s, csrf
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Title determination
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def page_title(md_path: Path) -> str:
|
||
parts = md_path.relative_to(REPO).parts
|
||
if md_path.name == 'home.md' and len(parts) == 1:
|
||
return 'Заглавная страница'
|
||
return _cap(md_path.stem)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Markdown → wikitext conversion
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def convert_pandoc(text: str) -> str:
|
||
return subprocess.run(
|
||
['pandoc', '-f', 'markdown', '-t', 'mediawiki'],
|
||
input=text, capture_output=True, text=True
|
||
).stdout
|
||
|
||
|
||
def convert_python(text: str, skip_info: bool = False) -> str:
|
||
lines = text.split('\n')
|
||
|
||
# Collect footnote definitions in one pass
|
||
footnotes: dict[str, str] = {}
|
||
for line in lines:
|
||
m = _FN_DEF.match(line)
|
||
if m:
|
||
footnotes[m.group(1)] = m.group(2)
|
||
|
||
def replace_fn(m):
|
||
key = m.group(0)[1:-1] # strip outer [ ] to match footnotes dict keys
|
||
content = footnotes.get(key, m.group(0))
|
||
content = re.sub(r'\[([^\]^][^\]]*)\]\(([^)]+)\)', r'[\2 \1]', content)
|
||
return f'<ref>{content}</ref>'
|
||
|
||
out = []
|
||
for line in lines:
|
||
if _FN_DEF.match(line):
|
||
continue
|
||
|
||
m = re.match(r'^(#{1,6})\s+(.*)', line)
|
||
if m:
|
||
eq = '=' * len(m.group(1))
|
||
out.append(f'{eq} {m.group(2)} {eq}')
|
||
continue
|
||
|
||
if re.match(r'^---+$', line.strip()):
|
||
out.append('----')
|
||
continue
|
||
|
||
line = re.sub(r'^(\s*)-(\s)', r'\1*\2', line)
|
||
line = re.sub(r'\*\*\*(.+?)\*\*\*', r"'''''\1'''''", line)
|
||
line = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", line)
|
||
line = re.sub(r'\*(.+?)\*', r"''\1''", line)
|
||
line = re.sub(r'(?<!!)\[([^\]^!][^\]]*)\]\(([^)]+)\)', r'[\2 \1]', line)
|
||
line = re.sub(r'\[\^\S+?\]', replace_fn, line)
|
||
out.append(line)
|
||
|
||
result = convert_tables('\n'.join(out), skip_info=skip_info)
|
||
if footnotes:
|
||
result += '\n<references />'
|
||
return result
|
||
|
||
|
||
def _split_cells(line: str) -> list[str]:
|
||
"""Split a markdown table row on | but not inside [[ ]]."""
|
||
cells = []
|
||
depth = 0
|
||
current = []
|
||
i = 0
|
||
# Strip leading/trailing |
|
||
line = line.strip()
|
||
if line.startswith('|'):
|
||
line = line[1:]
|
||
if line.endswith('|'):
|
||
line = line[:-1]
|
||
while i < len(line):
|
||
if line[i:i+2] == '[[':
|
||
depth += 1
|
||
current.append('[[')
|
||
i += 2
|
||
elif line[i:i+2] == ']]':
|
||
depth -= 1
|
||
current.append(']]')
|
||
i += 2
|
||
elif line[i] == '|' and depth == 0:
|
||
cells.append(''.join(current).strip())
|
||
current = []
|
||
i += 1
|
||
else:
|
||
current.append(line[i])
|
||
i += 1
|
||
cells.append(''.join(current).strip())
|
||
return cells
|
||
|
||
|
||
def convert_tables(text: str, skip_info: bool = False) -> str:
|
||
lines = text.split('\n')
|
||
out = []
|
||
in_table = False
|
||
skip_table = False
|
||
|
||
for line in lines:
|
||
if re.match(r'^\|', line):
|
||
cells = _split_cells(line)
|
||
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
||
# Separator row: start or continue table
|
||
if not in_table:
|
||
header_line = out.pop() if out else ''
|
||
hcells = _split_cells(header_line)
|
||
# Blank-header table (all-empty header cells) = OtterWiki info table
|
||
if skip_info and all(c == '' for c in hcells):
|
||
skip_table = True
|
||
in_table = True
|
||
else:
|
||
out += ['{| class="wikitable"', '|-', '! ' + ' !! '.join(hcells)]
|
||
in_table = True
|
||
skip_table = False
|
||
if not skip_table:
|
||
out.append('|-')
|
||
else:
|
||
if in_table:
|
||
if not skip_table:
|
||
out.append('|-')
|
||
out.append('| ' + ' || '.join(cells))
|
||
# else: skip info table row
|
||
else:
|
||
out.append(line)
|
||
else:
|
||
if in_table:
|
||
if not skip_table:
|
||
out.append('|}')
|
||
in_table = False
|
||
skip_table = False
|
||
out.append(line)
|
||
|
||
if in_table and not skip_table:
|
||
out.append('|}')
|
||
|
||
return '\n'.join(out)
|
||
|
||
|
||
_PERSON_FIELD_MAP = {
|
||
'родился': 'родился', 'родилась': 'родился',
|
||
'умер': 'умер', 'умерла': 'умер',
|
||
'отец': 'отец', 'мать': 'мать',
|
||
'супруг': 'супруг', 'супруга': 'супруг', 'муж': 'супруг', 'жена': 'супруг',
|
||
'дети': 'дети', 'ребёнок': 'дети',
|
||
'братья': 'братья', 'брат': 'братья', 'сестра': 'братья',
|
||
'сёстры': 'братья', 'сестры': 'братья',
|
||
'место рождения': 'место_рождения', 'место_рождения': 'место_рождения',
|
||
'прочее': 'прочее',
|
||
}
|
||
_PERSONA_PARAM_ORDER = ['родился', 'место_рождения', 'умер', 'отец', 'мать', 'супруг', 'дети', 'братья', 'прочее']
|
||
|
||
_PLACE_FIELD_MAP = {
|
||
'тип': 'тип',
|
||
'статус': 'статус',
|
||
'страна': 'страна',
|
||
'регион': 'регион', 'область': 'регион',
|
||
'район': 'район', 'расположение': 'район', 'самоуправление': 'район',
|
||
'река': 'река',
|
||
'основана': 'основана', 'основан': 'основана',
|
||
'население': 'население',
|
||
'адрес': 'адрес',
|
||
'период': 'период', 'годы': 'период',
|
||
'жильцы': 'жильцы', 'жильцы/семья': 'жильцы', 'семейное имя': 'прочее',
|
||
'латв. название': 'назв_латыш',
|
||
'белор. название': 'назв_белор',
|
||
'координаты': 'координаты',
|
||
'сайт': 'сайт',
|
||
'телефон': 'телефон', 'email': 'телефон',
|
||
'полное название': 'прочее', 'классы': 'прочее',
|
||
'штаб-квартира': 'прочее', 'сотрудников': 'прочее',
|
||
}
|
||
_PLACE_PARAM_ORDER = ['тип', 'статус', 'страна', 'регион', 'район', 'река', 'основана',
|
||
'население', 'адрес', 'период', 'жильцы', 'назв_латыш', 'назв_белор',
|
||
'координаты', 'сайт', 'телефон', 'прочее']
|
||
|
||
|
||
def _extract_infobox(text: str, title: str, name_param: str, template: str,
|
||
field_map: dict, param_order: list) -> tuple[str, str]:
|
||
"""Extract photo + blank-header info table; return ({{Template|...}}, cleaned_text)."""
|
||
lines = text.split('\n')
|
||
photo = None
|
||
fields: dict[str, str] = {}
|
||
remove: set[int] = set()
|
||
|
||
for i, line in enumerate(lines):
|
||
if re.match(r'^\|', line):
|
||
break # reached info table — stop looking for photo
|
||
m = re.match(r'^\s*\[!\[[^\]]*\]\(\./(?:[^/)]+/)?([^)?]+?)(?:\?[^)]*)?\)\]', line)
|
||
if m:
|
||
photo = m.group(1)
|
||
remove.add(i)
|
||
break
|
||
|
||
in_table = False
|
||
for i, line in enumerate(lines):
|
||
if re.match(r'^\|', line):
|
||
cells = _split_cells(line)
|
||
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
||
if not in_table:
|
||
prev = i - 1
|
||
if prev >= 0 and re.match(r'^\|', lines[prev]):
|
||
hcells = _split_cells(lines[prev])
|
||
if all(c == '' for c in hcells):
|
||
in_table = True
|
||
remove.add(prev)
|
||
if in_table:
|
||
remove.add(i)
|
||
elif in_table:
|
||
remove.add(i)
|
||
if len(cells) >= 2:
|
||
key = re.sub(r'\*\*(.+?)\*\*', r'\1', cells[0]).strip().lower()
|
||
val = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", cells[1].strip())
|
||
param = field_map.get(key)
|
||
if param and param not in fields:
|
||
fields[param] = val
|
||
elif in_table:
|
||
in_table = False
|
||
|
||
if not photo and not fields:
|
||
return '', text
|
||
|
||
parts = ['{{' + template, f'| {name_param:<16} = {title}']
|
||
if photo:
|
||
parts.append(f'| фото = {photo}')
|
||
for param in param_order:
|
||
if param in fields:
|
||
parts.append(f'| {param:<16} = {fields[param]}')
|
||
infobox = '\n'.join(parts) + '\n}}'
|
||
cleaned = '\n'.join(line for i, line in enumerate(lines) if i not in remove)
|
||
return infobox, cleaned
|
||
|
||
|
||
def extract_person_infobox(text: str, title: str) -> tuple[str, str]:
|
||
return _extract_infobox(text, title, 'имя', 'Персона', _PERSON_FIELD_MAP, _PERSONA_PARAM_ORDER)
|
||
|
||
|
||
def extract_place_infobox(text: str, title: str) -> tuple[str, str]:
|
||
return _extract_infobox(text, title, 'название', 'Место', _PLACE_FIELD_MAP, _PLACE_PARAM_ORDER)
|
||
|
||
|
||
def strip_first_heading(text: str) -> str:
|
||
"""Remove the first H1 line — MW displays the page title itself."""
|
||
return re.sub(r'^#[^#][^\n]*\n?', '', text, count=1)
|
||
|
||
|
||
def convert(text: str, skip_info: bool = False, is_place: bool = False, title: str = '') -> str:
|
||
text = strip_first_heading(text)
|
||
infobox = ''
|
||
if skip_info:
|
||
infobox, text = extract_person_infobox(text, title)
|
||
elif is_place:
|
||
infobox, text = extract_place_infobox(text, title)
|
||
result = convert_pandoc(text) if _pandoc_available() else convert_python(text, skip_info=skip_info)
|
||
if infobox:
|
||
result = infobox + '\n' + result.lstrip('\n')
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Post-processing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def fix_links(text: str) -> str:
|
||
pattern = (r'\[\[([^\]|]+)\|'
|
||
r'(?:Люди|Места|Воспоминания|люди|места|воспоминания|Место)'
|
||
r'/([^\]]+)\]\]')
|
||
|
||
def replace_link(m):
|
||
display = m.group(1).strip()
|
||
page = _cap(m.group(2).strip().lower())
|
||
if display.lower() == page.lower():
|
||
return f'[[{page}]]'
|
||
return f'[[{page}|{display}]]'
|
||
|
||
text = re.sub(pattern, replace_link, text)
|
||
# Also handle bare section paths: [[Section/PageName]] → [[PageName]]
|
||
text = re.sub(
|
||
r'\[\[(?:Люди|Места|Воспоминания|люди|места|воспоминания|Место)/([^\]|]+)\]\]',
|
||
lambda m: f'[[{m.group(1).strip().lower().title()}]]',
|
||
text
|
||
)
|
||
return text
|
||
|
||
|
||
def fix_images(text: str) -> str:
|
||
# Handle linked images: [](./file.jpg)
|
||
# and plain images: 
|
||
pattern = r'(?:\[)?!\[[^\]]*\]\(\./(?:[^/)]+/)?([^)?]+?)(\?[^)]*)?\)(?:\]\([^)]*\))?'
|
||
|
||
def replace_img(m):
|
||
filename = m.group(1)
|
||
size_m = re.search(r'thumbnail=(\d+)', m.group(2) or '')
|
||
return f'[[File:{filename}|{size_m.group(1)}px]]' if size_m else f'[[File:{filename}]]'
|
||
|
||
return re.sub(pattern, replace_img, text)
|
||
|
||
|
||
_CATEGORY_MAP = {'люди': 'Люди', 'места': 'Места', 'воспоминания': 'Воспоминания'}
|
||
|
||
|
||
def category_suffix(md_path: Path) -> str:
|
||
parts = md_path.relative_to(REPO).parts
|
||
if len(parts) == 1:
|
||
return '' if md_path.name == 'home.md' else '\n\n[[Category:Статьи]]'
|
||
cat = _CATEGORY_MAP.get(parts[0].lower())
|
||
return f'\n\n[[Category:{cat}]]' if cat else ''
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MW operations
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def post_page(session, csrf: str, title: str, text: str, dry_run: bool) -> bool:
|
||
if dry_run:
|
||
print(f'[DRY] {title}')
|
||
return True
|
||
data = session.post(API, data={
|
||
'action': 'edit', 'title': title, 'text': text,
|
||
'token': csrf, 'format': 'json'
|
||
}).json()
|
||
if 'error' in data:
|
||
print(f'[ERR] {title}: {data["error"].get("info", data["error"])}')
|
||
return False
|
||
print(f'[OK] {title}')
|
||
return True
|
||
|
||
|
||
def upload_image(session, csrf: str, image_path: Path, dry_run: bool) -> bool:
|
||
basename = image_path.name
|
||
if dry_run:
|
||
print(f'[DRY] File:{basename}')
|
||
return True
|
||
with open(image_path, 'rb') as f:
|
||
data = session.post(API, data={
|
||
'action': 'upload', 'filename': basename,
|
||
'token': csrf, 'format': 'json', 'ignorewarnings': '1'
|
||
}, files={'file': f}).json()
|
||
if 'error' in data:
|
||
print(f'[ERR] File:{basename}: {data["error"].get("info", data["error"])}')
|
||
return False
|
||
if data.get('upload', {}).get('result') == 'Success':
|
||
print(f'[OK] File:{basename}')
|
||
else:
|
||
print(f'[SKIP] File:{basename} (already exists or no change)')
|
||
return True
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def collect_images() -> list[Path]:
|
||
images = []
|
||
for folder in ('люди', 'места'):
|
||
p = REPO / folder
|
||
if p.exists():
|
||
for ext in ('*.jpg', '*.jpeg', '*.JPG', '*.JPEG', '*.png', '*.PNG'):
|
||
images.extend(p.rglob(ext))
|
||
return images
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='Migrate OtterWiki to MediaWiki')
|
||
parser.add_argument('--user', required=True)
|
||
parser.add_argument('--password', required=True)
|
||
parser.add_argument('--dry-run', action='store_true')
|
||
args = parser.parse_args()
|
||
|
||
session = csrf = None
|
||
if not args.dry_run:
|
||
session, csrf = mw_login(args.user, args.password)
|
||
|
||
pages_ok = pages_err = images_ok = images_err = 0
|
||
|
||
for md_path in sorted(REPO.rglob('*.md')):
|
||
title = page_title(md_path)
|
||
raw = md_path.read_text(encoding='utf-8')
|
||
parts = md_path.relative_to(REPO).parts
|
||
is_people = len(parts) > 0 and parts[0].lower() == 'люди'
|
||
is_place = len(parts) > 0 and parts[0].lower() == 'места'
|
||
wikitext = convert(raw, skip_info=is_people, is_place=is_place, title=title)
|
||
wikitext = fix_links(wikitext)
|
||
wikitext = fix_images(wikitext)
|
||
wikitext += category_suffix(md_path)
|
||
if post_page(session, csrf, title, wikitext, args.dry_run):
|
||
pages_ok += 1
|
||
else:
|
||
pages_err += 1
|
||
|
||
for img in collect_images():
|
||
if upload_image(session, csrf, img, args.dry_run):
|
||
images_ok += 1
|
||
else:
|
||
images_err += 1
|
||
|
||
print(f'\nDone: {pages_ok} pages, {images_ok} images, {pages_err + images_err} errors')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|