From 52190b63b8d3577d6b7f63cfe2b10dffb7f6fc0d Mon Sep 17 00:00:00 2001 From: Alvis Date: Fri, 3 Apr 2026 08:41:37 +0000 Subject: [PATCH] =?UTF-8?q?Add=20OtterWiki=E2=86=92MediaWiki=20migration?= =?UTF-8?q?=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- wiki/migrate.py | 277 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 wiki/migrate.py diff --git a/wiki/migrate.py b/wiki/migrate.py new file mode 100644 index 0000000..ea20581 --- /dev/null +++ b/wiki/migrate.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""OtterWiki → MediaWiki migration script.""" + +import argparse +import re +import subprocess +from pathlib import Path + +import requests + +REPO = Path('/mnt/ssd/dbs/otter/app-data/repository') +API = 'http://localhost:8099/api.php' + +_FN_DEF = re.compile(r'^\[(\^[^\]]+)\]:\s*(.*)') + +# Cached pandoc availability (None = not yet checked) +_PANDOC_AVAILABLE: bool | None = None + + +def _cap(s: str) -> str: + """Capitalize first character, leave rest unchanged.""" + return s[0].upper() + s[1:] if s else s + + +def _pandoc_available() -> bool: + global _PANDOC_AVAILABLE + if _PANDOC_AVAILABLE is None: + try: + _PANDOC_AVAILABLE = subprocess.run( + ['pandoc', '--version'], capture_output=True + ).returncode == 0 + except FileNotFoundError: + _PANDOC_AVAILABLE = False + return _PANDOC_AVAILABLE + + +# --------------------------------------------------------------------------- +# MediaWiki session +# --------------------------------------------------------------------------- + +def mw_login(user: str, password: str): + s = requests.Session() + r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'type': 'login', 'format': 'json'}) + token = r.json()['query']['tokens']['logintoken'] + s.post(API, data={'action': 'login', 'lgname': user, 'lgpassword': password, + 'lgtoken': token, 'format': 'json'}) + r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'format': 'json'}) + csrf = r.json()['query']['tokens']['csrftoken'] + return s, csrf + + +# --------------------------------------------------------------------------- +# Title determination +# --------------------------------------------------------------------------- + +def page_title(md_path: Path) -> str: + parts = md_path.relative_to(REPO).parts + if md_path.name == 'home.md' and len(parts) == 1: + return 'Заглавная страница' + return _cap(md_path.stem) + + +# --------------------------------------------------------------------------- +# Markdown → wikitext conversion +# --------------------------------------------------------------------------- + +def convert_pandoc(text: str) -> str: + return subprocess.run( + ['pandoc', '-f', 'markdown', '-t', 'mediawiki'], + input=text, capture_output=True, text=True + ).stdout + + +def convert_python(text: str) -> str: + lines = text.split('\n') + + # Collect footnote definitions in one pass + footnotes: dict[str, str] = {} + for line in lines: + m = _FN_DEF.match(line) + if m: + footnotes[m.group(1)] = m.group(2) + + def replace_fn(m): + key = m.group(0) + return f'{footnotes.get(key, key)}' + + out = [] + for line in lines: + if _FN_DEF.match(line): + continue + + m = re.match(r'^(#{1,6})\s+(.*)', line) + if m: + eq = '=' * len(m.group(1)) + out.append(f'{eq} {m.group(2)} {eq}') + continue + + if re.match(r'^---+$', line.strip()): + out.append('----') + continue + + line = re.sub(r'\*\*\*(.+?)\*\*\*', r"'''''\1'''''", line) + line = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", line) + line = re.sub(r'\*(.+?)\*', r"''\1''", line) + line = re.sub(r'\[\^\S+?\]', replace_fn, line) + out.append(line) + + return convert_tables('\n'.join(out)) + + +def convert_tables(text: str) -> str: + lines = text.split('\n') + out = [] + in_table = False + + for line in lines: + if re.match(r'^\|', line): + cells = [c.strip() for c in line.strip().strip('|').split('|')] + if all(re.match(r'^:?-+:?$', c) for c in cells if c): + if not in_table: + header_line = out.pop() if out else '' + hcells = [c.strip() for c in header_line.strip().strip('|').split('|')] + out += ['{| class="wikitable"', '|-', '! ' + ' !! '.join(hcells)] + in_table = True + out.append('|-') + else: + if in_table: + out.append('| ' + ' || '.join(cells)) + else: + out.append(line) + else: + if in_table: + out.append('|}') + in_table = False + out.append(line) + + if in_table: + out.append('|}') + + return '\n'.join(out) + + +def convert(text: str) -> str: + return convert_pandoc(text) if _pandoc_available() else convert_python(text) + + +# --------------------------------------------------------------------------- +# Post-processing +# --------------------------------------------------------------------------- + +def fix_links(text: str) -> str: + pattern = (r'\[\[([^\]|]+)\|' + r'(?:Люди|Места|Воспоминания|люди|места|воспоминания|Место)' + r'/([^\]]+)\]\]') + + def replace_link(m): + display = m.group(1).strip() + page = _cap(m.group(2).strip()) + if display == page or display == (page[0].lower() + page[1:] if page else ''): + return f'[[{page}]]' + return f'[[{page}|{display}]]' + + return re.sub(pattern, replace_link, text) + + +def fix_images(text: str) -> str: + pattern = r'!\[[^\]]*\]\(\./[^/)]+/([^)?]+)(\?[^)]*)?\)' + + def replace_img(m): + filename = m.group(1) + size_m = re.search(r'thumbnail=(\d+)', m.group(2) or '') + return f'[[File:{filename}|{size_m.group(1)}px]]' if size_m else f'[[File:{filename}]]' + + return re.sub(pattern, replace_img, text) + + +_CATEGORY_MAP = {'люди': 'Люди', 'места': 'Места', 'воспоминания': 'Воспоминания'} + + +def category_suffix(md_path: Path) -> str: + parts = md_path.relative_to(REPO).parts + if len(parts) == 1: + return '' if md_path.name == 'home.md' else '\n\n[[Category:Статьи]]' + cat = _CATEGORY_MAP.get(parts[0].lower()) + return f'\n\n[[Category:{cat}]]' if cat else '' + + +# --------------------------------------------------------------------------- +# MW operations +# --------------------------------------------------------------------------- + +def post_page(session, csrf: str, title: str, text: str, dry_run: bool) -> bool: + if dry_run: + print(f'[DRY] {title}') + return True + data = session.post(API, data={ + 'action': 'edit', 'title': title, 'text': text, + 'token': csrf, 'format': 'json' + }).json() + if 'error' in data: + print(f'[ERR] {title}: {data["error"].get("info", data["error"])}') + return False + print(f'[OK] {title}') + return True + + +def upload_image(session, csrf: str, image_path: Path, dry_run: bool) -> bool: + basename = image_path.name + if dry_run: + print(f'[DRY] File:{basename}') + return True + with open(image_path, 'rb') as f: + data = session.post(API, data={ + 'action': 'upload', 'filename': basename, + 'token': csrf, 'format': 'json', 'ignorewarnings': '1' + }, files={'file': f}).json() + if 'error' in data: + print(f'[ERR] File:{basename}: {data["error"].get("info", data["error"])}') + return False + if data.get('upload', {}).get('result') == 'Success': + print(f'[OK] File:{basename}') + else: + print(f'[SKIP] File:{basename} (already exists or no change)') + return True + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def collect_images() -> list[Path]: + images = [] + for folder in ('люди', 'места'): + p = REPO / folder + if p.exists(): + for ext in ('*.jpg', '*.jpeg', '*.JPG', '*.JPEG', '*.png', '*.PNG'): + images.extend(p.rglob(ext)) + return images + + +def main(): + parser = argparse.ArgumentParser(description='Migrate OtterWiki to MediaWiki') + parser.add_argument('--user', required=True) + parser.add_argument('--password', required=True) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + + session = csrf = None + if not args.dry_run: + session, csrf = mw_login(args.user, args.password) + + pages_ok = pages_err = images_ok = images_err = 0 + + for md_path in sorted(REPO.rglob('*.md')): + title = page_title(md_path) + raw = md_path.read_text(encoding='utf-8') + wikitext = convert(raw) + wikitext = fix_links(wikitext) + wikitext = fix_images(wikitext) + wikitext += category_suffix(md_path) + if post_page(session, csrf, title, wikitext, args.dry_run): + pages_ok += 1 + else: + pages_err += 1 + + for img in collect_images(): + if upload_image(session, csrf, img, args.dry_run): + images_ok += 1 + else: + images_err += 1 + + print(f'\nDone: {pages_ok} pages, {images_ok} images, {pages_err + images_err} errors') + + +if __name__ == '__main__': + main()