Add OtterWiki→MediaWiki migration script
This commit is contained in:
Alvis
277
wiki/migrate.py
Normal file
277
wiki/migrate.py
Normal file
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python3
|
||||
"""OtterWiki → MediaWiki migration script."""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
REPO = Path('/mnt/ssd/dbs/otter/app-data/repository')
|
||||
API = 'http://localhost:8099/api.php'
|
||||
|
||||
_FN_DEF = re.compile(r'^\[(\^[^\]]+)\]:\s*(.*)')
|
||||
|
||||
# Cached pandoc availability (None = not yet checked)
|
||||
_PANDOC_AVAILABLE: bool | None = None
|
||||
|
||||
|
||||
def _cap(s: str) -> str:
|
||||
"""Capitalize first character, leave rest unchanged."""
|
||||
return s[0].upper() + s[1:] if s else s
|
||||
|
||||
|
||||
def _pandoc_available() -> bool:
|
||||
global _PANDOC_AVAILABLE
|
||||
if _PANDOC_AVAILABLE is None:
|
||||
try:
|
||||
_PANDOC_AVAILABLE = subprocess.run(
|
||||
['pandoc', '--version'], capture_output=True
|
||||
).returncode == 0
|
||||
except FileNotFoundError:
|
||||
_PANDOC_AVAILABLE = False
|
||||
return _PANDOC_AVAILABLE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MediaWiki session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def mw_login(user: str, password: str):
|
||||
s = requests.Session()
|
||||
r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'type': 'login', 'format': 'json'})
|
||||
token = r.json()['query']['tokens']['logintoken']
|
||||
s.post(API, data={'action': 'login', 'lgname': user, 'lgpassword': password,
|
||||
'lgtoken': token, 'format': 'json'})
|
||||
r = s.get(API, params={'action': 'query', 'meta': 'tokens', 'format': 'json'})
|
||||
csrf = r.json()['query']['tokens']['csrftoken']
|
||||
return s, csrf
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Title determination
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def page_title(md_path: Path) -> str:
|
||||
parts = md_path.relative_to(REPO).parts
|
||||
if md_path.name == 'home.md' and len(parts) == 1:
|
||||
return 'Заглавная страница'
|
||||
return _cap(md_path.stem)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown → wikitext conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def convert_pandoc(text: str) -> str:
|
||||
return subprocess.run(
|
||||
['pandoc', '-f', 'markdown', '-t', 'mediawiki'],
|
||||
input=text, capture_output=True, text=True
|
||||
).stdout
|
||||
|
||||
|
||||
def convert_python(text: str) -> str:
|
||||
lines = text.split('\n')
|
||||
|
||||
# Collect footnote definitions in one pass
|
||||
footnotes: dict[str, str] = {}
|
||||
for line in lines:
|
||||
m = _FN_DEF.match(line)
|
||||
if m:
|
||||
footnotes[m.group(1)] = m.group(2)
|
||||
|
||||
def replace_fn(m):
|
||||
key = m.group(0)
|
||||
return f'<ref>{footnotes.get(key, key)}</ref>'
|
||||
|
||||
out = []
|
||||
for line in lines:
|
||||
if _FN_DEF.match(line):
|
||||
continue
|
||||
|
||||
m = re.match(r'^(#{1,6})\s+(.*)', line)
|
||||
if m:
|
||||
eq = '=' * len(m.group(1))
|
||||
out.append(f'{eq} {m.group(2)} {eq}')
|
||||
continue
|
||||
|
||||
if re.match(r'^---+$', line.strip()):
|
||||
out.append('----')
|
||||
continue
|
||||
|
||||
line = re.sub(r'\*\*\*(.+?)\*\*\*', r"'''''\1'''''", line)
|
||||
line = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", line)
|
||||
line = re.sub(r'\*(.+?)\*', r"''\1''", line)
|
||||
line = re.sub(r'\[\^\S+?\]', replace_fn, line)
|
||||
out.append(line)
|
||||
|
||||
return convert_tables('\n'.join(out))
|
||||
|
||||
|
||||
def convert_tables(text: str) -> str:
|
||||
lines = text.split('\n')
|
||||
out = []
|
||||
in_table = False
|
||||
|
||||
for line in lines:
|
||||
if re.match(r'^\|', line):
|
||||
cells = [c.strip() for c in line.strip().strip('|').split('|')]
|
||||
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
||||
if not in_table:
|
||||
header_line = out.pop() if out else ''
|
||||
hcells = [c.strip() for c in header_line.strip().strip('|').split('|')]
|
||||
out += ['{| class="wikitable"', '|-', '! ' + ' !! '.join(hcells)]
|
||||
in_table = True
|
||||
out.append('|-')
|
||||
else:
|
||||
if in_table:
|
||||
out.append('| ' + ' || '.join(cells))
|
||||
else:
|
||||
out.append(line)
|
||||
else:
|
||||
if in_table:
|
||||
out.append('|}')
|
||||
in_table = False
|
||||
out.append(line)
|
||||
|
||||
if in_table:
|
||||
out.append('|}')
|
||||
|
||||
return '\n'.join(out)
|
||||
|
||||
|
||||
def convert(text: str) -> str:
|
||||
return convert_pandoc(text) if _pandoc_available() else convert_python(text)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Post-processing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def fix_links(text: str) -> str:
|
||||
pattern = (r'\[\[([^\]|]+)\|'
|
||||
r'(?:Люди|Места|Воспоминания|люди|места|воспоминания|Место)'
|
||||
r'/([^\]]+)\]\]')
|
||||
|
||||
def replace_link(m):
|
||||
display = m.group(1).strip()
|
||||
page = _cap(m.group(2).strip())
|
||||
if display == page or display == (page[0].lower() + page[1:] if page else ''):
|
||||
return f'[[{page}]]'
|
||||
return f'[[{page}|{display}]]'
|
||||
|
||||
return re.sub(pattern, replace_link, text)
|
||||
|
||||
|
||||
def fix_images(text: str) -> str:
|
||||
pattern = r'!\[[^\]]*\]\(\./[^/)]+/([^)?]+)(\?[^)]*)?\)'
|
||||
|
||||
def replace_img(m):
|
||||
filename = m.group(1)
|
||||
size_m = re.search(r'thumbnail=(\d+)', m.group(2) or '')
|
||||
return f'[[File:{filename}|{size_m.group(1)}px]]' if size_m else f'[[File:{filename}]]'
|
||||
|
||||
return re.sub(pattern, replace_img, text)
|
||||
|
||||
|
||||
_CATEGORY_MAP = {'люди': 'Люди', 'места': 'Места', 'воспоминания': 'Воспоминания'}
|
||||
|
||||
|
||||
def category_suffix(md_path: Path) -> str:
|
||||
parts = md_path.relative_to(REPO).parts
|
||||
if len(parts) == 1:
|
||||
return '' if md_path.name == 'home.md' else '\n\n[[Category:Статьи]]'
|
||||
cat = _CATEGORY_MAP.get(parts[0].lower())
|
||||
return f'\n\n[[Category:{cat}]]' if cat else ''
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MW operations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def post_page(session, csrf: str, title: str, text: str, dry_run: bool) -> bool:
|
||||
if dry_run:
|
||||
print(f'[DRY] {title}')
|
||||
return True
|
||||
data = session.post(API, data={
|
||||
'action': 'edit', 'title': title, 'text': text,
|
||||
'token': csrf, 'format': 'json'
|
||||
}).json()
|
||||
if 'error' in data:
|
||||
print(f'[ERR] {title}: {data["error"].get("info", data["error"])}')
|
||||
return False
|
||||
print(f'[OK] {title}')
|
||||
return True
|
||||
|
||||
|
||||
def upload_image(session, csrf: str, image_path: Path, dry_run: bool) -> bool:
|
||||
basename = image_path.name
|
||||
if dry_run:
|
||||
print(f'[DRY] File:{basename}')
|
||||
return True
|
||||
with open(image_path, 'rb') as f:
|
||||
data = session.post(API, data={
|
||||
'action': 'upload', 'filename': basename,
|
||||
'token': csrf, 'format': 'json', 'ignorewarnings': '1'
|
||||
}, files={'file': f}).json()
|
||||
if 'error' in data:
|
||||
print(f'[ERR] File:{basename}: {data["error"].get("info", data["error"])}')
|
||||
return False
|
||||
if data.get('upload', {}).get('result') == 'Success':
|
||||
print(f'[OK] File:{basename}')
|
||||
else:
|
||||
print(f'[SKIP] File:{basename} (already exists or no change)')
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collect_images() -> list[Path]:
|
||||
images = []
|
||||
for folder in ('люди', 'места'):
|
||||
p = REPO / folder
|
||||
if p.exists():
|
||||
for ext in ('*.jpg', '*.jpeg', '*.JPG', '*.JPEG', '*.png', '*.PNG'):
|
||||
images.extend(p.rglob(ext))
|
||||
return images
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Migrate OtterWiki to MediaWiki')
|
||||
parser.add_argument('--user', required=True)
|
||||
parser.add_argument('--password', required=True)
|
||||
parser.add_argument('--dry-run', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
session = csrf = None
|
||||
if not args.dry_run:
|
||||
session, csrf = mw_login(args.user, args.password)
|
||||
|
||||
pages_ok = pages_err = images_ok = images_err = 0
|
||||
|
||||
for md_path in sorted(REPO.rglob('*.md')):
|
||||
title = page_title(md_path)
|
||||
raw = md_path.read_text(encoding='utf-8')
|
||||
wikitext = convert(raw)
|
||||
wikitext = fix_links(wikitext)
|
||||
wikitext = fix_images(wikitext)
|
||||
wikitext += category_suffix(md_path)
|
||||
if post_page(session, csrf, title, wikitext, args.dry_run):
|
||||
pages_ok += 1
|
||||
else:
|
||||
pages_err += 1
|
||||
|
||||
for img in collect_images():
|
||||
if upload_image(session, csrf, img, args.dry_run):
|
||||
images_ok += 1
|
||||
else:
|
||||
images_err += 1
|
||||
|
||||
print(f'\nDone: {pages_ok} pages, {images_ok} images, {pages_err + images_err} errors')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user