Initial toolkit: scripts, references, skills, and term database
This commit is contained in:
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env fish
|
||||
# Compile a Typst file to PDF.
|
||||
# Usage: compile-typst <path-to-file.typ> [output.pdf]
|
||||
# Output defaults to /tmp/<basename>.pdf
|
||||
# The Typst project root is set to the parent of the file's directory
|
||||
# (so imports like ../lib/... resolve correctly).
|
||||
|
||||
set src (realpath $argv[1])
|
||||
set src_dir (dirname $src)
|
||||
set root (dirname $src_dir)
|
||||
|
||||
if set -q argv[2]
|
||||
set out "$argv[2]"
|
||||
else
|
||||
set base (basename $src .typ)
|
||||
set out "/tmp/$base.pdf"
|
||||
end
|
||||
|
||||
typst compile --root $root $src $out
|
||||
echo $out
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env fish
|
||||
# Convert target.dj to English docx
|
||||
# Usage: dj2docx <path-to-target.dj> [output-filename]
|
||||
# Output filename defaults to /tmp/<parent-dirname>-英文.docx
|
||||
|
||||
set tgt (realpath $argv[1])
|
||||
if set -q argv[2]
|
||||
set out "$argv[2]"
|
||||
else
|
||||
set parent (basename (dirname $tgt))
|
||||
set out "/tmp/$parent-英文.docx"
|
||||
end
|
||||
pandoc $tgt -f djot -t docx -o $out
|
||||
echo $out
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env fish
|
||||
# Convert .docx to .dj (djot) via pandoc
|
||||
# Usage: docx2dj.fish <input.docx> [output.dj]
|
||||
# No output path → stdout
|
||||
|
||||
set docx (realpath $argv[1])
|
||||
|
||||
if test (count $argv) -ge 2
|
||||
pandoc $docx -f docx -t djot --wrap=none -o $argv[2]
|
||||
echo $argv[2]
|
||||
else
|
||||
pandoc $docx -f docx -t djot --wrap=none
|
||||
end
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Generate bilingual.dj from DOCX manuscript only (no PDF).
|
||||
Source: Chinese from DOCX. Target: English from DOCX.
|
||||
"""
|
||||
import re, subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DOCX = ROOT / "translate-files/佛教徒的人生态度/定稿 佛教徒的人生态度 善鑫慧炬照禅道靖妙一观轩慈德20260527.docx"
|
||||
OUT_DIR = ROOT / "translate-files/佛教徒的人生态度"
|
||||
|
||||
def has_cjk(s):
|
||||
return any('\u4e00' <= c <= '\u9fff' for c in s)
|
||||
|
||||
def pandoc(path):
|
||||
r = subprocess.run(['pandoc', path, '-f', 'docx', '-t', 'plain', '--wrap=none'],
|
||||
capture_output=True, text=True)
|
||||
return r.stdout
|
||||
|
||||
def extract_pairs(text):
|
||||
"""Return [(cn_para, en_para), ...] from body onwards."""
|
||||
lines = text.split('\n')
|
||||
body_start = None
|
||||
for i, l in enumerate(lines):
|
||||
if '生活在这个世间' in l:
|
||||
body_start = i
|
||||
break
|
||||
pairs = []
|
||||
i = body_start
|
||||
while i < len(lines):
|
||||
cn = lines[i].strip()
|
||||
if not cn or not has_cjk(cn):
|
||||
i += 1
|
||||
continue
|
||||
en = ''
|
||||
if i + 2 < len(lines) and lines[i+1].strip() == '':
|
||||
ec = lines[i+2].strip()
|
||||
if ec and not has_cjk(ec):
|
||||
en = ec
|
||||
i += 3
|
||||
else:
|
||||
i += 1
|
||||
else:
|
||||
i += 1
|
||||
continue
|
||||
pairs.append((cn, en))
|
||||
return pairs
|
||||
|
||||
SANSKRIT = [
|
||||
'bodhisattva', 'bodhicitta', 'samsara', 'Dharma', 'karma',
|
||||
'nirvana', 'Sangha', 'sutra', 'Mahayana', 'Sravaka',
|
||||
'Vinaya', 'Lamrim', 'Ksitigarbha', 'Samantabhadra',
|
||||
'Chan', 'Arhatship', 'Theravada',
|
||||
]
|
||||
|
||||
def apply_fixes(en_text, italicized):
|
||||
"""Apply typesetting fixes to English text."""
|
||||
# Fix: "2.How" → "2. How"
|
||||
en_text = re.sub(r'(\d)\.([A-Z][a-z])', r'\1. \2', en_text)
|
||||
# Fix: "said,"When → "said, "When
|
||||
en_text = re.sub(r'(said|says),"', r'\1, "', en_text)
|
||||
# Italicize Sanskrit on first occurrence
|
||||
for term in SANSKRIT:
|
||||
if term not in italicized:
|
||||
pattern = re.compile(r'\b' + re.escape(term) + r'\b')
|
||||
m = pattern.search(en_text)
|
||||
if m:
|
||||
s, e = m.start(), m.end()
|
||||
en_text = en_text[:s] + '*' + en_text[s:e] + '*' + en_text[e:]
|
||||
italicized.add(term)
|
||||
return en_text
|
||||
|
||||
def generate(pairs, out_path):
|
||||
italicized = set()
|
||||
lines = []
|
||||
|
||||
# Title
|
||||
lines.append('# 佛教徒的人生态度')
|
||||
lines.append('# The Life Attitudes of Buddhists')
|
||||
lines.append('')
|
||||
lines.append('------2014年秋讲于第九届菩提静修营')
|
||||
lines.append('---Lecture Given at the 9th Bodhi Meditation Retreat, 2014')
|
||||
lines.append('')
|
||||
lines.append(' 济群法师 ')
|
||||
lines.append('Master Jiqun')
|
||||
lines.append('')
|
||||
|
||||
# TOC
|
||||
lines.append('- 一、消极还是积极')
|
||||
lines.append('- 二、悲观还是乐观')
|
||||
lines.append('- 三、禁欲还是纵欲')
|
||||
lines.append('- 四、重生还是重死')
|
||||
lines.append('- 五、自利还是利他')
|
||||
lines.append('- 六、出世还是入世')
|
||||
lines.append('- 七、无情还是多情')
|
||||
lines.append('- 八、随缘还是进取')
|
||||
lines.append('- 九、结束语')
|
||||
lines.append('')
|
||||
lines.append('- I. Passive or Proactive')
|
||||
lines.append('- II. Pessimism or Optimism')
|
||||
lines.append('- III. Abstinence or Indulgence')
|
||||
lines.append('- IV. Focus on Life or on Death')
|
||||
lines.append('- V. Benefit Oneself or Benefit Others')
|
||||
lines.append('- VI. Transcending the World or Engaging with the World')
|
||||
lines.append('- VII. To Love or Not to Love')
|
||||
lines.append('- VIII. Adapting to Conditions or Striving for Progress')
|
||||
lines.append('- IX. Conclusion')
|
||||
lines.append('')
|
||||
|
||||
# Body
|
||||
for cn, en in pairs:
|
||||
en_fixed = apply_fixes(en, italicized)
|
||||
lines.append(cn)
|
||||
lines.append(en_fixed)
|
||||
lines.append('')
|
||||
|
||||
with open(out_path, 'w') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
print(f"Written: {out_path}")
|
||||
print(f" Paragraphs: {len(pairs)}")
|
||||
print(f" Sanskrit italicized: {sorted(italicized)}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("Extracting DOCX...")
|
||||
text = pandoc(DOCX)
|
||||
pairs = extract_pairs(text)
|
||||
print(f" Pairs: {len(pairs)}")
|
||||
|
||||
out = OUT_DIR / "bilingual.dj"
|
||||
generate(pairs, out)
|
||||
@@ -0,0 +1,262 @@
|
||||
"""
|
||||
Generate bilingual.dj for 佛教徒的人生态度.
|
||||
Source: Chinese from DOCX manuscript.
|
||||
Target: English from PDF typeset.
|
||||
|
||||
Strategy: find each DOCX English paragraph in PDF body, extract
|
||||
the PDF text region for that paragraph using position boundaries.
|
||||
"""
|
||||
import re, subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DOCX = ROOT / "translate-files/佛教徒的人生态度/定稿 佛教徒的人生态度 善鑫慧炬照禅道靖妙一观轩慈德20260527.docx"
|
||||
PDF = ROOT / "translate-files/佛教徒的人生态度/0607-二排-果澄-佛教徒的人生态度-一校-多人-0607.pdf"
|
||||
OUT_DIR = ROOT / "translate-files/佛教徒的人生态度"
|
||||
|
||||
|
||||
def has_cjk(s):
|
||||
return any('\u4e00' <= c <= '\u9fff' for c in s)
|
||||
|
||||
|
||||
def pandoc(path):
|
||||
r = subprocess.run(['pandoc', path, '-f', 'docx', '-t', 'plain', '--wrap=none'],
|
||||
capture_output=True, text=True)
|
||||
return r.stdout
|
||||
|
||||
|
||||
def extract_docx_pairs(text):
|
||||
"""Return [(cn_para, en_para), ...] from body onwards."""
|
||||
lines = text.split('\n')
|
||||
body_start = None
|
||||
for i, l in enumerate(lines):
|
||||
if '生活在这个世间' in l:
|
||||
body_start = i
|
||||
break
|
||||
pairs = []
|
||||
i = body_start
|
||||
while i < len(lines):
|
||||
cn = lines[i].strip()
|
||||
if not cn or not has_cjk(cn):
|
||||
i += 1
|
||||
continue
|
||||
en = ''
|
||||
if i + 2 < len(lines) and lines[i+1].strip() == '':
|
||||
ec = lines[i+2].strip()
|
||||
if ec and not has_cjk(ec):
|
||||
en = ec
|
||||
i += 3
|
||||
else:
|
||||
i += 1
|
||||
else:
|
||||
i += 1
|
||||
continue
|
||||
pairs.append((cn, en))
|
||||
return pairs
|
||||
|
||||
|
||||
def extract_pdf_body(text):
|
||||
"""Return cleaned PDF body string."""
|
||||
lines = text.split('\n')
|
||||
body_start = None
|
||||
for i, l in enumerate(lines):
|
||||
if 'iving in this world' in l.strip():
|
||||
body_start = i
|
||||
break
|
||||
slug_re = re.compile(r'佛教徒的人生态度.*indd \d+')
|
||||
hdr_re = re.compile(r'^(The Life Attitudes of Buddhists|The Mindful Peace Academy Collection)$')
|
||||
pn_re = re.compile(r'^\d{1,3}$')
|
||||
|
||||
tl = []
|
||||
for i in range(body_start, len(lines)):
|
||||
s = lines[i].strip()
|
||||
if not s or s == '\x0c':
|
||||
continue
|
||||
if slug_re.search(s) or hdr_re.match(s) or pn_re.match(s):
|
||||
continue
|
||||
tl.append(s)
|
||||
|
||||
# Join hyphenation breaks — handle consecutive breaks
|
||||
joined = []
|
||||
i = 0
|
||||
while i < len(tl):
|
||||
line = tl[i].rstrip()
|
||||
if line.endswith('-') and i + 1 < len(tl):
|
||||
n = tl[i+1].lstrip()
|
||||
if n and n[0].islower():
|
||||
merged = line[:-1] + n
|
||||
# Check if MORE consecutive breaks follow
|
||||
j = i + 2
|
||||
while j < len(tl) and merged.rstrip().endswith('-'):
|
||||
nn = tl[j].lstrip()
|
||||
if nn and nn[0].islower():
|
||||
merged = merged.rstrip()[:-1] + nn
|
||||
j += 1
|
||||
else:
|
||||
break
|
||||
joined.append(merged)
|
||||
i = j
|
||||
continue
|
||||
joined.append(line)
|
||||
i += 1
|
||||
|
||||
body = ' '.join(joined)
|
||||
body = re.sub(r'\s+', ' ', body).strip()
|
||||
body = body.replace('L iving', 'Living')
|
||||
return body
|
||||
|
||||
|
||||
def norm(s):
|
||||
s = re.sub(r'\s+', ' ', s).strip().lower()
|
||||
s = s.replace('\u201c', '"').replace('\u201d', '"')
|
||||
s = s.replace('\u2018', "'").replace('\u2019', "'")
|
||||
return s
|
||||
|
||||
|
||||
def find_positions(pairs, pdf_body):
|
||||
"""For each DOCX English para, find start position in PDF body.
|
||||
Returns list of (start_pos or None, matched_text or None).
|
||||
"""
|
||||
positions = []
|
||||
last_pos = 0
|
||||
for cn, en in pairs:
|
||||
needle = norm(en)
|
||||
haystack = norm(pdf_body[last_pos:])
|
||||
|
||||
# Try full match
|
||||
idx = haystack.find(needle)
|
||||
if idx < 0:
|
||||
# Try first 80 chars
|
||||
key = needle[:80]
|
||||
idx = haystack.find(key)
|
||||
if idx < 0:
|
||||
# Try first 40 chars
|
||||
key = needle[:40]
|
||||
idx = haystack.find(key)
|
||||
if idx < 0:
|
||||
# Try first 25 chars
|
||||
key = needle[:25]
|
||||
idx = haystack.find(key)
|
||||
|
||||
if idx >= 0:
|
||||
pos = last_pos + idx
|
||||
positions.append(pos)
|
||||
last_pos = pos + max(len(needle), 30)
|
||||
else:
|
||||
positions.append(None)
|
||||
return positions
|
||||
|
||||
|
||||
def extract_segments(pdf_body, positions):
|
||||
"""For each position, extract the PDF text region.
|
||||
Region extends from positions[i] to positions[i+1] (or end),
|
||||
trimmed to avoid bleeding into the next paragraph.
|
||||
"""
|
||||
segments = []
|
||||
for i, pos in enumerate(positions):
|
||||
if pos is None:
|
||||
segments.append(None)
|
||||
continue
|
||||
|
||||
start = pos
|
||||
end = len(pdf_body)
|
||||
for j in range(i + 1, len(positions)):
|
||||
if positions[j] is not None:
|
||||
end = positions[j]
|
||||
break
|
||||
|
||||
raw = pdf_body[start:end].strip()
|
||||
|
||||
# Trim: if raw contains what looks like the NEXT paragraph's heading,
|
||||
# cut at the last sentence boundary before it.
|
||||
# Headings match patterns like: "I Passive", "1) Expressions", "1. The Definitions"
|
||||
heading_pattern = re.compile(
|
||||
r'\s+(?=[IVX]+\.?\s+[A-Z]' # Roman numeral chapter
|
||||
r'|\d+\)\s+[A-Z]' # 1) Sub-heading
|
||||
r'|\(\d+\)\s+[A-Z]' # (1) Sub-heading
|
||||
r'|\d+\.\s+[A-Z][a-z]+.*?(?:Passive|Pessimism|Abstinence|Focus|Benefit|Transcending|Love|Adapting|Conclusion|Desire|Being|What|How|The|Buddhism|Set|Free|A Middle)' # Numbered heading
|
||||
r')'
|
||||
)
|
||||
m = heading_pattern.search(raw)
|
||||
if m:
|
||||
# Cut before this heading
|
||||
raw = raw[:m.start()].strip()
|
||||
|
||||
segments.append(raw)
|
||||
return segments
|
||||
|
||||
|
||||
def generate(pairs, segments, out_path):
|
||||
lines = []
|
||||
|
||||
# Title
|
||||
lines.append('# 佛教徒的人生态度')
|
||||
lines.append('# The Life Attitudes of Buddhists')
|
||||
lines.append('')
|
||||
lines.append('------2014年秋讲于第九届菩提静修营')
|
||||
lines.append('---Lecture Given at the 9th Bodhi Meditation Retreat, 2014')
|
||||
lines.append('')
|
||||
lines.append(' 济群法师 ')
|
||||
lines.append('Master Jiqun')
|
||||
lines.append('')
|
||||
|
||||
# TOC from DOCX
|
||||
lines.append('- 一、消极还是积极')
|
||||
lines.append('- 二、悲观还是乐观')
|
||||
lines.append('- 三、禁欲还是纵欲')
|
||||
lines.append('- 四、重生还是重死')
|
||||
lines.append('- 五、自利还是利他')
|
||||
lines.append('- 六、出世还是入世')
|
||||
lines.append('- 七、无情还是多情')
|
||||
lines.append('- 八、随缘还是进取')
|
||||
lines.append('- 九、结束语')
|
||||
lines.append('')
|
||||
lines.append('- I. Passive or Proactive')
|
||||
lines.append('- II. Pessimism or Optimism')
|
||||
lines.append('- III. Abstinence or Indulgence')
|
||||
lines.append('- IV. Focus on Life or on Death')
|
||||
lines.append('- V. Benefit Oneself or Benefit Others')
|
||||
lines.append('- VI. Transcending the World or Engaging with the World')
|
||||
lines.append('- VII. To Love or Not to Love')
|
||||
lines.append('- VIII. Adapting to Conditions or Striving for Progress')
|
||||
lines.append('- IX. Conclusion')
|
||||
lines.append('')
|
||||
|
||||
# Body
|
||||
for (cn, en), seg in zip(pairs, segments):
|
||||
target = seg if seg else en
|
||||
lines.append(cn)
|
||||
lines.append(target)
|
||||
lines.append('')
|
||||
|
||||
with open(out_path, 'w') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
matched = sum(1 for s in segments if s is not None)
|
||||
print(f"Written: {out_path}")
|
||||
print(f" Paragraphs: {len(pairs)}, matched from PDF: {matched}, fallback to DOCX: {len(pairs) - matched}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("Extracting DOCX...")
|
||||
docx_text = pandoc(DOCX)
|
||||
pairs = extract_docx_pairs(docx_text)
|
||||
print(f" Pairs: {len(pairs)}")
|
||||
|
||||
print("Extracting PDF...")
|
||||
r = subprocess.run(['pdftotext', '-layout', PDF, '/tmp/_bilingual_pdf.txt'], check=True)
|
||||
with open('/tmp/_bilingual_pdf.txt') as f:
|
||||
pdf_raw = f.read()
|
||||
pdf_body = extract_pdf_body(pdf_raw)
|
||||
print(f" PDF body: {len(pdf_body)} chars")
|
||||
|
||||
print("Finding positions...")
|
||||
positions = find_positions(pairs, pdf_body)
|
||||
found = sum(1 for p in positions if p is not None)
|
||||
print(f" Found: {found}/{len(pairs)}")
|
||||
|
||||
print("Extracting segments...")
|
||||
segments = extract_segments(pdf_body, positions)
|
||||
|
||||
out = OUT_DIR / "bilingual.dj"
|
||||
generate(pairs, segments, out)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Extract source.dj and bilingual.dj from .docx.md file.
|
||||
|
||||
Handles:
|
||||
- CN/EN paragraph pairs (CN line → EN line)
|
||||
- Headings with merged CN+EN on same line (pandoc artifact: CN**EN)
|
||||
- TOC with markdown links [CN text](#anchor)
|
||||
- {#anchor} pandoc heading anchors
|
||||
- Markdown heading cleanup
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SRC = ROOT / "translate-files/佛法与企业管理/副本59 佛法与企业管理-maple 初翻.docx.md"
|
||||
OUT = ROOT / "translate-files/佛法与企业管理"
|
||||
|
||||
def has_cjk(s):
|
||||
return any('\u4e00' <= c <= '\u9fff' for c in s)
|
||||
|
||||
def strip_anchors(s):
|
||||
"""Remove {#anchor} and markdown links [text](#anchor) — keep text."""
|
||||
s = re.sub(r'\{#[^}]*\}', '', s) # {#anchor}
|
||||
s = re.sub(r'\[([^\]]*)\]\([^)]*\)', r'\1', s) # [text](#link) → text
|
||||
return s
|
||||
|
||||
def split_cnen(line):
|
||||
"""Split merged CN+EN line. First strips anchors, then finds CJK→EN boundary."""
|
||||
s = strip_anchors(line).strip()
|
||||
if not has_cjk(s):
|
||||
return ('', s)
|
||||
# Find where CJK ends and ASCII English begins
|
||||
# Pattern: CJK, optional ws, optional *, optional ws, then English letter
|
||||
m = re.search(r'[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef]\s*\**\s*([A-Za-z])', s)
|
||||
if m:
|
||||
split_at = m.start(1)
|
||||
cn = s[:split_at].rstrip('* ').strip()
|
||||
en = s[split_at:].strip()
|
||||
if en and not has_cjk(en):
|
||||
return (cn, en)
|
||||
return (s, '')
|
||||
|
||||
def clean_cn(s):
|
||||
"""Clean CN heading/paragraph."""
|
||||
s = re.sub(r'^\d+\.\s*', '', s) # leading number (e.g. "1. ")
|
||||
s = re.sub(r'^#+\s*\**', '', s) # heading markers: # **
|
||||
s = re.sub(r'\**\s*$', '', s) # trailing **
|
||||
s = re.sub(r'\t\d+$', '', s) # trailing page number
|
||||
s = s.strip()
|
||||
return s
|
||||
|
||||
def clean_en(s):
|
||||
"""Clean EN line."""
|
||||
s = re.sub(r'^\d+[、,.]\s*', '', s) # leading number/separator
|
||||
s = re.sub(r'\*+$', '', s) # trailing orphaned italic * (from split)
|
||||
s = s.strip()
|
||||
return s
|
||||
|
||||
def is_toc_line(line, cn_raw):
|
||||
"""True if this looks like a TOC entry (markdown link or tab+page-number)."""
|
||||
if re.search(r'\[.*\]\(.*\)', line):
|
||||
return True
|
||||
if re.search(r'\t\d+', cn_raw):
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_doc(text):
|
||||
"""Parse into list of (cn, en) pairs."""
|
||||
lines = text.split('\n')
|
||||
pairs = []
|
||||
i = 0
|
||||
|
||||
while i < len(lines):
|
||||
line = lines[i].strip()
|
||||
if not line:
|
||||
i += 1
|
||||
continue
|
||||
|
||||
stripped = strip_anchors(line)
|
||||
has_cn = has_cjk(stripped)
|
||||
|
||||
if has_cn:
|
||||
cn_raw, en_raw = split_cnen(line)
|
||||
|
||||
if en_raw:
|
||||
# Merged CN+EN on same line
|
||||
pairs.append((clean_cn(cn_raw), clean_en(en_raw)))
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Look ahead for EN
|
||||
nxt = lines[i+1].strip() if i+1 < len(lines) else ''
|
||||
nnxt = lines[i+2].strip() if i+2 < len(lines) else ''
|
||||
|
||||
if nxt and not has_cjk(nxt) and not is_toc_line(line, cn_raw):
|
||||
# Standard: CN → EN
|
||||
pairs.append((clean_cn(cn_raw), clean_en(nxt)))
|
||||
i += 2
|
||||
elif not nxt and nnxt and not has_cjk(nnxt) and not is_toc_line(line, cn_raw):
|
||||
# CN → blank → EN (heading pattern)
|
||||
pairs.append((clean_cn(cn_raw), clean_en(nnxt)))
|
||||
i += 3
|
||||
else:
|
||||
# Solo CN (TOC entry, orphan, or heading)
|
||||
pairs.append((clean_cn(cn_raw), ''))
|
||||
i += 1
|
||||
else:
|
||||
# Pure EN — TOC entry, pair with first unpaired CN TOC entry
|
||||
en = clean_en(line)
|
||||
for j in range(len(pairs)):
|
||||
if not pairs[j][1] and has_cjk(pairs[j][0]):
|
||||
pairs[j] = (pairs[j][0], en)
|
||||
break
|
||||
else:
|
||||
pairs.append(('', en))
|
||||
i += 1
|
||||
|
||||
return pairs
|
||||
|
||||
def classify(pairs):
|
||||
"""Classify each pair as title, subtitle, toc, heading, or para."""
|
||||
result = []
|
||||
# Pairs 0-1: title and subtitle
|
||||
result.append(('title', 0))
|
||||
result.append(('subtitle', 1))
|
||||
# Pairs 2-9: TOC (一、 through 八、)
|
||||
for i in range(2, min(10, len(pairs))):
|
||||
result.append(('toc', i))
|
||||
# Remaining: heuristics
|
||||
for i in range(10, len(pairs)):
|
||||
cn = pairs[i][0]
|
||||
if re.match(r'^[一二三四五六七八九十]、', cn):
|
||||
result.append(('heading', i))
|
||||
elif re.match(r'^\d+\\?\.\s', cn):
|
||||
result.append(('subheading', i))
|
||||
else:
|
||||
result.append(('para', i))
|
||||
return result
|
||||
|
||||
def write_bilingual(pairs, out_path):
|
||||
types = classify(pairs)
|
||||
lines = []
|
||||
# Title
|
||||
lines.append('# ' + pairs[0][0])
|
||||
lines.append('# ' + pairs[0][1])
|
||||
lines.append('')
|
||||
# Subtitle
|
||||
lines.append('---' + pairs[1][0])
|
||||
lines.append('---' + pairs[1][1])
|
||||
lines.append('')
|
||||
# Author
|
||||
lines.append('济群法师')
|
||||
lines.append('Master Jiqun')
|
||||
lines.append('')
|
||||
# TOC — CN block then EN block (not interleaved)
|
||||
for typ, idx in types:
|
||||
if typ == 'toc':
|
||||
lines.append('- ' + pairs[idx][0])
|
||||
lines.append('')
|
||||
for typ, idx in types:
|
||||
if typ == 'toc':
|
||||
lines.append('- ' + pairs[idx][1])
|
||||
lines.append('')
|
||||
# Body
|
||||
for typ, idx in types:
|
||||
if typ in ('title', 'subtitle', 'toc'):
|
||||
continue
|
||||
cn, en = pairs[idx]
|
||||
if cn:
|
||||
lines.append(cn)
|
||||
if en:
|
||||
lines.append(en)
|
||||
if cn or en:
|
||||
lines.append('')
|
||||
with open(out_path, 'w') as f:
|
||||
f.write('\n'.join(lines))
|
||||
print(f"bilingual.dj: {len(pairs)} pairs -> {out_path}")
|
||||
|
||||
def write_source(pairs, out_path):
|
||||
lines = [cn for cn, en in pairs if cn]
|
||||
with open(out_path, 'w') as f:
|
||||
f.write('\n'.join(lines) + '\n')
|
||||
print(f"source.dj: {len(lines)} CN lines -> {out_path}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
text = SRC.read_text()
|
||||
pairs = parse_doc(text)
|
||||
print(f"Parsed {len(pairs)} pairs from .docx.md")
|
||||
solo_cn = sum(1 for cn, en in pairs if cn and not en)
|
||||
solo_en = sum(1 for cn, en in pairs if en and not cn)
|
||||
both = sum(1 for cn, en in pairs if cn and en)
|
||||
print(f" Both: {both}, CN-only: {solo_cn}, EN-only: {solo_en}")
|
||||
write_source(pairs, OUT / "source.dj")
|
||||
write_bilingual(pairs, OUT / "bilingual.dj")
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Generate bilingual.dj from DOCX for 「生命也可以被设计的」.
|
||||
One-pass approach: walk interleaved paragraphs, handle multi-CN sequences.
|
||||
"""
|
||||
import re, subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DOCX = ROOT / "translate-files/生命也可以被设计的/中英文定稿-260324-生命也是可以被设计的-妙一宽山静雅初翻 慈鎏妙一审议 宽山定稿.docx"
|
||||
OUT_DIR = ROOT / "translate-files/生命也可以被设计的"
|
||||
|
||||
def has_cjk(s):
|
||||
return any('\u4e00' <= c <= '\u9fff' for c in s)
|
||||
|
||||
def pandoc(path):
|
||||
r = subprocess.run(['pandoc', path, '-f', 'docx', '-t', 'plain', '--wrap=none'],
|
||||
capture_output=True, text=True)
|
||||
return r.stdout
|
||||
|
||||
def split_toc_line(line):
|
||||
s = line.strip()
|
||||
s = re.sub(r'\s+\d+\s*$', '', s)
|
||||
m = re.match(r'^(.+[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef)\)])\s+([A-Z].+)$', s)
|
||||
if m:
|
||||
return m.group(1).strip(), m.group(2).strip()
|
||||
return None, None
|
||||
|
||||
def extract_toc_entries(text):
|
||||
lines = text.split('\n')
|
||||
toc_start = None
|
||||
toc_end = None
|
||||
for i, l in enumerate(lines):
|
||||
s = l.strip()
|
||||
if s.startswith('一、') and ('EDUCATION' in s or 'NURTURING' in s):
|
||||
if toc_start is None:
|
||||
toc_start = i
|
||||
if toc_start is not None and s and has_cjk(s) and re.search(r'\d+$', s):
|
||||
toc_end = i
|
||||
elif toc_start is not None and toc_end is not None and s and not re.search(r'\d+$', s) and has_cjk(s):
|
||||
break
|
||||
|
||||
cn_entries = []
|
||||
en_entries = []
|
||||
for i in range(toc_start, toc_end + 1):
|
||||
cn, en = split_toc_line(lines[i])
|
||||
if cn and en:
|
||||
cn_entries.append(cn)
|
||||
en_entries.append(en)
|
||||
return cn_entries, en_entries
|
||||
|
||||
def extract_body_pairs(text):
|
||||
"""One-pass: walk interleaved paras, joining consecutive same-language lines."""
|
||||
lines = text.split('\n')
|
||||
|
||||
# Find body start
|
||||
body_start = None
|
||||
for i, l in enumerate(lines):
|
||||
if '现在是一个浮躁的时代' in l:
|
||||
body_start = i
|
||||
break
|
||||
|
||||
# Extract non-blank paragraphs with language tags
|
||||
tagged = []
|
||||
for l in lines[body_start:]:
|
||||
s = l.strip()
|
||||
if s:
|
||||
tagged.append(('cn' if has_cjk(s) else 'en', s))
|
||||
|
||||
# Accumulate consecutive same-language paragraphs (page-break splits only, not headings)
|
||||
merged = []
|
||||
for lang, text in tagged:
|
||||
# Heading-like patterns that should not be merged
|
||||
prev_is_heading = (merged and merged[-1][0] == lang
|
||||
and bool(re.match(r'^[\dIVX]+[\.\s]', merged[-1][1].strip())
|
||||
and len(merged[-1][1].strip()) < 60))
|
||||
if (merged and merged[-1][0] == lang
|
||||
and not prev_is_heading
|
||||
and len(merged[-1][1]) > 30
|
||||
and not re.search(r'[。!?:)\u201d\u2019\uff0c\uff0e\.!\?]$', merged[-1][1])):
|
||||
# Long previous line, doesn't end naturally → page-break split, join
|
||||
merged[-1] = (lang, merged[-1][1].rstrip() + text.lstrip())
|
||||
else:
|
||||
merged.append((lang, text))
|
||||
|
||||
# Build pairs: group consecutive same-language items into blocks, then zip
|
||||
blocks = []
|
||||
for lang, text in merged:
|
||||
if blocks and blocks[-1][0] == lang:
|
||||
blocks[-1][1].append(text)
|
||||
else:
|
||||
blocks.append((lang, [text]))
|
||||
|
||||
pairs = []
|
||||
i = 0
|
||||
while i < len(blocks):
|
||||
if blocks[i][0] == 'cn':
|
||||
cn_block = blocks[i][1]
|
||||
# Find next EN block
|
||||
if i + 1 < len(blocks) and blocks[i+1][0] == 'en':
|
||||
en_block = blocks[i+1][1]
|
||||
n = min(len(cn_block), len(en_block))
|
||||
for j in range(n):
|
||||
pairs.append((cn_block[j], en_block[j]))
|
||||
if len(cn_block) != len(en_block):
|
||||
print(f" WARNING: block mismatch CN={len(cn_block)} EN={len(en_block)} at CN[{j}]: {cn_block[j][:60]}...")
|
||||
i += 2
|
||||
else:
|
||||
print(f" WARNING: CN block without EN block: {cn_block[0][:60]}...")
|
||||
i += 1
|
||||
else:
|
||||
print(f" WARNING: orphan EN block: {blocks[i][1][0][:60]}...")
|
||||
i += 1
|
||||
|
||||
return pairs
|
||||
|
||||
SANSKRIT = [
|
||||
'bodhisattva', 'bodhicitta', 'samsara', 'Dharma', 'karma',
|
||||
'nirvana', 'Sangha', 'sutra', 'Mahayana', 'Sravaka',
|
||||
'Vinaya', 'Lamrim', 'Ksitigarbha', 'Samantabhadra',
|
||||
'Chan', 'Arhatship', 'Theravada', 'buddha', 'Buddha',
|
||||
'buddhas', 'Buddhas', 'Bodhisattva', 'Bodhisattvas',
|
||||
]
|
||||
|
||||
def apply_fixes(en_text, italicized):
|
||||
en_text = re.sub(r'(\d)\.([A-Z][a-z])', r'\1. \2', en_text)
|
||||
en_text = re.sub(r'(said|says),\"', r'\1, "', en_text)
|
||||
en_text = re.sub(r'\.([A-Z][a-z])', r'. \1', en_text)
|
||||
for term in SANSKRIT:
|
||||
if term not in italicized:
|
||||
pattern = re.compile(r'\b' + re.escape(term) + r'\b')
|
||||
m = pattern.search(en_text)
|
||||
if m:
|
||||
s, e = m.start(), m.end()
|
||||
en_text = en_text[:s] + '*' + en_text[s:e] + '*' + en_text[e:]
|
||||
italicized.add(term)
|
||||
return en_text
|
||||
|
||||
def generate(toc_cn, toc_en, pairs, out_path):
|
||||
italicized = set()
|
||||
lines = []
|
||||
|
||||
lines.append('# 生命也是可以被设计的')
|
||||
lines.append('# Life Can Also Be Designed')
|
||||
lines.append('')
|
||||
lines.append('济群法师 2025年冬为母爱书院开示')
|
||||
lines.append('A teaching given by the Master Jiqun in the winter of 2025 at Amrita Retreat Center for Motherly Love Academy')
|
||||
lines.append('')
|
||||
|
||||
for e in toc_cn:
|
||||
lines.append(f'- {e}')
|
||||
lines.append('')
|
||||
for e in toc_en:
|
||||
lines.append(f'- {e}')
|
||||
lines.append('')
|
||||
|
||||
for cn, en in pairs:
|
||||
en_fixed = apply_fixes(en, italicized)
|
||||
lines.append(cn)
|
||||
lines.append(en_fixed)
|
||||
lines.append('')
|
||||
|
||||
with open(out_path, 'w') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
print(f"Written: {out_path}")
|
||||
print(f" TOC entries: {len(toc_cn)}")
|
||||
print(f" Body pairs: {len(pairs)}")
|
||||
print(f" Sanskrit italicized: {sorted(italicized)}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("Extracting DOCX...")
|
||||
text = pandoc(DOCX)
|
||||
|
||||
print("Extracting TOC...")
|
||||
toc_cn, toc_en = extract_toc_entries(text)
|
||||
for cn, en in zip(toc_cn, toc_en):
|
||||
print(f" {cn} → {en}")
|
||||
|
||||
print("Extracting body...")
|
||||
pairs = extract_body_pairs(text)
|
||||
print(f" Pairs: {len(pairs)}")
|
||||
|
||||
out = OUT_DIR / "bilingual.dj"
|
||||
generate(toc_cn, toc_en, pairs, out)
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a bilingual .dj file from source (Chinese) and target (English) .dj files.
|
||||
|
||||
Usage:
|
||||
gen-bilingual.py source.dj target.dj > bilingual.dj
|
||||
|
||||
Output format: source line, target line, blank line, repeated. Paragraph breaks
|
||||
are preserved: blank lines in the input produce blank lines in the output.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print(__doc__, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
src_path = Path(sys.argv[1])
|
||||
tgt_path = Path(sys.argv[2])
|
||||
|
||||
if not src_path.exists():
|
||||
print(f"Source file not found: {src_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not tgt_path.exists():
|
||||
print(f"Target file not found: {tgt_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
src_lines = src_path.read_text(encoding="utf-8").splitlines()
|
||||
tgt_lines = tgt_path.read_text(encoding="utf-8").splitlines()
|
||||
|
||||
if len(src_lines) != len(tgt_lines):
|
||||
print(
|
||||
f"Line count mismatch: source={len(src_lines)} target={len(tgt_lines)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
out = []
|
||||
for s, t in zip(src_lines, tgt_lines):
|
||||
if s == "":
|
||||
out.append("")
|
||||
else:
|
||||
out.append(s)
|
||||
out.append(t)
|
||||
out.append("")
|
||||
|
||||
# Ensure the output always ends with a single trailing blank line to match
|
||||
# the project convention: source, target, blank, source, target, blank...
|
||||
if out and out[-1] != "":
|
||||
out.append("")
|
||||
|
||||
sys.stdout.write("\n".join(out))
|
||||
if out:
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Compare manuscript (DOCX English body) against typeset (PDF English body).
|
||||
Usage: python3 scripts/proofread-pdf.py <docx_path> <pdf_path>
|
||||
Output: sentences from DOCX not found in PDF, and word-level changes within matched sentences.
|
||||
"""
|
||||
import re, sys, subprocess
|
||||
|
||||
|
||||
def extract_docx_en(path):
|
||||
with open(path) as f:
|
||||
lines = f.readlines()
|
||||
|
||||
body_start = None
|
||||
for i, line in enumerate(lines):
|
||||
if '生活在这个世间' in line:
|
||||
body_start = i
|
||||
break
|
||||
if body_start is None:
|
||||
sys.exit("Could not find body start in DOCX")
|
||||
|
||||
docx_en = []
|
||||
skip_next = 0
|
||||
for i in range(body_start, len(lines)):
|
||||
if skip_next > 0:
|
||||
skip_next -= 1
|
||||
continue
|
||||
line = lines[i].strip()
|
||||
if not line:
|
||||
continue
|
||||
has_cjk = any('\u4e00' <= c <= '\u9fff' for c in line)
|
||||
if has_cjk:
|
||||
if i + 1 < len(lines) and lines[i+1].strip() == '':
|
||||
if i + 2 < len(lines):
|
||||
en_line = lines[i+2].strip()
|
||||
if en_line and not any('\u4e00' <= c <= '\u9fff' for c in en_line):
|
||||
docx_en.append(en_line)
|
||||
skip_next = 2
|
||||
else:
|
||||
docx_en.append(line)
|
||||
|
||||
# Split into sentences, filter out headings
|
||||
text = ' '.join(docx_en)
|
||||
sentences = re.split(r'(?<=[.!?"”])\s+', text)
|
||||
return [(s.strip(), len(s.strip())) for s in sentences if len(s.strip()) >= 20]
|
||||
|
||||
|
||||
def extract_pdf_en(path):
|
||||
with open(path) as f:
|
||||
lines = f.readlines()
|
||||
|
||||
body_start = None
|
||||
for i, line in enumerate(lines):
|
||||
if 'iving in this world' in line:
|
||||
body_start = i
|
||||
break
|
||||
if body_start is None:
|
||||
sys.exit("Could not find body start in PDF")
|
||||
|
||||
slug_re = re.compile(r'佛教徒的人生态度.*indd \d+')
|
||||
header_re = re.compile(r'^(The Life Attitudes of Buddhists|The Mindful Peace Academy Collection)$')
|
||||
page_num_re = re.compile(r'^\d{1,3}$')
|
||||
|
||||
text_lines = []
|
||||
for i in range(body_start, len(lines)):
|
||||
s = lines[i].strip()
|
||||
if not s or s == '\x0c':
|
||||
continue
|
||||
if slug_re.search(s) or header_re.match(s) or page_num_re.match(s):
|
||||
continue
|
||||
text_lines.append(s)
|
||||
|
||||
# Join hyphenated breaks
|
||||
joined = []
|
||||
i = 0
|
||||
while i < len(text_lines):
|
||||
line = text_lines[i]
|
||||
if line.rstrip().endswith('-') and i + 1 < len(text_lines):
|
||||
n = text_lines[i+1].lstrip()
|
||||
if n and n[0].islower():
|
||||
joined.append(line.rstrip()[:-1] + n)
|
||||
i += 2
|
||||
continue
|
||||
joined.append(line)
|
||||
i += 1
|
||||
|
||||
body = ' '.join(joined)
|
||||
body = re.sub(r'\s+', ' ', body).strip()
|
||||
body = body.replace('L iving', 'Living')
|
||||
return body
|
||||
|
||||
|
||||
def normalize_for_search(s):
|
||||
"""Normalize text for fuzzy matching."""
|
||||
s = re.sub(r'\s+', ' ', s).strip().lower()
|
||||
# Normalize quotes
|
||||
s = s.replace('\u201c', '"').replace('\u201d', '"')
|
||||
s = s.replace('\u2018', "'").replace('\u2019', "'")
|
||||
return s
|
||||
|
||||
|
||||
def find_sentence_in_pdf(sentence, pdf_body):
|
||||
"""Try to locate sentence in PDF body. Returns (found, matched_text)."""
|
||||
s_norm = normalize_for_search(sentence)
|
||||
# Try full sentence
|
||||
if s_norm in pdf_body.lower():
|
||||
return True, sentence
|
||||
# Try first 60 chars
|
||||
key = s_norm[:60]
|
||||
if key in pdf_body.lower():
|
||||
return True, sentence
|
||||
# Try first 30 chars
|
||||
key = s_norm[:30]
|
||||
if key in pdf_body.lower():
|
||||
return True, sentence
|
||||
return False, None
|
||||
|
||||
|
||||
def find_word_diff(docx_sentence, pdf_sentence):
|
||||
"""Find word-level differences between two matched sentences."""
|
||||
if not pdf_sentence:
|
||||
return []
|
||||
dw = re.findall(r'\S+', docx_sentence)
|
||||
pw = re.findall(r'\S+', pdf_sentence)
|
||||
diffs = []
|
||||
for dwi, pwi in zip(dw, pw):
|
||||
if dwi.lower() != pwi.lower():
|
||||
diffs.append((dwi, pwi))
|
||||
if len(dw) != len(pw):
|
||||
diffs.append((f"[{len(dw)} words]", f"[{len(pw)} words]"))
|
||||
return diffs
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("Usage: proofread-pdf.py <docx_path> <pdf_path>")
|
||||
|
||||
docx_path, pdf_path = sys.argv[1], sys.argv[2]
|
||||
docx_txt = '/tmp/proofread_docx.txt'
|
||||
pdf_txt = '/tmp/proofread_pdf.txt'
|
||||
|
||||
subprocess.run(['pandoc', docx_path, '-f', 'docx', '-t', 'plain', '--wrap=none', '-o', docx_txt], check=True)
|
||||
subprocess.run(['pdftotext', '-layout', pdf_path, pdf_txt], check=True)
|
||||
|
||||
docx_sentences = extract_docx_en(docx_txt)
|
||||
pdf_body = extract_pdf_en(pdf_txt)
|
||||
pdf_normalized = normalize_for_search(pdf_body)
|
||||
|
||||
missing = []
|
||||
found_count = 0
|
||||
for sentence, length in docx_sentences:
|
||||
s_norm = normalize_for_search(sentence)
|
||||
if s_norm in pdf_normalized:
|
||||
found_count += 1
|
||||
elif s_norm[:60] in pdf_normalized:
|
||||
found_count += 1
|
||||
elif s_norm[:30] in pdf_normalized:
|
||||
found_count += 1
|
||||
else:
|
||||
missing.append(sentence)
|
||||
|
||||
print(f"DOCX body sentences: {len(docx_sentences)}")
|
||||
print(f"Matched in PDF: {found_count}")
|
||||
print(f"Missing: {len(missing)}")
|
||||
print()
|
||||
|
||||
if missing:
|
||||
print("=== Sentences from DOCX NOT found in PDF ===")
|
||||
for i, s in enumerate(missing):
|
||||
print(f"\n--- Missing #{i+1} ---")
|
||||
print(s[:200])
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env fish
|
||||
# Split combined bilingual .dj into source.dj (CN) and target.dj (EN)
|
||||
# Usage: split-bilingual.fish <combined.dj>
|
||||
# Output: source.dj and target.dj in same directory, paragraphs separated by blanks
|
||||
|
||||
set dj (realpath $argv[1])
|
||||
set dir (dirname $dj)
|
||||
|
||||
rm -f "$dir/source.dj" "$dir/target.dj"
|
||||
|
||||
for line in (cat $dj)
|
||||
if string match -qr '[\x{4e00}-\x{9fff}]' -- $line
|
||||
echo $line >> "$dir/source.dj"
|
||||
echo >> "$dir/source.dj"
|
||||
else if test -n (string trim -- $line)
|
||||
echo $line >> "$dir/target.dj"
|
||||
echo >> "$dir/target.dj"
|
||||
end
|
||||
end
|
||||
|
||||
echo "source.dj: $dir/source.dj"
|
||||
echo "target.dj: $dir/target.dj"
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Extract cleaned English body from DOCX manuscript and typeset PDF.
|
||||
Usage: python3 ten-elements-c7fcd9.py <docx_path> <pdf_path>
|
||||
Output: two cleaned text files in /tmp/ for diffing.
|
||||
"""
|
||||
import re, sys, subprocess
|
||||
from pathlib import Path
|
||||
|
||||
DOCX_TXT = '/tmp/ten_elements_docx_body.txt'
|
||||
PDF_TXT = '/tmp/ten_elements_pdf_body.txt'
|
||||
|
||||
|
||||
def extract_docx_body(path):
|
||||
with open(path) as f:
|
||||
lines = f.readlines()
|
||||
for i, line in enumerate(lines):
|
||||
if 'The Dhyana Tea program team' in line:
|
||||
body_start = i
|
||||
break
|
||||
else:
|
||||
sys.exit("Could not find body start in DOCX")
|
||||
body = [l.strip() for l in lines[body_start:] if l.strip()]
|
||||
return '\n'.join(body)
|
||||
|
||||
|
||||
def extract_pdf_body(path):
|
||||
with open(path) as f:
|
||||
lines = f.readlines()
|
||||
|
||||
slug_re = re.compile(r'正念禅修十要素.*indd \d+')
|
||||
header_re = re.compile(
|
||||
r'^(The Mindful Peace Academy Collection|The Ten Key Elements of Mindfulness Meditation)$'
|
||||
)
|
||||
page_re = re.compile(r'^\d{1,3}$')
|
||||
skip_re = re.compile(
|
||||
r'^(I|II|III|IV|Three Basic Elements|The Three Key Elements of Samatha|'
|
||||
r'The Four Key Elements of Vipassana|Conclusion|Contents)$'
|
||||
)
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if 'Dhyana Tea program team' in line.strip():
|
||||
body_start = i
|
||||
break
|
||||
else:
|
||||
sys.exit("Could not find body start in PDF")
|
||||
|
||||
raw = []
|
||||
for line in lines[body_start:]:
|
||||
s = line.strip()
|
||||
if not s or s == '\x0c':
|
||||
continue
|
||||
if slug_re.search(s) or header_re.match(s) or page_re.match(s) or skip_re.match(s):
|
||||
continue
|
||||
raw.append(s)
|
||||
|
||||
# Join hyphenated line breaks
|
||||
joined = []
|
||||
i = 0
|
||||
while i < len(raw):
|
||||
line = raw[i]
|
||||
if line.rstrip().endswith('-') and i + 1 < len(raw):
|
||||
nxt = raw[i + 1].lstrip()
|
||||
if nxt and nxt[0].islower():
|
||||
joined.append(line.rstrip()[:-1] + nxt)
|
||||
i += 2
|
||||
continue
|
||||
joined.append(line)
|
||||
i += 1
|
||||
|
||||
body = ' '.join(joined)
|
||||
body = re.sub(r'\s+', ' ', body).strip()
|
||||
body = body.replace('L iving', 'Living')
|
||||
body = re.sub(r'T\s+he\b', 'The', body)
|
||||
return body
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit(f"Usage: {Path(__file__).name} <docx_path> <pdf_path>")
|
||||
|
||||
docx_path, pdf_path = sys.argv[1], sys.argv[2]
|
||||
|
||||
subprocess.run(
|
||||
['pandoc', docx_path, '-f', 'docx', '-t', 'plain', '--wrap=none',
|
||||
'-o', '/tmp/_docx_raw.txt'], check=True
|
||||
)
|
||||
subprocess.run(
|
||||
['pdftotext', '-layout', pdf_path, '/tmp/_pdf_raw.txt'], check=True
|
||||
)
|
||||
|
||||
docx_body = extract_docx_body('/tmp/_docx_raw.txt')
|
||||
pdf_body = extract_pdf_body('/tmp/_pdf_raw.txt')
|
||||
|
||||
Path(DOCX_TXT).write_text(docx_body)
|
||||
Path(PDF_TXT).write_text(pdf_body)
|
||||
|
||||
print(f"DOCX body → {DOCX_TXT} ({len(docx_body)} chars)")
|
||||
print(f"PDF body → {PDF_TXT} ({len(pdf_body)} chars)")
|
||||
Reference in New Issue
Block a user