From 12134faea9fe27d01d415bc636b361d7afd83cf4 Mon Sep 17 00:00:00 2001 From: iacore Date: Thu, 18 Jun 2026 08:24:11 +0800 Subject: [PATCH] skills: update skills --- .gitignore | 1 + AGENTS.md | 2 +- .../gen-bilingual-buddhist-attitude-185a23.py | 10 +- scripts/gen-bilingual-buddhist-attitude.py | 12 +- scripts/gen-bilingual-life-design-de8422.py | 10 +- skills/chinese-text-normalize/SKILL.md | 43 +++++++ .../references/bold-fragments.md | 110 ++++++++++++++++++ skills/readme.dj | 2 +- skills/terms-search/SKILL.md | 16 +-- .../references/translation-alignment.md | 2 +- skills/translation-review/SKILL.md | 2 +- .../references/terms-db-alignment.md | 4 +- .../references/translation-pitfalls.md | 2 +- skills/translation/SKILL.md | 1 + .../references/bilingual-format.md | 36 ++++++ .../references/markdown-to-djot.md | 17 +++ 16 files changed, 242 insertions(+), 28 deletions(-) create mode 100644 skills/chinese-text-normalize/references/bold-fragments.md create mode 100644 skills/translation/references/bilingual-format.md diff --git a/.gitignore b/.gitignore index 930cce6..c224253 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .codegraph/ +.firecrawl/ diff --git a/AGENTS.md b/AGENTS.md index e62b680..7a4c371 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ Skills in `skills/`. Loaded via `~/.hermes/config.yaml`: ```yaml skills: external_dirs: - - /home/user/documents/mpi/skills + - $MPI_PROJECT_ROOT/skills ``` {% `hermes config set` stringifies list values — edit config.yaml directly. %} diff --git a/scripts/gen-bilingual-buddhist-attitude-185a23.py b/scripts/gen-bilingual-buddhist-attitude-185a23.py index 7e68efb..23c259b 100644 --- a/scripts/gen-bilingual-buddhist-attitude-185a23.py +++ b/scripts/gen-bilingual-buddhist-attitude-185a23.py @@ -1,10 +1,12 @@ """Generate bilingual.dj from DOCX manuscript only (no PDF). Source: Chinese from DOCX. Target: English from DOCX. """ -import re, subprocess, os +import re, subprocess +from pathlib import Path -DOCX = "/home/user/documents/mpi/translate-files/佛教徒的人生态度/定稿 佛教徒的人生态度 善鑫慧炬照禅道靖妙一观轩慈德20260527.docx" -OUT_DIR = "/home/user/documents/mpi/translate-files/佛教徒的人生态度" +ROOT = Path(__file__).resolve().parent.parent +DOCX = ROOT / "translate-files/佛教徒的人生态度/定稿 佛教徒的人生态度 善鑫慧炬照禅道靖妙一观轩慈德20260527.docx" +OUT_DIR = ROOT / "translate-files/佛教徒的人生态度" def has_cjk(s): return any('\u4e00' <= c <= '\u9fff' for c in s) @@ -124,5 +126,5 @@ if __name__ == '__main__': pairs = extract_pairs(text) print(f" Pairs: {len(pairs)}") - out = os.path.join(OUT_DIR, 'bilingual.dj') + out = OUT_DIR / "bilingual.dj" generate(pairs, out) diff --git a/scripts/gen-bilingual-buddhist-attitude.py b/scripts/gen-bilingual-buddhist-attitude.py index 7ec0c7a..aaec54a 100644 --- a/scripts/gen-bilingual-buddhist-attitude.py +++ b/scripts/gen-bilingual-buddhist-attitude.py @@ -6,11 +6,13 @@ Target: English from PDF typeset. Strategy: find each DOCX English paragraph in PDF body, extract the PDF text region for that paragraph using position boundaries. """ -import re, subprocess, os +import re, subprocess +from pathlib import Path -DOCX = "/home/user/documents/mpi/translate-files/佛教徒的人生态度/定稿 佛教徒的人生态度 善鑫慧炬照禅道靖妙一观轩慈德20260527.docx" -PDF = "/home/user/documents/mpi/translate-files/佛教徒的人生态度/0607-二排-果澄-佛教徒的人生态度-一校-多人-0607.pdf" -OUT_DIR = "/home/user/documents/mpi/translate-files/佛教徒的人生态度" +ROOT = Path(__file__).resolve().parent.parent +DOCX = ROOT / "translate-files/佛教徒的人生态度/定稿 佛教徒的人生态度 善鑫慧炬照禅道靖妙一观轩慈德20260527.docx" +PDF = ROOT / "translate-files/佛教徒的人生态度/0607-二排-果澄-佛教徒的人生态度-一校-多人-0607.pdf" +OUT_DIR = ROOT / "translate-files/佛教徒的人生态度" def has_cjk(s): @@ -256,5 +258,5 @@ if __name__ == '__main__': print("Extracting segments...") segments = extract_segments(pdf_body, positions) - out = os.path.join(OUT_DIR, 'bilingual.dj') + out = OUT_DIR / "bilingual.dj" generate(pairs, segments, out) diff --git a/scripts/gen-bilingual-life-design-de8422.py b/scripts/gen-bilingual-life-design-de8422.py index 90f7063..75bc370 100644 --- a/scripts/gen-bilingual-life-design-de8422.py +++ b/scripts/gen-bilingual-life-design-de8422.py @@ -1,10 +1,12 @@ """Generate bilingual.dj from DOCX for 「生命也可以被设计的」. One-pass approach: walk interleaved paragraphs, handle multi-CN sequences. """ -import re, subprocess, os +import re, subprocess +from pathlib import Path -DOCX = "/home/user/documents/mpi/translate-files/生命也可以被设计的/中英文定稿-260324-生命也是可以被设计的-妙一宽山静雅初翻 慈鎏妙一审议 宽山定稿.docx" -OUT_DIR = "/home/user/documents/mpi/translate-files/生命也可以被设计的" +ROOT = Path(__file__).resolve().parent.parent +DOCX = ROOT / "translate-files/生命也可以被设计的/中英文定稿-260324-生命也是可以被设计的-妙一宽山静雅初翻 慈鎏妙一审议 宽山定稿.docx" +OUT_DIR = ROOT / "translate-files/生命也可以被设计的" def has_cjk(s): return any('\u4e00' <= c <= '\u9fff' for c in s) @@ -177,5 +179,5 @@ if __name__ == '__main__': pairs = extract_body_pairs(text) print(f" Pairs: {len(pairs)}") - out = os.path.join(OUT_DIR, 'bilingual.dj') + out = OUT_DIR / "bilingual.dj" generate(toc_cn, toc_en, pairs, out) diff --git a/skills/chinese-text-normalize/SKILL.md b/skills/chinese-text-normalize/SKILL.md index 8fa848b..aa576f4 100644 --- a/skills/chinese-text-normalize/SKILL.md +++ b/skills/chinese-text-normalize/SKILL.md @@ -34,6 +34,49 @@ The script handles three file patterns: - Inline page numbers (standalone 1-2 digit lines) - Trailing blank lines +## Beyond the script: bold fragments, conjoined paragraphs, encoding + +The script handles simple fixed-width body text. Some PDF→markdown conversions produce more complex artifacts that need manual multi-pass Python scripts via `execute_code`: + +### Bold marker fragmentation + +`**...text...**` blocks split across blank lines with stray `**` at fragment boundaries: + +``` +**第三条 特色——依据五大要素，构建次第修学。营造良好氛围，提供有效** + +引导。 +``` + +**Fix**: Join fragments, remove stray `**` from join point, add closing `**` to final result. See `references/bold-fragments.md` for full pattern catalog and multi-pass workflow. + +**Critical pitfall**: Do NOT join lines where BOTH the first and second line are complete bold blocks (start+end with `**`). These are separate entries, not fragments: +``` +**第一条 ...之道。** ← complete bold item + ← blank line +**第二条 ...合一。** ← complete bold item (DON'T JOIN) +``` + +### Conjoined paragraphs + +Separate paragraphs/sections merged into one line — opposite problem to the script. Common in song lyrics, dense instructional sections. Requires semantic splitting. See `references/bold-fragments.md`. + +### Encoding artifacts + +`川` (U+5DDD) replacing `"` (curly quote) — search-and-replace: `" 道理川` → `"道理"`, `" 自己的川` → `"自己的"`. + +### Multi-pass approach + +1. **Pass 1**: Join word fragments split by blank lines (conservative — only when current line doesn't end with `。！？` or is NOT a complete bold block) +2. **Pass 2**: Split obviously conjoined paragraphs (manual string replacements for known patterns) +3. **Pass 3**: Fix stray bold markers, encoding artifacts, stray page numbers +4. Verify after each pass; revert with `git checkout` if over-aggressive + +### Heuristic pitfalls + +- **Short-line join** (< 15 chars): Over-joins section headers with body, Q&A pairs (`正念是什么？\n\n就是...`). Only use for clear word-fragment continuations. +- **Bold-end join**: Lines ending with `**` are ambiguous — either broken bold fragment or complete bold item. Check if the content before `**` forms a complete sentence (ends with `。`). + ## Pitfalls - **TOC detection boundaries**: The vertical TOC end is detected by finding the first line with 3+ CJK characters. If a page number like "2" sits between TOC and body, it lands in the TOC section — harmless but visible. diff --git a/skills/chinese-text-normalize/references/bold-fragments.md b/skills/chinese-text-normalize/references/bold-fragments.md new file mode 100644 index 0000000..e1fc972 --- /dev/null +++ b/skills/chinese-text-normalize/references/bold-fragments.md @@ -0,0 +1,110 @@ +# Bold fragments & conjoined paragraphs — fix patterns + +From session fixing `静心学堂学员手册.md` (1575→1478 lines, ~100 fixes). + +## Pattern A: Bold marker fragmentation + +**Problem**: `**...text...**` block split across blank line with stray `**` markers: + +``` +**第三条 特色——依据五大要素，构建次第修学。营造良好氛围，提供有效** + +引导。 +``` + +**Detection**: Line ends with `**`, next non-blank line continues the sentence (does NOT start with `**`). + +**Fix** (Python): +```python +# curr ends with **, nxt is continuation (no leading **) +curr_fixed = curr.rstrip()[:-2].rstrip() # strip trailing ** +nxt_fixed = nxt.lstrip() +if nxt_fixed.endswith('**'): + nxt_fixed = nxt_fixed[:-2].rstrip() + joined = curr_fixed + nxt_fixed + '**' +else: + joined = curr_fixed + nxt_fixed # lost closing ** — may need manual fix +``` + +### Anti-pattern: Complete bold items + +Do NOT join when BOTH lines are complete bold blocks (start+end with `**`): + +``` +**第一条 ...之道。** ← DON'T JOIN + ← blank line +**第二条 ...合一。** ← DON'T JOIN +``` + +**Detection**: Both `curr` and `nxt` start with `**` and end with `**`. + +## Pattern B: Conjoined paragraphs (Type 2) + +Separate sections merged into one line. Common cases: + +### Section headers merged with body +``` +导言：这本指引怎么用这本指引是什么这是一本修学地图... +``` +→ Split into: +``` +导言：这本指引怎么用 + +这本指引是什么 + +这是一本修学地图... +``` + +### Song titles merged mid-lyrics +``` +...生生世世不再久违《菩提花开》如果你渴求一滴水... +``` +→ Split into: +``` +...生生世世不再久违 + +### 《菩提花开》 + +如果你渴求一滴水... +``` + +### List items merged into one line +``` +不在班级群发布...不从事违法活动不在班级平台拉拢... +``` +→ Split into bullet list: +``` +- 不在班级群发布... +- 不从事违法活动 +- 不在班级平台拉拢... +``` + +**Approach**: Manual string replacements for known patterns. Regex is unreliable for semantic splits. + +## Pattern C: Stray page numbers + +Standalone digits at line ends, often from PDF page number artifacts: +- `42`, `43`, `46`, `47` at end of content lines + +**Fix**: Strip trailing digits that aren't part of dates, durations, or course numbers. + +## Pattern D: Encoding artifacts + +`川` (U+5DDD) replacing curly quotes `"` (U+201C/U+201D): +``` +把" 道理川变成" 自己的川 → 把"道理"变成"自己的" +``` + +**Fix**: Replace `" 道理川` → `"道理"`, `" 自己的川` → `"自己的"`. + +## Multi-pass workflow + +1. **Pass 1 — Join word fragments**: Scan for lines split by blank line where first line doesn't end with `。！？` and neither line is structural (header/list/table). Skip complete bold items. +2. **Pass 2 — Split conjoined**: Apply known string replacements for merged sections, song transitions, list items. +3. **Pass 3 — Clean artifacts**: Fix stray `**` markers, encoding issues, stray page numbers. +4. **Verify**: `git diff` after each pass; `git checkout` if over-aggressive. + +## Rejected heuristics + +- **Short-line join** (< 15 chars): Over-joins section headers (`中级和高级（以后的事）`) with body, and Q&A pairs (`正念是什么？\n\n就是...`). Only use for clear mid-word fragments. +- **Blind `**` stripping**: Removes valid bold formatting from complete bold items. diff --git a/skills/readme.dj b/skills/readme.dj index 42eeb6d..1b79e2d 100644 --- a/skills/readme.dj +++ b/skills/readme.dj @@ -9,7 +9,7 @@ Add to `~/.hermes/config.yaml`: ```yaml skills: external_dirs: - - /home/user/documents/mpi/skills + - $MPI_PROJECT_ROOT/skills ``` {% Edit config.yaml directly — `hermes config set` stores list values as strings. %} diff --git a/skills/terms-search/SKILL.md b/skills/terms-search/SKILL.md index 1e6a864..3ac9570 100644 --- a/skills/terms-search/SKILL.md +++ b/skills/terms-search/SKILL.md @@ -6,14 +6,14 @@ category: research # Terms Search -Database: `/home/user/documents/mpi/terms-search/termlib.duckdb` -CLI: `/home/user/documents/mpi/terms-search/search.py` -Server: `/home/user/documents/mpi/terms-search/server.py` +Database: `$MPI_PROJECT_ROOT/terms-search/termlib.duckdb` +CLI: `$MPI_PROJECT_ROOT/terms-search/search.py` +Server: `$MPI_PROJECT_ROOT/terms-search/server.py` ## CLI (preferred) ``` -/home/user/documents/mpi/terms-search/search.py [limit] +$MPI_PROJECT_ROOT/terms-search/search.py [limit] ``` Multi-word queries are ANDed. Searches both `zh` and `en` columns. @@ -22,7 +22,7 @@ Multi-word queries are ANDed. Searches both `zh` and `en` columns. ```python import sys -sys.path.insert(0, '/home/user/documents/mpi/terms-search') +sys.path.insert(0, '$MPI_PROJECT_ROOT/terms-search') from search import search results = search("空性", limit=5, src="DoT定稿") # → list of {zh, en, loc, source} dicts @@ -32,7 +32,7 @@ Use this inside `execute_code` scripts for batch lookups — no subprocess neede ## HTTP API (use only when CLI is insufficient) -Start: `python3 /home/user/documents/mpi/terms-search/server.py` (port 8910) +Start: `python3 $MPI_PROJECT_ROOT/terms-search/server.py` (port 8910) - `GET /` — plain HTML UI (form + results table, no CSS) - `GET /` — plain HTML UI (form + results table, no CSS) @@ -64,14 +64,14 @@ Errors return `{"error": "..."}` with HTTP 500 (API) or shown inline (UI). ## Direct DuckDB ``` -duckdb /home/user/documents/mpi/terms-search/termlib.duckdb +duckdb $MPI_PROJECT_ROOT/terms-search/termlib.duckdb ``` Key tables: `unified_terms_flat` (zh, en, loc, source), individual source tables, `unified_terms` view. ## Rebuilding -Terms data comes from `/home/user/documents/mpi/guide/03 术语库/`. To rebuild: +Terms data comes from `$MPI_PROJECT_ROOT/guide/03 术语库/`. To rebuild: 1. Convert source xlsx/ods → CSV+YAML in `_output/` 2. Rebuild DuckDB from CSVs 3. Materialize `unified_terms_flat` view → table for performance diff --git a/skills/terms-search/references/translation-alignment.md b/skills/terms-search/references/translation-alignment.md index ae26a75..3be44a3 100644 --- a/skills/terms-search/references/translation-alignment.md +++ b/skills/terms-search/references/translation-alignment.md @@ -10,7 +10,7 @@ After producing a first-pass translation, or when the user asks to check termino 1. Read the full translated file. Extract all Chinese terms from `{% "TERM" (pinyin) = ENGLISH ... %}` blocks. -2. Start the search server: `python3 /home/user/documents/mpi/terms-search/server.py &` (port 8910). It may already be running — check with `curl -s http://localhost:8910/`. +2. Start the search server: `python3 $MPI_PROJECT_ROOT/terms-search/server.py &` (port 8910). It may already be running — check with `curl -s http://localhost:8910/`. 3. Batch-search each term via the HTTP API: ``` diff --git a/skills/translation-review/SKILL.md b/skills/translation-review/SKILL.md index b8446ff..1cf8c0d 100644 --- a/skills/translation-review/SKILL.md +++ b/skills/translation-review/SKILL.md @@ -87,7 +87,7 @@ Use `terminal: cat` — `read_file` deduplicates within a session. **Terms database drift** (systematic): - Cross-reference glossary terms against the MPI terms database -- CLI preferred: `python3 /home/user/documents/mpi/terms-search/search.py `. For a review, batch many queries in one `execute_code` script (subprocess loop) — one terminal call per term is slow and noisy. +- CLI preferred: `python3 $MPI_PROJECT_ROOT/terms-search/search.py `. For a review, batch many queries in one `execute_code` script (subprocess loop) — one terminal call per term is slow and noisy. - Source priority: DoT定稿 > 内部特色词 > 佛教术语 > 经论名 - Fix both glossary comments AND body text - See `references/terms-db-alignment.md` for batch-lookup patterns diff --git a/skills/translation-review/references/terms-db-alignment.md b/skills/translation-review/references/terms-db-alignment.md index e41e681..6f48b4f 100644 --- a/skills/translation-review/references/terms-db-alignment.md +++ b/skills/translation-review/references/terms-db-alignment.md @@ -8,7 +8,7 @@ Import directly in `execute_code` scripts — no subprocess, no server, no text ```python import sys -sys.path.insert(0, '/home/user/documents/mpi/terms-search') +sys.path.insert(0, '$MPI_PROJECT_ROOT/terms-search') from search import search results = search("三级修学", limit=5) @@ -20,7 +20,7 @@ results = search("空性", loc="心经", src="DoT定稿", limit=5) ```python import sys -sys.path.insert(0, '/home/user/documents/mpi/terms-search') +sys.path.insert(0, '$MPI_PROJECT_ROOT/terms-search') from search import search terms = ["三无漏学", "八步三禅", "闻思修", ...] diff --git a/skills/translation-review/references/translation-pitfalls.md b/skills/translation-review/references/translation-pitfalls.md index cb8ffa5..5284815 100644 --- a/skills/translation-review/references/translation-pitfalls.md +++ b/skills/translation-review/references/translation-pitfalls.md @@ -73,5 +73,5 @@ initial/middle/great scope." ### Translating before consulting terms DB Always search key terms BEFORE translating. The dharma-translation skill says to do -this, but it's easy to skip. Use the CLI: `/home/user/documents/mpi/terms-search/search.py `. +this, but it's easy to skip. Use the CLI: `$MPI_PROJECT_ROOT/terms-search/search.py `. Prioritize DoT定稿 > 内部特色词 > 佛教术语 > 经论名. diff --git a/skills/translation/SKILL.md b/skills/translation/SKILL.md index c164c5f..2ba2347 100644 --- a/skills/translation/SKILL.md +++ b/skills/translation/SKILL.md @@ -56,4 +56,5 @@ When translating guided meditation scripts, exercise guides, or posture instruct - `references/meditation-translation.md` — lighter workflow for meditation/mindfulness content - `references/markdown-to-djot.md` — converting .docx.md to .dj for translation prep +- `references/bilingual-format.md` — bilingual.dj layout: source/target adjacent, blank separator between pairs - `references/diacritics-convention.md` — diacritics rules diff --git a/skills/translation/references/bilingual-format.md b/skills/translation/references/bilingual-format.md new file mode 100644 index 0000000..188ce45 --- /dev/null +++ b/skills/translation/references/bilingual-format.md @@ -0,0 +1,36 @@ +# Bilingual DJ Format + +## Layout + +Each pair: source line immediately followed by target line. Blank line separates pairs. + +``` +source-line +target-line + +source-line +target-line +``` + +NOT: +``` +source-line + ← WRONG: extra blank between source and target +target-line +``` + +## Creating initial bilingual from source only + +Every source line gets an empty target placeholder + blank separator: + +``` +source-A + +source-B +``` + +(2 blank lines between consecutive source lines: empty target + separator.) + +## Verification + +Source line count × 3 − 1 = bilingual line count (before trailing newline strip). diff --git a/skills/translation/references/markdown-to-djot.md b/skills/translation/references/markdown-to-djot.md index c4bdc29..79aa3cf 100644 --- a/skills/translation/references/markdown-to-djot.md +++ b/skills/translation/references/markdown-to-djot.md @@ -11,6 +11,23 @@ sed -n '1,218p' combined.md > a1.md sed -n '220,282p' combined.md > a2.md ``` +## TOC stripping + +Pandoc docx→md produces a markdown TOC with tab-separated page numbers: + +```markdown +[一、对佛教的感悟\t1](#一、对佛教的感悟) +[二、佛教与人类文明\t5](#二、佛教与人类文明) +``` + +Strip before conversion: + +```bash +sed -i '/^\[.*\t.*\](#.*)$/d' input.md +``` + +Or in Python: skip lines matching `line.startswith("[") and "\t" in line and "](#" in line`. + ## Heading anchor cleanup Pandoc's docx→md conversion adds `{#heading-id}` anchors to every heading: