diff --git a/terms-search/.gitignore b/terms-search/.gitignore new file mode 100644 index 0000000..cf57406 --- /dev/null +++ b/terms-search/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +__pypackages__/ diff --git a/terms-search/SKILL.md b/terms-search/SKILL.md new file mode 100644 index 0000000..a0aae18 --- /dev/null +++ b/terms-search/SKILL.md @@ -0,0 +1,55 @@ +--- +name: terms-search +description: Full-text search across the MPI term database. Use when translating or looking up Chinese-English Buddhist/MPI terminology. +category: research +--- + +# Terms Search + +Database: `/home/user/documents/mpi/terms-search/termlib.duckdb` +CLI: `/home/user/documents/mpi/terms-search/search.py` + +## Search + +``` +terms-search [limit] +terms-search loc: # filter by 出处 +terms-search src: # filter by source table +terms-search src:
# list all from table +``` + +Multi-word queries are ANDed. Searches both `zh` and `en` columns. + +## Source tables + +| src | rows | description | +|---|---|---| +| BAICKZ | 7,679 | Main term bank with example sentences | +| 佛教术语 | 1,795 | Buddhist terminology from 定稿书目术语库 | +| DoT定稿 | 896 | DoT final translation decisions | +| DoT初步 | 412 | DoT preliminary queries | +| 偈颂经文名言 | 263 | Verses and sutra quotes | +| 成语俗语 | 184 | Idioms and common expressions | +| 经论名 | 89 | Sutra/shastra titles | +| 内部特色词 | 87 | MPI internal terminology | +| 海内外建筑名称 | 28+9 | MPI building/place names | +| MPI组织架构 | 4+28 | MPI org structure | +| 导师金句 | 24 | Teacher quotes | +| 静心学堂课程 | 17+20 | Course names | +| 禅意项目 | 14+11 | Zen program terms | +| 公案 | 8 | Chan koans | + +## Direct DuckDB + +``` +duckdb /home/user/documents/mpi/terms-search/termlib.duckdb +``` + +Key tables: `unified_terms_flat` (zh, en, loc, source), individual source tables, `unified_terms` view. + +## Rebuilding + +Terms data comes from `/home/user/documents/mpi/guide/03 术语库/`. To rebuild: +1. Convert source xlsx/ods → CSV+YAML in `_output/` (see `/tmp/convert_sheets3.py`) +2. Rebuild DuckDB from CSVs (see `/tmp/duckdb_import.py` and `/tmp/fix_dot.py`) +3. Materialize `unified_terms_flat` view → table for performance diff --git a/terms-search/search.py b/terms-search/search.py new file mode 100755 index 0000000..ffc0baf --- /dev/null +++ b/terms-search/search.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Full-text search over MPI term database. Queries unified_terms_flat via DuckDB LIKE.""" +import sys, os +import duckdb + +DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'termlib.duckdb') + +def search(con, query, loc_filter=None, src_filter=None, limit=None): + terms = query.split() + clauses = [] + params = [] + + for t in terms: + like = f'%{t}%' + clauses.append('(zh LIKE ? OR en LIKE ?)') + params.extend([like, like]) + + where = ' AND '.join(clauses) if clauses else '1=1' + + if loc_filter: + where += ' AND loc LIKE ?' + params.append(f'%{loc_filter}%') + if src_filter: + where += ' AND source = ?' + params.append(src_filter) + + sql = f'SELECT zh, en, loc, source FROM unified_terms_flat WHERE {where}' + if limit is not None: + sql += ' LIMIT ?' + params.append(limit) + + return con.execute(sql, params).fetchall() + +def main(): + if len(sys.argv) < 2: + print("Usage: terms-search [limit]") + print(" queries: '空性', 'emptiness 中观'") + print() + print("Special prefixes (can be standalone or combined with query):") + print(" loc: — filter by loc (e.g. loc:心经)") + print(" src:
— filter by source table (e.g. src:佛教术语)") + print() + print("Examples:") + print(" terms-search 空性") + print(" terms-search 'suffering 苦' loc:心经") + print(" terms-search src:公案") + sys.exit(1) + + raw = sys.argv[1] + limit = int(sys.argv[2]) if len(sys.argv) > 2 else 20 + + # Parse prefixes + loc_filter = None + src_filter = None + query_parts = [] + + for token in raw.split(): + if token.startswith('loc:'): + loc_filter = token[4:] + elif token.startswith('src:'): + src_filter = token[4:] + else: + query_parts.append(token) + + query = ' '.join(query_parts) + + con = duckdb.connect(DB, read_only=True) + + if not query and not loc_filter and not src_filter: + print("No search terms or filters. Usage: terms-search [limit]") + return + + rows = search(con, query, loc_filter, src_filter, limit) + + if not rows: + print(f"No results for: {raw}") + return + + print(f"Results: {len(rows)}") + print() + for zh, en, loc, src in rows: + print(f'zh: {zh}') + print(f'en: {en}') + print(f'loc: {loc or "-"} | src: {src}') + print() + + con.close() + +if __name__ == '__main__': + main() diff --git a/terms-search/server.py b/terms-search/server.py new file mode 100644 index 0000000..26be79a --- /dev/null +++ b/terms-search/server.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +import sys, os +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import duckdb +from flask import Flask, request, jsonify, render_template_string +import search as s + +app = Flask(__name__) + +HTML = r''' + +Terms Search +

Terms Search

+
+ + + + + + + +{% if searched %} +{% if error %}

Error: {{ error }}

+{% else %} +

{{ count }} results

+
+ + {% for r in results %} + + + + + + + {% endfor %} +
ZHENLOCSRC
{{ r.zh }}{{ r.en }}{{ r.loc or '' }}{{ r.source }}
+{% endif %} +{% endif %} +''' + +def do_search(q, loc, src, limit): + with duckdb.connect(s.DB, read_only=True) as con: + rows = s.search(con, q, loc, src, limit) + return [{'zh': r[0], 'en': r[1], 'loc': r[2] or None, 'source': r[3]} for r in rows] + +@app.route('/search') +def search(): + try: + q = request.args.get('q', '') + loc = request.args.get('loc') + src = request.args.get('src') + limit = request.args.get('limit', type=int) + results = do_search(q, loc, src, limit) + return jsonify({'count': len(results), 'results': results}) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/sources') +def sources(): + try: + with duckdb.connect(s.DB, read_only=True) as con: + rows = con.execute( + 'SELECT source, COUNT(*) AS cnt FROM unified_terms_flat GROUP BY source ORDER BY cnt DESC' + ).fetchall() + return jsonify([{'source': r[0], 'count': r[1]} for r in rows]) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +@app.route('/') +def ui(): + q = request.args.get('q', '') + loc = request.args.get('loc', '') + src = request.args.get('src', '') + limit = request.args.get('limit', '') + searched = bool(request.args) + results = [] + count = 0 + error = None + if searched and (q or loc or src): + try: + lim = int(limit) if limit else None + results = do_search(q, loc or None, src or None, lim) + count = len(results) + except Exception as e: + error = str(e) + return render_template_string(HTML, q=q, loc=loc, src=src, limit=limit, + searched=searched, results=results, count=count, error=error) + +if __name__ == '__main__': + app.run(port=8910)