terms-search: first working version
This commit is contained in:
iacore
@@ -0,0 +1,2 @@
|
||||
__pycache__/
|
||||
__pypackages__/
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
name: terms-search
|
||||
description: Full-text search across the MPI term database. Use when translating or looking up Chinese-English Buddhist/MPI terminology.
|
||||
category: research
|
||||
---
|
||||
|
||||
# Terms Search
|
||||
|
||||
Database: `/home/user/documents/mpi/terms-search/termlib.duckdb`
|
||||
CLI: `/home/user/documents/mpi/terms-search/search.py`
|
||||
|
||||
## Search
|
||||
|
||||
```
|
||||
terms-search <query> [limit]
|
||||
terms-search <query> loc:<source> # filter by 出处
|
||||
terms-search <query> src:<table> # filter by source table
|
||||
terms-search src:<table> # list all from table
|
||||
```
|
||||
|
||||
Multi-word queries are ANDed. Searches both `zh` and `en` columns.
|
||||
|
||||
## Source tables
|
||||
|
||||
| src | rows | description |
|
||||
|---|---|---|
|
||||
| BAICKZ | 7,679 | Main term bank with example sentences |
|
||||
| 佛教术语 | 1,795 | Buddhist terminology from 定稿书目术语库 |
|
||||
| DoT定稿 | 896 | DoT final translation decisions |
|
||||
| DoT初步 | 412 | DoT preliminary queries |
|
||||
| 偈颂经文名言 | 263 | Verses and sutra quotes |
|
||||
| 成语俗语 | 184 | Idioms and common expressions |
|
||||
| 经论名 | 89 | Sutra/shastra titles |
|
||||
| 内部特色词 | 87 | MPI internal terminology |
|
||||
| 海内外建筑名称 | 28+9 | MPI building/place names |
|
||||
| MPI组织架构 | 4+28 | MPI org structure |
|
||||
| 导师金句 | 24 | Teacher quotes |
|
||||
| 静心学堂课程 | 17+20 | Course names |
|
||||
| 禅意项目 | 14+11 | Zen program terms |
|
||||
| 公案 | 8 | Chan koans |
|
||||
|
||||
## Direct DuckDB
|
||||
|
||||
```
|
||||
duckdb /home/user/documents/mpi/terms-search/termlib.duckdb
|
||||
```
|
||||
|
||||
Key tables: `unified_terms_flat` (zh, en, loc, source), individual source tables, `unified_terms` view.
|
||||
|
||||
## Rebuilding
|
||||
|
||||
Terms data comes from `/home/user/documents/mpi/guide/03 术语库/`. To rebuild:
|
||||
1. Convert source xlsx/ods → CSV+YAML in `_output/` (see `/tmp/convert_sheets3.py`)
|
||||
2. Rebuild DuckDB from CSVs (see `/tmp/duckdb_import.py` and `/tmp/fix_dot.py`)
|
||||
3. Materialize `unified_terms_flat` view → table for performance
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Full-text search over MPI term database. Queries unified_terms_flat via DuckDB LIKE."""
|
||||
import sys, os
|
||||
import duckdb
|
||||
|
||||
DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'termlib.duckdb')
|
||||
|
||||
def search(con, query, loc_filter=None, src_filter=None, limit=None):
|
||||
terms = query.split()
|
||||
clauses = []
|
||||
params = []
|
||||
|
||||
for t in terms:
|
||||
like = f'%{t}%'
|
||||
clauses.append('(zh LIKE ? OR en LIKE ?)')
|
||||
params.extend([like, like])
|
||||
|
||||
where = ' AND '.join(clauses) if clauses else '1=1'
|
||||
|
||||
if loc_filter:
|
||||
where += ' AND loc LIKE ?'
|
||||
params.append(f'%{loc_filter}%')
|
||||
if src_filter:
|
||||
where += ' AND source = ?'
|
||||
params.append(src_filter)
|
||||
|
||||
sql = f'SELECT zh, en, loc, source FROM unified_terms_flat WHERE {where}'
|
||||
if limit is not None:
|
||||
sql += ' LIMIT ?'
|
||||
params.append(limit)
|
||||
|
||||
return con.execute(sql, params).fetchall()
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: terms-search <query> [limit]")
|
||||
print(" queries: '空性', 'emptiness 中观'")
|
||||
print()
|
||||
print("Special prefixes (can be standalone or combined with query):")
|
||||
print(" loc:<source> — filter by loc (e.g. loc:心经)")
|
||||
print(" src:<table> — filter by source table (e.g. src:佛教术语)")
|
||||
print()
|
||||
print("Examples:")
|
||||
print(" terms-search 空性")
|
||||
print(" terms-search 'suffering 苦' loc:心经")
|
||||
print(" terms-search src:公案")
|
||||
sys.exit(1)
|
||||
|
||||
raw = sys.argv[1]
|
||||
limit = int(sys.argv[2]) if len(sys.argv) > 2 else 20
|
||||
|
||||
# Parse prefixes
|
||||
loc_filter = None
|
||||
src_filter = None
|
||||
query_parts = []
|
||||
|
||||
for token in raw.split():
|
||||
if token.startswith('loc:'):
|
||||
loc_filter = token[4:]
|
||||
elif token.startswith('src:'):
|
||||
src_filter = token[4:]
|
||||
else:
|
||||
query_parts.append(token)
|
||||
|
||||
query = ' '.join(query_parts)
|
||||
|
||||
con = duckdb.connect(DB, read_only=True)
|
||||
|
||||
if not query and not loc_filter and not src_filter:
|
||||
print("No search terms or filters. Usage: terms-search <query> [limit]")
|
||||
return
|
||||
|
||||
rows = search(con, query, loc_filter, src_filter, limit)
|
||||
|
||||
if not rows:
|
||||
print(f"No results for: {raw}")
|
||||
return
|
||||
|
||||
print(f"Results: {len(rows)}")
|
||||
print()
|
||||
for zh, en, loc, src in rows:
|
||||
print(f'zh: {zh}')
|
||||
print(f'en: {en}')
|
||||
print(f'loc: {loc or "-"} | src: {src}')
|
||||
print()
|
||||
|
||||
con.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import duckdb
|
||||
from flask import Flask, request, jsonify, render_template_string
|
||||
import search as s
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
HTML = r'''<!DOCTYPE html>
|
||||
<meta charset="utf-8">
|
||||
<title>Terms Search</title>
|
||||
<h1>Terms Search</h1>
|
||||
<form method="get">
|
||||
<input name="q" value="{{ q }}" placeholder="query (e.g. 空性 emptiness)">
|
||||
<input name="loc" value="{{ loc }}" placeholder="loc filter">
|
||||
<input name="src" value="{{ src }}" placeholder="src filter">
|
||||
<input name="limit" value="{{ limit }}" size="4" placeholder="limit">
|
||||
<button type="submit">Search</button>
|
||||
</form>
|
||||
|
||||
{% if searched %}
|
||||
{% if error %}<p>Error: {{ error }}</p>
|
||||
{% else %}
|
||||
<p>{{ count }} results</p>
|
||||
<table border="1" cellpadding="4" cellspacing="0">
|
||||
<tr><th>ZH</th><th>EN</th><th>LOC</th><th>SRC</th></tr>
|
||||
{% for r in results %}
|
||||
<tr>
|
||||
<td>{{ r.zh }}</td>
|
||||
<td>{{ r.en }}</td>
|
||||
<td>{{ r.loc or '' }}</td>
|
||||
<td>{{ r.source }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
'''
|
||||
|
||||
def do_search(q, loc, src, limit):
|
||||
with duckdb.connect(s.DB, read_only=True) as con:
|
||||
rows = s.search(con, q, loc, src, limit)
|
||||
return [{'zh': r[0], 'en': r[1], 'loc': r[2] or None, 'source': r[3]} for r in rows]
|
||||
|
||||
@app.route('/search')
|
||||
def search():
|
||||
try:
|
||||
q = request.args.get('q', '')
|
||||
loc = request.args.get('loc')
|
||||
src = request.args.get('src')
|
||||
limit = request.args.get('limit', type=int)
|
||||
results = do_search(q, loc, src, limit)
|
||||
return jsonify({'count': len(results), 'results': results})
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
@app.route('/sources')
|
||||
def sources():
|
||||
try:
|
||||
with duckdb.connect(s.DB, read_only=True) as con:
|
||||
rows = con.execute(
|
||||
'SELECT source, COUNT(*) AS cnt FROM unified_terms_flat GROUP BY source ORDER BY cnt DESC'
|
||||
).fetchall()
|
||||
return jsonify([{'source': r[0], 'count': r[1]} for r in rows])
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
@app.route('/')
|
||||
def ui():
|
||||
q = request.args.get('q', '')
|
||||
loc = request.args.get('loc', '')
|
||||
src = request.args.get('src', '')
|
||||
limit = request.args.get('limit', '')
|
||||
searched = bool(request.args)
|
||||
results = []
|
||||
count = 0
|
||||
error = None
|
||||
if searched and (q or loc or src):
|
||||
try:
|
||||
lim = int(limit) if limit else None
|
||||
results = do_search(q, loc or None, src or None, lim)
|
||||
count = len(results)
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
return render_template_string(HTML, q=q, loc=loc, src=src, limit=limit,
|
||||
searched=searched, results=results, count=count, error=error)
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(port=8910)
|
||||
Reference in New Issue
Block a user