Migrate from duckdb to sqlite

skills/mpi-terms-search/SKILL.md

@@ -6,7 +6,7 @@ category: research
 
 # Terms Search
 
-Database: `toolkit/terms-database/termlib.duckdb`
+Database: `toolkit/terms-database/termlib.sqlite` (SQLite)
 CLI: `toolkit/terms-database/search.py`
 Server: `toolkit/terms-database/server.py`
 
@@ -61,20 +61,19 @@ Errors return `{"error": "..."}` with HTTP 500 (API) or shown inline (UI).
 | 禅意项目 | 14+11 | Zen program terms |
 | 公案 | 8 | Chan koans |
 
-## Direct DuckDB
+## Direct SQLite
 
 ```
-duckdb toolkit/terms-database/termlib.duckdb
+sqlite3 toolkit/terms-database/termlib.sqlite
 ```
 
-Key tables: `unified_terms_flat` (zh, en, loc, source), individual source tables, `unified_terms` view.
+Key table: `terms` (zh, en, loc, source).
 
 ## Rebuilding
 
 Terms data comes from `guide/03 术语库/`. To rebuild:
 1. Convert source xlsx/ods → CSV+YAML in `_output/`
-2. Rebuild DuckDB from CSVs
+2. Load CSVs into SQLite as the `terms` table (zh, en, loc, source)
-3. Materialize `unified_terms_flat` view → table for performance
 
 **Full rebuild pipeline:** See `references/termbase-rebuild.md` (absorbed from the `termbase-management` skill).
 

skills/mpi-terms-search/references/termbase-rebuild.md

@@ -1,11 +1,11 @@
 # Termbase Rebuild (from absorbed termbase-management)
 
-How to rebuild the terms DuckDB from source spreadsheets. This is the full pipeline from the now-archived `termbase-management` skill.
+How to rebuild the terms SQLite database from source spreadsheets. This is the full pipeline from the now-archived `termbase-management` skill.
 
 ## Prerequisites
 
 ```bash
-pip install openpyxl odfpy pyyaml duckdb
+pip install openpyxl odfpy pyyaml
 ```
 
 ## Step 1: Inspect spreadsheet structure
@@ -69,17 +69,23 @@ def dedup_headers(headers):
 - **CSV**: `csv.writer` — column-major, preserves all raw data
 - **YAML**: `yaml.dump(data, allow_unicode=True, default_flow_style=False, sort_keys=False, width=200)` — list of dicts
 
-## Step 3: Load into DuckDB
+## Step 3: Load into SQLite
 
 ```python
-import duckdb
+import sqlite3
-con = duckdb.connect('termlib.duckdb')
+import csv
 
-# Simple CSVs work with auto-detect:
+con = sqlite3.connect('termlib.sqlite')
-con.execute("""
+
-    CREATE TABLE table_name AS 
+# Simple CSVs work with csv.reader:
-    SELECT * FROM read_csv_auto('file.csv', header=true, all_varchar=true)
+with open('file.csv', 'r', encoding='utf-8') as f:
-""")
+    rows = list(csv.reader(f))
+headers = rows[0]
+data = rows[1:]
+col_defs = ', '.join(f'"{h}" TEXT' for h in headers)
+con.execute(f'CREATE TABLE "table_name" ({col_defs})')
+con.executemany(f'INSERT INTO "table_name" VALUES ({", ".join(["?"] * len(headers))})', data)
+con.commit()
 ```
 
 ### Pitfall: Multiline CSV fields
@@ -104,10 +110,10 @@ for i in range(0, len(data), batch_size):
     con.execute(f'INSERT INTO "{table}" VALUES {placeholders}', flat)
 ```
 
-### Pitfall: DuckDB CLI opens in-memory by default
+### SQLite CLI
-Running plain `duckdb` gives an empty database. Always pass the file path:
+Open the database file directly:
 ```
-duckdb path/to/termlib.duckdb
+sqlite3 path/to/termlib.sqlite
 ```
 
 ## Step 4: Create unified views
@@ -122,6 +128,6 @@ See `references/unified-view.sql` for the pattern. Key patterns:
 
 - **ODS reading**: Must traverse `odf.text.P` child elements, not direct text nodes
 - **Duplicate headers**: JSON/YAML dict silently overwrites duplicate keys — always deduplicate
-- **Multiline CSV + DuckDB**: `read_csv_auto` fails on CSVs with quoted newlines — use Python csv.reader
+- **Multiline CSV**: Use Python `csv.reader` for CSVs with embedded newlines
-- **DuckDB path**: Always explicit file path; `duckdb` alone is in-memory
+- **SQLite path**: Always pass the file path to `sqlite3`
 - **`execute_code` sandbox**: Does NOT share pip-installed packages — use `terminal` for Python scripts

terms-database/Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.14-slim
 
 WORKDIR /app
 
-RUN pip install --no-cache-dir flask duckdb gunicorn
+RUN pip install --no-cache-dir flask gunicorn
 
 COPY . .
 

terms-database/search.py

@@ -1,29 +1,31 @@
 #!/usr/bin/env -S uv run --script
 # /// script
-# dependencies = ["duckdb"]
+# dependencies = []
 # ///
-"""Full-text search over MPI term database. Queries unified_terms_flat via DuckDB LIKE.
+"""Full-text search over MPI term database (SQLite).
 
 Module usage:
-    from search import search_terms
+    from search import search
-    results = search_terms("空性")
+    results = search("空性")
-    results = search_terms("空性", limit=5, loc="心经", src="佛教术语")
+    results = search("空性", limit=5, loc="心经", src="佛教术语")
     # returns list of dicts: {zh, en, loc, source}
 
 CLI usage:
-    python search.py <query> [limit]
+    toolkit/terms-database/search.py <query> [limit]
-    python search.py 空性 loc:心经 src:公案
+    toolkit/terms-database/search.py 空性 loc:心经 src:公案
 """
 
 import sys
 import os
-import duckdb
+import sqlite3
 
-DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "termlib.duckdb")
+DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "termlib.sqlite")
 
 
 def _connect():
-    return duckdb.connect(DB, read_only=True)
+    con = sqlite3.connect(DB)
+    con.execute("PRAGMA journal_mode=WAL")
+    return con
 
 
 def _search_rows(con, query, loc=None, src=None, limit=None):
@@ -45,7 +47,7 @@ def _search_rows(con, query, loc=None, src=None, limit=None):
         where += " AND source = ?"
         params.append(src)
 
-    sql = f"SELECT zh, en, loc, source FROM unified_terms_flat WHERE {where}"
+    sql = f"SELECT zh, en, loc, source FROM terms WHERE {where}"
     if limit is not None:
         sql += " LIMIT ?"
         params.append(limit)
@@ -55,7 +57,7 @@ def _search_rows(con, query, loc=None, src=None, limit=None):
 
 def search(query, loc=None, src=None, limit=20):
     """Search the terms database. Returns list of {zh, en, loc, source} dicts.
-    
+
     query: str — space-separated search terms (AND logic)
     loc: str — filter by loc column (LIKE match)
     src: str — filter by source column (exact match)

terms-database/server.py

@@ -2,21 +2,27 @@
 import sys, os
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import duckdb
+import sqlite3
 from flask import Flask, request, jsonify, render_template
 import search as s
 
 app = Flask(__name__)
 
+def _connect():
+    con = sqlite3.connect(s.DB)
+    con.execute("PRAGMA journal_mode=WAL")
+    return con
+
+
 def do_search(q, loc, src, limit):
-    with duckdb.connect(s.DB, read_only=True) as con:
+    rows = s.search(q, loc=loc, src=src, limit=limit)
-        rows = s.search(con, q, loc, src, limit)
+    return [{'zh': r['zh'], 'en': r['en'], 'loc': r['loc'] or None, 'source': r['source']} for r in rows]
-    return [{'zh': r[0], 'en': r[1], 'loc': r[2] or None, 'source': r[3]} for r in rows]
+
 
 def do_sources():
-    with duckdb.connect(s.DB, read_only=True) as con:
+    with _connect() as con:
         rows = con.execute(
-            'SELECT source, COUNT(*) AS cnt FROM unified_terms_flat GROUP BY source ORDER BY cnt DESC'
+            'SELECT source, COUNT(*) AS cnt FROM terms GROUP BY source ORDER BY cnt DESC'
         ).fetchall()
     return [{'source': r[0], 'count': r[1]} for r in rows]