sentence database builder script

sentence-db.js

@@ -0,0 +1,98 @@
+import sqlite3 from "better-sqlite3";
+import { MeCab } from "kuromoji-ko";
+import fs from "node:fs/promises";
+import yaml from "yaml";
+
+const mecab = await MeCab.create({
+  engine: "ko",
+  dictPath: "./node_modules/kuromoji-ko/dict",
+});
+
+const db = sqlite3(".sentence.db");
+db.exec(/* sql */ `
+  CREATE TABLE IF NOT EXISTS sentence (
+    ko        TEXT NOT NULL,
+    en        TEXT,
+    original  TEXT NOT NULL,
+    [url]     TEXT NOT NULL,
+    notes     TEXT,
+    filepath  TEXT NOT NULL
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u ON sentence(original, [url]);
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
+    ko,
+    en,
+    original  UNINDEXED,
+    [url]     UNINDEXED,
+    notes     UNINDEXED,
+    filepath  UNINDEXED,
+    content='sentence',
+    tokenize='porter'
+  );
+
+  -- Triggers to keep the FTS index up to date.
+  CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
+    INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en);
+  END;
+  CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
+    INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en);
+  END;
+  CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
+    INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en);
+    INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en);
+  END;
+`);
+
+const stmt = db.prepare(/* sql */ `
+  INSERT OR IGNORE INTO sentence (ko, original, en, [url], notes, filepath)
+  VALUES (:parsed, :ko, :en, :audio, :notes, :filepath)
+  `);
+const tx = db.transaction((ss) => {
+  for (const s of ss) {
+    if (s.notes === "Gemini") continue;
+    stmt.run({
+      ko: s.ko.replaceAll("**", ""),
+      parsed: mecab
+        .parse(s.ko)
+        .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+        .map((t) => t.lemma)
+        .join(" "),
+      en: (s.en || "").replaceAll("**", ""),
+      audio: s.audio || "",
+      notes: Array.isArray(s.notes) ? s.notes.join("\n") : s.notes || null,
+      filepath: s.filepath,
+    });
+  }
+});
+
+const SEPARATOR = "---\n";
+
+for await (const md of fs.glob("docs/**/*.md")) {
+  let text = await fs.readFile(md, "utf-8");
+  if (!text.startsWith(SEPARATOR)) continue;
+  try {
+    const ss = [];
+
+    const { vocabulary, sentence } =
+      yaml.parse(text.slice(SEPARATOR.length).split(SEPARATOR, 1)[0]) || {};
+    if (!vocabulary) continue;
+
+    for (const { examples = [] } of Object.values(vocabulary).flat(1)) {
+      ss.push(...examples.map((o) => ({ ...o, filepath: md })));
+    }
+
+    if (sentence) {
+      ss.push(
+        ...Object.values(sentence)
+          .flat(1)
+          .map((o) => ({ ...o, filepath: md })),
+      );
+    }
+
+    tx(ss);
+  } catch (e) {
+    console.error(e, md);
+  }
+}