test add tokenizer

.gitignore

@@ -146,3 +146,6 @@ vite.config.ts.timestamp-*
 docs/.vitepress/dist
 docs/.vitepress/cache
 docs/public/uploads/
+
+.gtts/
+*.db

README.md

@@ -11,3 +11,61 @@ docker run -it -p 5500:5500 synesthesiam/opentts:ko
 ```
 
 It's required to have Docker running.
+
+## Full Text Search
+
+This is done with [kuromoji-ko](https://www.npmjs.com/package/kuromoji-ko).
+
+```js
+import { MeCab } from "kuromoji-ko";
+
+const mecab = await MeCab.create({
+  engine: "ko",
+  dictPath: "./node_modules/kuromoji-ko/dict",
+});
+```
+
+### Tagset
+
+From <https://www.sketchengine.eu/hannanum-korean-tagset-simplified/>.
+
+| ORIGINAL          | SIMPLIFIED (in Sketch Engine) | DESCRIPTION    | Example               |
+| ----------------- | ----------------------------- | -------------- | --------------------- |
+| NOMINALS          |                               |                |                       |
+| NC                | **N**                         | noun           |                       |
+| NQ                | **NP**                        | proper noun    |                       |
+| NB                | **N**                         | noun           |                       |
+| NP                | **Pron**                      | pronoun        |                       |
+| NN                | **Num**                       | numeral        |                       |
+| PREDICATES        |                               |                |                       |
+| PV                | **V**                         | verb           |                       |
+| PA                | **Adj**                       | adjective      |                       |
+| PX                | **VAux**                      | auxiliary verb |                       |
+| MODIFIERS         |                               |                |                       |
+| MM                | **Det**                       | determiner     |                       |
+| MA                | **Adv**                       | adverb         |                       |
+| INTERJECTION      |                               |                |                       |
+| II                | **Interj**                    | interjection   |                       |
+| RELATIONAL SUFFIX |                               |                |                       |
+| JC                | **Suff**                      |                |                       |
+| JX                | **Suff**                      |                |                       |
+| JP                | **Suff**                      |                |                       |
+| ENDING            |                               |                |                       |
+| EP                | **Suff**                      |                |                       |
+| EC                | **Suff**                      |                |                       |
+| ET                | **Suff**                      |                |                       |
+| EF                | **Suff**                      |                |                       |
+| AFFIXES           |                               |                |                       |
+| XP                | **Pref**                      |                | 접두사                |
+| XS                | **Suff**                      |                |                       |
+| SYMBOLS           |                               |                |                       |
+| SP                | **Sym**                       |                | 쉼표                  |
+| SL                | **Sym**                       |                | 여는 따옴표 및 묶음표 |
+| SD                | **Sym**                       |                | 이음표                |
+| SU                | **Sym**                       |                | 단위 기호             |
+| SF                | **Sym**                       |                | 마침표                |
+| SR                | **Sym**                       |                | 닫는 따옴표 및 묶음표 |
+| SE                | **Sym**                       |                | 줄임표                |
+| SY                | **Sym**                       |                | 기타 기호             |
+| FOREIGN WORDS     |                               |                |                       |
+| F                 | **X**                         |                |                       |

docs/.vitepress/theme/components/AudioButton.vue

@@ -1,5 +1,6 @@
 <template>
   <span>
+    <span ref="main"><slot /></span>
     <button type="button" @click="playAudio()">▶️</button>
     <audio ref="audio" style="display: none"></audio>
   </span>
@@ -17,6 +18,7 @@ defineExpose({
   playAudio,
 });
 
+const mainEl = useTemplateRef("main");
 const audioEl = useTemplateRef("audio");
 
 async function playAudio() {
@@ -31,6 +33,10 @@ async function playAudio() {
   let src = props.src;
   if (src.startsWith(uploadURL)) {
     src = src.replace(uploadURL, "/uploads");
+    if (mainEl.value) {
+      src += `?text=${encodeURIComponent(mainEl.value.innerText.trim())}`;
+    }
+
     audioEl.value.onerror = () => {
       if (!audioEl.value) return;
       console.log(audioEl.value.error, src);

docs/.vitepress/vite-plugins/gtts.ts

@@ -40,7 +40,7 @@ async function gtts(
 ) {
   text = text.replace(/["]/g, " ");
 
-  const saveroot = "docs/.vitepress/cache/gtts";
+  const saveroot = ".gtts";
   fs.mkdirSync(saveroot, { recursive: true });
 
   const fileListJSON = path.join(saveroot, "gtts.json");

docs/.vitepress/vite-plugins/mp3-downloader.ts

@@ -2,19 +2,55 @@ import fs from "node:fs";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import path from "node:path";
 import type { Connect, Plugin, ViteDevServer } from "vite";
+import sqlite3 from "better-sqlite3";
+import { Tokenizer } from "./tokenizer";
 
 export function mp3DownloaderPlugin(): Plugin {
   return {
     name: "mp3-downloader",
-    configureServer(server: ViteDevServer) {
+    async configureServer(server: ViteDevServer) {
+      const db = sqlite3(".sentence-mp3.db");
+      db.exec(/* sql */ `
+        CREATE TABLE IF NOT EXISTS sentence (
+          id        INTEGER NOT NULL PRIMARY KEY,
+          ko        TEXT NOT NULL,
+          original  TEXT NOT NULL,
+          [url]     TEXT NOT NULL
+        );
+
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u_url ON sentence([url]);
+
+        CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
+          ko,
+          original  UNINDEXED,
+          [url]     UNINDEXED,
+          content='sentence',
+          content_rowid='id'
+        );
+
+        -- Triggers to keep the FTS index up to date.
+        CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
+          INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+        END;
+        CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
+          INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+        END;
+        CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
+          INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+          INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+        END;
+      `);
+
       // Vite dev server middlewares are connect-style: (req, res, next)
       server.middlewares.use(async (req, res, next) => {
         const reqUrl = req.url;
         if (!reqUrl || !reqUrl.startsWith("/uploads/")) return next();
-        if (!reqUrl.endsWith(".mp3")) return next();
+
+        const url = new URL(reqUrl, "http://localhost");
+        if (!url.pathname.endsWith(".mp3")) return next();
 
         try {
-          await mp3Downloader(req, res, next);
+          await mp3Downloader(db, await Tokenizer.init(), url, res, next);
         } catch (e) {
           console.error(e, reqUrl);
           next();
@@ -25,13 +61,24 @@ export function mp3DownloaderPlugin(): Plugin {
 }
 
 async function mp3Downloader(
-  req: Connect.IncomingMessage,
+  db: ReturnType<typeof sqlite3>,
+  tokenizer: Tokenizer,
+  url: URL,
   res: ServerResponse<IncomingMessage>,
   next: Connect.NextFunction,
 ) {
-  const url = new URL(`https://www.howtostudykorean.com/wp-content${req.url}`);
+  const downloadURL = new URL(
+    `https://www.howtostudykorean.com/wp-content${url.pathname}`,
+  );
 
-  const segs = req.url!.slice(1).split("/");
+  const text = url.searchParams.get("text");
+  if (text) {
+    db.prepare(
+      "INSERT OR IGNORE INTO sentence ([url], ko, original) VALUES (?, ?, ?)",
+    ).run(downloadURL.href, tokenizer.normalize(text), text);
+  }
+
+  const segs = url.pathname.slice(1).split("/");
   const filename = segs.pop()!;
   const saveroot = "docs/public";
   const saveto = path.join(saveroot, ...segs, filename);
@@ -42,7 +89,7 @@ async function mp3Downloader(
 
   fs.mkdirSync(path.join(saveroot, ...segs), { recursive: true });
 
-  const r = await fetch(url, { redirect: "follow" });
+  const r = await fetch(downloadURL, { redirect: "follow" });
   if (!r.ok) {
     console.error(r);
     return next();

docs/.vitepress/vite-plugins/tokenizer.ts

@@ -0,0 +1,22 @@
+import { MeCab } from "kuromoji-ko";
+
+export class Tokenizer {
+  static async init() {
+    return new Tokenizer(
+      await MeCab.create({
+        engine: "ko",
+        dictPath: "./node_modules/kuromoji-ko/dict",
+      }),
+    );
+  }
+
+  private constructor(public mecab: MeCab) {}
+
+  normalize(s: string) {
+    return this.mecab
+      .parse(s)
+      .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+      .map((t) => t.lemma)
+      .join(" ");
+  }
+}

docs/howtostudykorean/unit-1/09-conjugate-이다.md

@@ -197,7 +197,7 @@ vocabulary:
       examples:
         - ko: 저는 어렸을 때부터 **야구**를 좋아했어요
           en: I've liked **baseball** since I was young
-        - ko: "**야구선수**는 공을 세게 던졌어요"
+        - ko: "**야구**선수는 공을 세게 던졌어요"
           en: The **baseball** player threw the ball hard
         - ko: 저는 **야구**를 잘해요
           en: I play **baseball** well
@@ -1162,7 +1162,7 @@ Common Usages:
 Example:
 
 - 저는 어렸을 때부터 **야구**를 좋아했어요 = <span class="md spoiler">I’ve liked **baseball** since I was young</span>
-- **야구선수**는 공을 세게 던졌어요 = <span class="md spoiler">The **baseball** player threw the ball hard</span>
+- **야구**선수는 공을 세게 던졌어요 = <span class="md spoiler">The **baseball** player threw the ball hard</span>
 - 저는 **야구**를 잘해요 = <span class="md spoiler">I play **baseball** well</span>
 - **야구**는 좋은 스포츠야 = <span class="md spoiler">**Baseball** is a good sport</span>
 - 우리는 **야구**를 1시에 할 거예요 = <span class="md spoiler">We will play **baseball** at 1:00</span>

package.json

@@ -27,11 +27,13 @@
   "dependencies": {
     "@supabase/supabase-js": "^2.110.5",
     "@tauri-apps/plugin-http": "~2.5.9",
+    "@types/better-sqlite3": "^9.6.0",
     "@types/markdown-it": "^14.2.0",
     "@types/node": "^24.13.3",
+    "better-sqlite3": "^13.0.3",
+    "kuromoji-ko": "^1.0.8",
     "markdown-it": "^14.3.0",
     "pdfjs-dist": "^5.7.284",
-    "shell-quote": "^1.10.0",
     "tauri-plugin-tts-api": "^0.1.11",
     "vite": "^8.1.5",
     "yaml": "^2.9.0",

pnpm-lock.yaml

@@ -14,21 +14,27 @@ importers:
       '@tauri-apps/plugin-http':
         specifier: ~2.5.9
         version: 2.5.9
+      '@types/better-sqlite3':
+        specifier: ^9.6.0
+        version: 9.6.0
       '@types/markdown-it':
         specifier: ^14.2.0
         version: 14.2.0
       '@types/node':
         specifier: ^24.13.3
         version: 24.13.3
+      better-sqlite3:
+        specifier: ^13.0.3
+        version: 13.0.3
+      kuromoji-ko:
+        specifier: ^1.0.8
+        version: 1.0.8
       markdown-it:
         specifier: ^14.3.0
         version: 14.3.0
       pdfjs-dist:
         specifier: ^5.7.284
         version: 5.7.284
-      shell-quote:
-        specifier: ^1.10.0
-        version: 1.10.0
       tauri-plugin-tts-api:
         specifier: ^0.1.11
         version: 0.1.11
@@ -710,6 +716,9 @@ packages:
   '@tybys/wasm-util@0.10.3':
     resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==}
 
+  '@types/better-sqlite3@9.6.0':
+    resolution: {integrity: sha512-ZEEwBSgMu7GYJOynoagg5X9JbxfL6dTJDsgViJIqh67jV44kyOr9RXfmFjLK5rzC4MWssP06t9hu/JwGDnUbCg==}
+
   '@types/estree@1.0.9':
     resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==}
 
@@ -843,6 +852,10 @@ packages:
   argparse@2.0.1:
     resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
 
+  better-sqlite3@13.0.3:
+    resolution: {integrity: sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==}
+    engines: {node: '>=22'}
+
   birpc@2.9.0:
     resolution: {integrity: sha512-KrayHS5pBi69Xi9JmvoqrIgYGDkD6mcSe/i6YKi3w5kekCLzrX4+nawcXqrj2tIp50Kw/mT/s3p+GVK0A0sKxw==}
 
@@ -872,6 +885,9 @@ packages:
   devlop@1.1.0:
     resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==}
 
+  doublearray@0.0.2:
+    resolution: {integrity: sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw==}
+
   entities@4.5.0:
     resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
     engines: {node: '>=0.12'}
@@ -921,6 +937,9 @@ packages:
     resolution: {integrity: sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==}
     engines: {node: '>=20.0.0'}
 
+  kuromoji-ko@1.0.8:
+    resolution: {integrity: sha512-w2QP8rCMsE7yGJkQiY7LHhlhARspO42nSEroMtyTPzL7su+qfNQLVu5VQntx1mN9K8ta/5Qd+n63TiBHI57YTw==}
+
   lightningcss-android-arm64@1.32.0:
     resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==}
     engines: {node: '>= 12.0.0'}
@@ -1037,12 +1056,19 @@ packages:
     engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
     hasBin: true
 
+  node-addon-api@8.9.2:
+    resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==}
+    engines: {node: ^18 || ^20 || >= 21}
+
   oniguruma-parser@0.12.2:
     resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==}
 
   oniguruma-to-es@4.3.6:
     resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==}
 
+  pako@2.2.0:
+    resolution: {integrity: sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==}
+
   pdfjs-dist@5.7.284:
     resolution: {integrity: sha512-h4EdYQczmGhbOlqc3PPZwxevn7ApdWPbovAuWXOB/DjIyigSnwfy2oze7c6mRcSr9XgLp3eN3EeL4DyySTPMFw==}
     engines: {node: '>=22.13.0 || >=24'}
@@ -1091,10 +1117,6 @@ packages:
     engines: {node: '>=18.0.0', npm: '>=8.0.0'}
     hasBin: true
 
-  shell-quote@1.10.0:
-    resolution: {integrity: sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==}
-    engines: {node: '>= 0.4'}
-
   shiki@3.23.0:
     resolution: {integrity: sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA==}
 
@@ -1702,6 +1724,10 @@ snapshots:
       tslib: 2.8.1
     optional: true
 
+  '@types/better-sqlite3@9.6.0':
+    dependencies:
+      '@types/node': 24.13.3
+
   '@types/estree@1.0.9': {}
 
   '@types/hast@3.0.4':
@@ -1827,6 +1853,10 @@ snapshots:
 
   argparse@2.0.1: {}
 
+  better-sqlite3@13.0.3:
+    dependencies:
+      node-addon-api: 8.9.2
+
   birpc@2.9.0: {}
 
   ccount@2.0.1: {}
@@ -1847,6 +1877,8 @@ snapshots:
     dependencies:
       dequal: 2.0.3
 
+  doublearray@0.0.2: {}
+
   entities@4.5.0: {}
 
   entities@7.0.1: {}
@@ -1917,6 +1949,11 @@ snapshots:
 
   iceberg-js@0.8.1: {}
 
+  kuromoji-ko@1.0.8:
+    dependencies:
+      doublearray: 0.0.2
+      pako: 2.2.0
+
   lightningcss-android-arm64@1.32.0:
     optional: true
 
@@ -2020,6 +2057,8 @@ snapshots:
 
   nanoid@3.3.15: {}
 
+  node-addon-api@8.9.2: {}
+
   oniguruma-parser@0.12.2: {}
 
   oniguruma-to-es@4.3.6:
@@ -2028,6 +2067,8 @@ snapshots:
       regex: 6.1.0
       regex-recursion: 6.0.2
 
+  pako@2.2.0: {}
+
   pdfjs-dist@5.7.284:
     optionalDependencies:
       '@napi-rs/canvas': 0.1.100
@@ -2116,8 +2157,6 @@ snapshots:
       '@rollup/rollup-win32-x64-msvc': 4.62.2
       fsevents: 2.3.3
 
-  shell-quote@1.10.0: {}
-
   shiki@3.23.0:
     dependencies:
       '@shikijs/core': 3.23.0

pnpm-workspace.yaml

@@ -1,2 +1,3 @@
 onlyBuiltDependencies:
+  - better-sqlite3
   - esbuild

test-cases.js

@@ -0,0 +1,35 @@
+// testCases.map(c=>mecab.parse(c.sentence).filter(t=>t.pos.some(p=>/^[NVM]/.test(p))).map(t=>t.lemma));
+export const testCases = [
+  {
+    sentence: "학교에서 공부했어요",
+    expected: ["학교", "공부"],
+  },
+  {
+    sentence: "맛있는 김치찌개를 먹었어요",
+    expected: ["맛있다", "김치찌개", "먹다"],
+  },
+  {
+    sentence: "사랑하는 가족에게 선물을 줬어요",
+    expected: ["사랑", "가족", "선물", "주다"],
+  },
+  {
+    sentence: "빨리 달려가자",
+    expected: ["빨리", "달려가다"],
+  },
+  {
+    sentence: "예쁜 꽃이 피었어요",
+    expected: ["예쁘다", "꽃", "피다"],
+  },
+  {
+    sentence: "그는 학교에 갔다",
+    expected: ["그", "학교", "가다"],
+  },
+  {
+    sentence: "열심히 공부하는 학생입니다",
+    expected: ["열심히", "공부", "학생", "이다"],
+  },
+  {
+    sentence: "어제 영화를 봤어요",
+    expected: ["어제", "영화", "보다"],
+  },
+];