test add tokenizer

2026-08-28 00:54:20 +07:00
1 parent 7e8c38a80f
commit 9dfae88805
@@ -146,3 +146,6 @@ vite.config.ts.timestamp-*
 docs/.vitepress/dist
 docs/.vitepress/cache
 docs/public/uploads/
+
+.gtts/
+*.db
@@ -11,3 +11,61 @@ docker run -it -p 5500:5500 synesthesiam/opentts:ko
 ```

 It's required to have Docker running.
+
+## Full Text Search
+
+This is done with [kuromoji-ko](https://www.npmjs.com/package/kuromoji-ko).
+
+```js
+import { MeCab } from "kuromoji-ko";
+
+const mecab = await MeCab.create({
+  engine: "ko",
+  dictPath: "./node_modules/kuromoji-ko/dict",
+});
+```
+
+### Tagset
+
+From <https://www.sketchengine.eu/hannanum-korean-tagset-simplified/>.
+
+| ORIGINAL          | SIMPLIFIED (in Sketch Engine) | DESCRIPTION    | Example               |
+| ----------------- | ----------------------------- | -------------- | --------------------- |
+| NOMINALS          |                               |                |                       |
+| NC                | **N**                         | noun           |                       |
+| NQ                | **NP**                        | proper noun    |                       |
+| NB                | **N**                         | noun           |                       |
+| NP                | **Pron**                      | pronoun        |                       |
+| NN                | **Num**                       | numeral        |                       |
+| PREDICATES        |                               |                |                       |
+| PV                | **V**                         | verb           |                       |
+| PA                | **Adj**                       | adjective      |                       |
+| PX                | **VAux**                      | auxiliary verb |                       |
+| MODIFIERS         |                               |                |                       |
+| MM                | **Det**                       | determiner     |                       |
+| MA                | **Adv**                       | adverb         |                       |
+| INTERJECTION      |                               |                |                       |
+| II                | **Interj**                    | interjection   |                       |
+| RELATIONAL SUFFIX |                               |                |                       |
+| JC                | **Suff**                      |                |                       |
+| JX                | **Suff**                      |                |                       |
+| JP                | **Suff**                      |                |                       |
+| ENDING            |                               |                |                       |
+| EP                | **Suff**                      |                |                       |
+| EC                | **Suff**                      |                |                       |
+| ET                | **Suff**                      |                |                       |
+| EF                | **Suff**                      |                |                       |
+| AFFIXES           |                               |                |                       |
+| XP                | **Pref**                      |                | 접두사                |
+| XS                | **Suff**                      |                |                       |
+| SYMBOLS           |                               |                |                       |
+| SP                | **Sym**                       |                | 쉼표                  |
+| SL                | **Sym**                       |                | 여는 따옴표 및 묶음표 |
+| SD                | **Sym**                       |                | 이음표                |
+| SU                | **Sym**                       |                | 단위 기호             |
+| SF                | **Sym**                       |                | 마침표                |
+| SR                | **Sym**                       |                | 닫는 따옴표 및 묶음표 |
+| SE                | **Sym**                       |                | 줄임표                |
+| SY                | **Sym**                       |                | 기타 기호             |
+| FOREIGN WORDS     |                               |                |                       |
+| F                 | **X**                         |                |                       |
@@ -1,5 +1,6 @@
 <template>
  <span>
+    <span ref="main"><slot /></span>
    <button type="button" @click="playAudio()">▶️</button>
    <audio ref="audio" style="display: none"></audio>
  </span>
@@ -17,6 +18,7 @@ defineExpose({
  playAudio,
 });

+const mainEl = useTemplateRef("main");
 const audioEl = useTemplateRef("audio");

 async function playAudio() {
@@ -31,6 +33,10 @@ async function playAudio() {
  let src = props.src;
  if (src.startsWith(uploadURL)) {
    src = src.replace(uploadURL, "/uploads");
+    if (mainEl.value) {
+      src += `?text=${encodeURIComponent(mainEl.value.innerText.trim())}`;
+    }
+
    audioEl.value.onerror = () => {
      if (!audioEl.value) return;
      console.log(audioEl.value.error, src);
@@ -40,7 +40,7 @@ async function gtts(
 ) {
  text = text.replace(/["]/g, " ");

-  const saveroot = "docs/.vitepress/cache/gtts";
+  const saveroot = ".gtts";
  fs.mkdirSync(saveroot, { recursive: true });

  const fileListJSON = path.join(saveroot, "gtts.json");
@@ -2,19 +2,55 @@ import fs from "node:fs";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import path from "node:path";
 import type { Connect, Plugin, ViteDevServer } from "vite";
+import sqlite3 from "better-sqlite3";
+import { Tokenizer } from "./tokenizer";

 export function mp3DownloaderPlugin(): Plugin {
  return {
    name: "mp3-downloader",
-    configureServer(server: ViteDevServer) {
+    async configureServer(server: ViteDevServer) {
+      const db = sqlite3(".sentence-mp3.db");
+      db.exec(/* sql */ `
+        CREATE TABLE IF NOT EXISTS sentence (
+          id        INTEGER NOT NULL PRIMARY KEY,
+          ko        TEXT NOT NULL,
+          original  TEXT NOT NULL,
+          [url]     TEXT NOT NULL
+        );
+
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u_url ON sentence([url]);
+
+        CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
+          ko,
+          original  UNINDEXED,
+          [url]     UNINDEXED,
+          content='sentence',
+          content_rowid='id'
+        );
+
+        -- Triggers to keep the FTS index up to date.
+        CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
+          INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+        END;
+        CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
+          INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+        END;
+        CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
+          INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+          INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+        END;
+      `);
+
      // Vite dev server middlewares are connect-style: (req, res, next)
      server.middlewares.use(async (req, res, next) => {
        const reqUrl = req.url;
        if (!reqUrl || !reqUrl.startsWith("/uploads/")) return next();
-        if (!reqUrl.endsWith(".mp3")) return next();
+
+        const url = new URL(reqUrl, "http://localhost");
+        if (!url.pathname.endsWith(".mp3")) return next();

        try {
-          await mp3Downloader(req, res, next);
+          await mp3Downloader(db, await Tokenizer.init(), url, res, next);
        } catch (e) {
          console.error(e, reqUrl);
          next();
@@ -25,13 +61,24 @@ export function mp3DownloaderPlugin(): Plugin {
 }

 async function mp3Downloader(
-  req: Connect.IncomingMessage,
+  db: ReturnType<typeof sqlite3>,
+  tokenizer: Tokenizer,
+  url: URL,
  res: ServerResponse<IncomingMessage>,
  next: Connect.NextFunction,
 ) {
-  const url = new URL(`https://www.howtostudykorean.com/wp-content${req.url}`);
+  const downloadURL = new URL(
+    `https://www.howtostudykorean.com/wp-content${url.pathname}`,
+  );

-  const segs = req.url!.slice(1).split("/");
+  const text = url.searchParams.get("text");
+  if (text) {
+    db.prepare(
+      "INSERT OR IGNORE INTO sentence ([url], ko, original) VALUES (?, ?, ?)",
+    ).run(downloadURL.href, tokenizer.normalize(text), text);
+  }
+
+  const segs = url.pathname.slice(1).split("/");
  const filename = segs.pop()!;
  const saveroot = "docs/public";
  const saveto = path.join(saveroot, ...segs, filename);
@@ -42,7 +89,7 @@ async function mp3Downloader(

  fs.mkdirSync(path.join(saveroot, ...segs), { recursive: true });

-  const r = await fetch(url, { redirect: "follow" });
+  const r = await fetch(downloadURL, { redirect: "follow" });
  if (!r.ok) {
    console.error(r);
    return next();
@@ -0,0 +1,22 @@
+import { MeCab } from "kuromoji-ko";
+
+export class Tokenizer {
+  static async init() {
+    return new Tokenizer(
+      await MeCab.create({
+        engine: "ko",
+        dictPath: "./node_modules/kuromoji-ko/dict",
+      }),
+    );
+  }
+
+  private constructor(public mecab: MeCab) {}
+
+  normalize(s: string) {
+    return this.mecab
+      .parse(s)
+      .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+      .map((t) => t.lemma)
+      .join(" ");
+  }
+}
@@ -197,7 +197,7 @@ vocabulary:
      examples:
        - ko: 저는 어렸을 때부터 **야구**를 좋아했어요
          en: I've liked **baseball** since I was young
-        - ko: "**야구선수**는 공을 세게 던졌어요"
+        - ko: "**야구**선수는 공을 세게 던졌어요"
          en: The **baseball** player threw the ball hard
        - ko: 저는 **야구**를 잘해요
          en: I play **baseball** well
@@ -1162,7 +1162,7 @@ Common Usages:
 Example:

 - 저는 어렸을 때부터 **야구**를 좋아했어요 = <span class="md spoiler">I’ve liked **baseball** since I was young</span>
- **야구선수**는 공을 세게 던졌어요 = <span class="md spoiler">The **baseball** player threw the ball hard</span>
+- **야구**선수는 공을 세게 던졌어요 = <span class="md spoiler">The **baseball** player threw the ball hard</span>
 - 저는 **야구**를 잘해요 = <span class="md spoiler">I play **baseball** well</span>
 - **야구**는 좋은 스포츠야 = <span class="md spoiler">**Baseball** is a good sport</span>
 - 우리는 **야구**를 1시에 할 거예요 = <span class="md spoiler">We will play **baseball** at 1:00</span>
@@ -27,11 +27,13 @@
  "dependencies": {
    "@supabase/supabase-js": "^2.110.5",
    "@tauri-apps/plugin-http": "~2.5.9",
+    "@types/better-sqlite3": "^9.6.0",
    "@types/markdown-it": "^14.2.0",
    "@types/node": "^24.13.3",
+    "better-sqlite3": "^13.0.3",
+    "kuromoji-ko": "^1.0.8",
    "markdown-it": "^14.3.0",
    "pdfjs-dist": "^5.7.284",
-    "shell-quote": "^1.10.0",
    "tauri-plugin-tts-api": "^0.1.11",
    "vite": "^8.1.5",
    "yaml": "^2.9.0",
@@ -14,21 +14,27 @@ importers:
      '@tauri-apps/plugin-http':
        specifier: ~2.5.9
        version: 2.5.9
+      '@types/better-sqlite3':
+        specifier: ^9.6.0
+        version: 9.6.0
      '@types/markdown-it':
        specifier: ^14.2.0
        version: 14.2.0
      '@types/node':
        specifier: ^24.13.3
        version: 24.13.3
+      better-sqlite3:
+        specifier: ^13.0.3
+        version: 13.0.3
+      kuromoji-ko:
+        specifier: ^1.0.8
+        version: 1.0.8
      markdown-it:
        specifier: ^14.3.0
        version: 14.3.0
      pdfjs-dist:
        specifier: ^5.7.284
        version: 5.7.284
-      shell-quote:
-        specifier: ^1.10.0
-        version: 1.10.0
      tauri-plugin-tts-api:
        specifier: ^0.1.11
        version: 0.1.11
@@ -710,6 +716,9 @@ packages:
  '@tybys/wasm-util@0.10.3':
    resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==}

+  '@types/better-sqlite3@9.6.0':
+    resolution: {integrity: sha512-ZEEwBSgMu7GYJOynoagg5X9JbxfL6dTJDsgViJIqh67jV44kyOr9RXfmFjLK5rzC4MWssP06t9hu/JwGDnUbCg==}
+
  '@types/estree@1.0.9':
    resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==}

@@ -843,6 +852,10 @@ packages:
  argparse@2.0.1:
    resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}

+  better-sqlite3@13.0.3:
+    resolution: {integrity: sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==}
+    engines: {node: '>=22'}
+
  birpc@2.9.0:
    resolution: {integrity: sha512-KrayHS5pBi69Xi9JmvoqrIgYGDkD6mcSe/i6YKi3w5kekCLzrX4+nawcXqrj2tIp50Kw/mT/s3p+GVK0A0sKxw==}

@@ -872,6 +885,9 @@ packages:
  devlop@1.1.0:
    resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==}

+  doublearray@0.0.2:
+    resolution: {integrity: sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw==}
+
  entities@4.5.0:
    resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
    engines: {node: '>=0.12'}
@@ -921,6 +937,9 @@ packages:
    resolution: {integrity: sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==}
    engines: {node: '>=20.0.0'}

+  kuromoji-ko@1.0.8:
+    resolution: {integrity: sha512-w2QP8rCMsE7yGJkQiY7LHhlhARspO42nSEroMtyTPzL7su+qfNQLVu5VQntx1mN9K8ta/5Qd+n63TiBHI57YTw==}
+
  lightningcss-android-arm64@1.32.0:
    resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==}
    engines: {node: '>= 12.0.0'}
@@ -1037,12 +1056,19 @@ packages:
    engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
    hasBin: true

+  node-addon-api@8.9.2:
+    resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==}
+    engines: {node: ^18 || ^20 || >= 21}
+
  oniguruma-parser@0.12.2:
    resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==}

  oniguruma-to-es@4.3.6:
    resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==}

+  pako@2.2.0:
+    resolution: {integrity: sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==}
+
  pdfjs-dist@5.7.284:
    resolution: {integrity: sha512-h4EdYQczmGhbOlqc3PPZwxevn7ApdWPbovAuWXOB/DjIyigSnwfy2oze7c6mRcSr9XgLp3eN3EeL4DyySTPMFw==}
    engines: {node: '>=22.13.0 || >=24'}
@@ -1091,10 +1117,6 @@ packages:
    engines: {node: '>=18.0.0', npm: '>=8.0.0'}
    hasBin: true

-  shell-quote@1.10.0:
-    resolution: {integrity: sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==}
-    engines: {node: '>= 0.4'}
-
  shiki@3.23.0:
    resolution: {integrity: sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA==}

@@ -1702,6 +1724,10 @@ snapshots:
      tslib: 2.8.1
    optional: true

+  '@types/better-sqlite3@9.6.0':
+    dependencies:
+      '@types/node': 24.13.3
+
  '@types/estree@1.0.9': {}

  '@types/hast@3.0.4':
@@ -1827,6 +1853,10 @@ snapshots:

  argparse@2.0.1: {}

+  better-sqlite3@13.0.3:
+    dependencies:
+      node-addon-api: 8.9.2
+
  birpc@2.9.0: {}

  ccount@2.0.1: {}
@@ -1847,6 +1877,8 @@ snapshots:
    dependencies:
      dequal: 2.0.3

+  doublearray@0.0.2: {}
+
  entities@4.5.0: {}

  entities@7.0.1: {}
@@ -1917,6 +1949,11 @@ snapshots:

  iceberg-js@0.8.1: {}

+  kuromoji-ko@1.0.8:
+    dependencies:
+      doublearray: 0.0.2
+      pako: 2.2.0
+
  lightningcss-android-arm64@1.32.0:
    optional: true

@@ -2020,6 +2057,8 @@ snapshots:

  nanoid@3.3.15: {}

+  node-addon-api@8.9.2: {}
+
  oniguruma-parser@0.12.2: {}

  oniguruma-to-es@4.3.6:
@@ -2028,6 +2067,8 @@ snapshots:
      regex: 6.1.0
      regex-recursion: 6.0.2

+  pako@2.2.0: {}
+
  pdfjs-dist@5.7.284:
    optionalDependencies:
      '@napi-rs/canvas': 0.1.100
@@ -2116,8 +2157,6 @@ snapshots:
      '@rollup/rollup-win32-x64-msvc': 4.62.2
      fsevents: 2.3.3

-  shell-quote@1.10.0: {}
-
  shiki@3.23.0:
    dependencies:
      '@shikijs/core': 3.23.0
@@ -1,2 +1,3 @@
 onlyBuiltDependencies:
+  - better-sqlite3
  - esbuild
@@ -0,0 +1,35 @@
+// testCases.map(c=>mecab.parse(c.sentence).filter(t=>t.pos.some(p=>/^[NVM]/.test(p))).map(t=>t.lemma));
+export const testCases = [
+  {
+    sentence: "학교에서 공부했어요",
+    expected: ["학교", "공부"],
+  },
+  {
+    sentence: "맛있는 김치찌개를 먹었어요",
+    expected: ["맛있다", "김치찌개", "먹다"],
+  },
+  {
+    sentence: "사랑하는 가족에게 선물을 줬어요",
+    expected: ["사랑", "가족", "선물", "주다"],
+  },
+  {
+    sentence: "빨리 달려가자",
+    expected: ["빨리", "달려가다"],
+  },
+  {
+    sentence: "예쁜 꽃이 피었어요",
+    expected: ["예쁘다", "꽃", "피다"],
+  },
+  {
+    sentence: "그는 학교에 갔다",
+    expected: ["그", "학교", "가다"],
+  },
+  {
+    sentence: "열심히 공부하는 학생입니다",
+    expected: ["열심히", "공부", "학생", "이다"],
+  },
+  {
+    sentence: "어제 영화를 봤어요",
+    expected: ["어제", "영화", "보다"],
+  },
+];