test add tokenizer

docs/.vitepress/vite-plugins/gtts.ts

@@ -40,7 +40,7 @@ async function gtts(
 ) {
   text = text.replace(/["]/g, " ");
 
-  const saveroot = "docs/.vitepress/cache/gtts";
+  const saveroot = ".gtts";
   fs.mkdirSync(saveroot, { recursive: true });
 
   const fileListJSON = path.join(saveroot, "gtts.json");

docs/.vitepress/vite-plugins/mp3-downloader.ts

@@ -2,19 +2,55 @@ import fs from "node:fs";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import path from "node:path";
 import type { Connect, Plugin, ViteDevServer } from "vite";
+import sqlite3 from "better-sqlite3";
+import { Tokenizer } from "./tokenizer";
 
 export function mp3DownloaderPlugin(): Plugin {
   return {
     name: "mp3-downloader",
-    configureServer(server: ViteDevServer) {
+    async configureServer(server: ViteDevServer) {
+      const db = sqlite3(".sentence-mp3.db");
+      db.exec(/* sql */ `
+        CREATE TABLE IF NOT EXISTS sentence (
+          id        INTEGER NOT NULL PRIMARY KEY,
+          ko        TEXT NOT NULL,
+          original  TEXT NOT NULL,
+          [url]     TEXT NOT NULL
+        );
+
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u_url ON sentence([url]);
+
+        CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
+          ko,
+          original  UNINDEXED,
+          [url]     UNINDEXED,
+          content='sentence',
+          content_rowid='id'
+        );
+
+        -- Triggers to keep the FTS index up to date.
+        CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
+          INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+        END;
+        CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
+          INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+        END;
+        CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
+          INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+          INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+        END;
+      `);
+
       // Vite dev server middlewares are connect-style: (req, res, next)
       server.middlewares.use(async (req, res, next) => {
         const reqUrl = req.url;
         if (!reqUrl || !reqUrl.startsWith("/uploads/")) return next();
-        if (!reqUrl.endsWith(".mp3")) return next();
+
+        const url = new URL(reqUrl, "http://localhost");
+        if (!url.pathname.endsWith(".mp3")) return next();
 
         try {
-          await mp3Downloader(req, res, next);
+          await mp3Downloader(db, await Tokenizer.init(), url, res, next);
         } catch (e) {
           console.error(e, reqUrl);
           next();
@@ -25,13 +61,24 @@ export function mp3DownloaderPlugin(): Plugin {
 }
 
 async function mp3Downloader(
-  req: Connect.IncomingMessage,
+  db: ReturnType<typeof sqlite3>,
+  tokenizer: Tokenizer,
+  url: URL,
   res: ServerResponse<IncomingMessage>,
   next: Connect.NextFunction,
 ) {
-  const url = new URL(`https://www.howtostudykorean.com/wp-content${req.url}`);
+  const downloadURL = new URL(
+    `https://www.howtostudykorean.com/wp-content${url.pathname}`,
+  );
 
-  const segs = req.url!.slice(1).split("/");
+  const text = url.searchParams.get("text");
+  if (text) {
+    db.prepare(
+      "INSERT OR IGNORE INTO sentence ([url], ko, original) VALUES (?, ?, ?)",
+    ).run(downloadURL.href, tokenizer.normalize(text), text);
+  }
+
+  const segs = url.pathname.slice(1).split("/");
   const filename = segs.pop()!;
   const saveroot = "docs/public";
   const saveto = path.join(saveroot, ...segs, filename);
@@ -42,7 +89,7 @@ async function mp3Downloader(
 
   fs.mkdirSync(path.join(saveroot, ...segs), { recursive: true });
 
-  const r = await fetch(url, { redirect: "follow" });
+  const r = await fetch(downloadURL, { redirect: "follow" });
   if (!r.ok) {
     console.error(r);
     return next();

docs/.vitepress/vite-plugins/tokenizer.ts

@@ -0,0 +1,22 @@
+import { MeCab } from "kuromoji-ko";
+
+export class Tokenizer {
+  static async init() {
+    return new Tokenizer(
+      await MeCab.create({
+        engine: "ko",
+        dictPath: "./node_modules/kuromoji-ko/dict",
+      }),
+    );
+  }
+
+  private constructor(public mecab: MeCab) {}
+
+  normalize(s: string) {
+    return this.mecab
+      .parse(s)
+      .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+      .map((t) => t.lemma)
+      .join(" ");
+  }
+}