test add tokenizer

docs/.vitepress/vite-plugins/tokenizer.ts

@@ -0,0 +1,22 @@
+import { MeCab } from "kuromoji-ko";
+
+export class Tokenizer {
+  static async init() {
+    return new Tokenizer(
+      await MeCab.create({
+        engine: "ko",
+        dictPath: "./node_modules/kuromoji-ko/dict",
+      }),
+    );
+  }
+
+  private constructor(public mecab: MeCab) {}
+
+  normalize(s: string) {
+    return this.mecab
+      .parse(s)
+      .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+      .map((t) => t.lemma)
+      .join(" ");
+  }
+}