23 lines
456 B
TypeScript
23 lines
456 B
TypeScript
import { MeCab } from "kuromoji-ko";
|
|
|
|
export class Tokenizer {
|
|
static async init() {
|
|
return new Tokenizer(
|
|
await MeCab.create({
|
|
engine: "ko",
|
|
dictPath: "./node_modules/kuromoji-ko/dict",
|
|
}),
|
|
);
|
|
}
|
|
|
|
private constructor(public mecab: MeCab) {}
|
|
|
|
normalize(s: string) {
|
|
return this.mecab
|
|
.parse(s)
|
|
.filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
|
|
.map((t) => t.lemma)
|
|
.join(" ");
|
|
}
|
|
}
|