diff --git a/.gitignore b/.gitignore
index 9000ab4..49ca683 100644
--- a/.gitignore
+++ b/.gitignore
@@ -146,3 +146,6 @@ vite.config.ts.timestamp-*
docs/.vitepress/dist
docs/.vitepress/cache
docs/public/uploads/
+
+.gtts/
+*.db
diff --git a/README.md b/README.md
index 27cc5d5..5c87cc7 100644
--- a/README.md
+++ b/README.md
@@ -11,3 +11,61 @@ docker run -it -p 5500:5500 synesthesiam/opentts:ko
```
It's required to have Docker running.
+
+## Full Text Search
+
+This is done with [kuromoji-ko](https://www.npmjs.com/package/kuromoji-ko).
+
+```js
+import { MeCab } from "kuromoji-ko";
+
+const mecab = await MeCab.create({
+ engine: "ko",
+ dictPath: "./node_modules/kuromoji-ko/dict",
+});
+```
+
+### Tagset
+
+From .
+
+| ORIGINAL | SIMPLIFIED (in Sketch Engine) | DESCRIPTION | Example |
+| ----------------- | ----------------------------- | -------------- | --------------------- |
+| NOMINALS | | | |
+| NC | **N** | noun | |
+| NQ | **NP** | proper noun | |
+| NB | **N** | noun | |
+| NP | **Pron** | pronoun | |
+| NN | **Num** | numeral | |
+| PREDICATES | | | |
+| PV | **V** | verb | |
+| PA | **Adj** | adjective | |
+| PX | **VAux** | auxiliary verb | |
+| MODIFIERS | | | |
+| MM | **Det** | determiner | |
+| MA | **Adv** | adverb | |
+| INTERJECTION | | | |
+| II | **Interj** | interjection | |
+| RELATIONAL SUFFIX | | | |
+| JC | **Suff** | | |
+| JX | **Suff** | | |
+| JP | **Suff** | | |
+| ENDING | | | |
+| EP | **Suff** | | |
+| EC | **Suff** | | |
+| ET | **Suff** | | |
+| EF | **Suff** | | |
+| AFFIXES | | | |
+| XP | **Pref** | | 접두사 |
+| XS | **Suff** | | |
+| SYMBOLS | | | |
+| SP | **Sym** | | 쉼표 |
+| SL | **Sym** | | 여는 따옴표 및 묶음표 |
+| SD | **Sym** | | 이음표 |
+| SU | **Sym** | | 단위 기호 |
+| SF | **Sym** | | 마침표 |
+| SR | **Sym** | | 닫는 따옴표 및 묶음표 |
+| SE | **Sym** | | 줄임표 |
+| SY | **Sym** | | 기타 기호 |
+| FOREIGN WORDS | | | |
+| F | **X** | | |
diff --git a/docs/.vitepress/theme/components/AudioButton.vue b/docs/.vitepress/theme/components/AudioButton.vue
index 43ea694..6c7f3e5 100644
--- a/docs/.vitepress/theme/components/AudioButton.vue
+++ b/docs/.vitepress/theme/components/AudioButton.vue
@@ -1,5 +1,6 @@
+
@@ -17,6 +18,7 @@ defineExpose({
playAudio,
});
+const mainEl = useTemplateRef("main");
const audioEl = useTemplateRef("audio");
async function playAudio() {
@@ -31,6 +33,10 @@ async function playAudio() {
let src = props.src;
if (src.startsWith(uploadURL)) {
src = src.replace(uploadURL, "/uploads");
+ if (mainEl.value) {
+ src += `?text=${encodeURIComponent(mainEl.value.innerText.trim())}`;
+ }
+
audioEl.value.onerror = () => {
if (!audioEl.value) return;
console.log(audioEl.value.error, src);
diff --git a/docs/.vitepress/vite-plugins/gtts.ts b/docs/.vitepress/vite-plugins/gtts.ts
index 3183606..58339e6 100644
--- a/docs/.vitepress/vite-plugins/gtts.ts
+++ b/docs/.vitepress/vite-plugins/gtts.ts
@@ -40,7 +40,7 @@ async function gtts(
) {
text = text.replace(/["]/g, " ");
- const saveroot = "docs/.vitepress/cache/gtts";
+ const saveroot = ".gtts";
fs.mkdirSync(saveroot, { recursive: true });
const fileListJSON = path.join(saveroot, "gtts.json");
diff --git a/docs/.vitepress/vite-plugins/mp3-downloader.ts b/docs/.vitepress/vite-plugins/mp3-downloader.ts
index b7f1491..08ab23d 100644
--- a/docs/.vitepress/vite-plugins/mp3-downloader.ts
+++ b/docs/.vitepress/vite-plugins/mp3-downloader.ts
@@ -2,19 +2,55 @@ import fs from "node:fs";
import type { IncomingMessage, ServerResponse } from "node:http";
import path from "node:path";
import type { Connect, Plugin, ViteDevServer } from "vite";
+import sqlite3 from "better-sqlite3";
+import { Tokenizer } from "./tokenizer";
export function mp3DownloaderPlugin(): Plugin {
return {
name: "mp3-downloader",
- configureServer(server: ViteDevServer) {
+ async configureServer(server: ViteDevServer) {
+ const db = sqlite3(".sentence-mp3.db");
+ db.exec(/* sql */ `
+ CREATE TABLE IF NOT EXISTS sentence (
+ id INTEGER NOT NULL PRIMARY KEY,
+ ko TEXT NOT NULL,
+ original TEXT NOT NULL,
+ [url] TEXT NOT NULL
+ );
+
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u_url ON sentence([url]);
+
+ CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
+ ko,
+ original UNINDEXED,
+ [url] UNINDEXED,
+ content='sentence',
+ content_rowid='id'
+ );
+
+ -- Triggers to keep the FTS index up to date.
+ CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
+ INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+ END;
+ CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
+ INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+ END;
+ CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
+ INSERT INTO sentence_fts(fts_idx, rowid, ko) VALUES('delete', old.id, old.ko);
+ INSERT INTO sentence_fts(rowid, ko) VALUES (new.id, new.ko);
+ END;
+ `);
+
// Vite dev server middlewares are connect-style: (req, res, next)
server.middlewares.use(async (req, res, next) => {
const reqUrl = req.url;
if (!reqUrl || !reqUrl.startsWith("/uploads/")) return next();
- if (!reqUrl.endsWith(".mp3")) return next();
+
+ const url = new URL(reqUrl, "http://localhost");
+ if (!url.pathname.endsWith(".mp3")) return next();
try {
- await mp3Downloader(req, res, next);
+ await mp3Downloader(db, await Tokenizer.init(), url, res, next);
} catch (e) {
console.error(e, reqUrl);
next();
@@ -25,13 +61,24 @@ export function mp3DownloaderPlugin(): Plugin {
}
async function mp3Downloader(
- req: Connect.IncomingMessage,
+ db: ReturnType,
+ tokenizer: Tokenizer,
+ url: URL,
res: ServerResponse,
next: Connect.NextFunction,
) {
- const url = new URL(`https://www.howtostudykorean.com/wp-content${req.url}`);
+ const downloadURL = new URL(
+ `https://www.howtostudykorean.com/wp-content${url.pathname}`,
+ );
- const segs = req.url!.slice(1).split("/");
+ const text = url.searchParams.get("text");
+ if (text) {
+ db.prepare(
+ "INSERT OR IGNORE INTO sentence ([url], ko, original) VALUES (?, ?, ?)",
+ ).run(downloadURL.href, tokenizer.normalize(text), text);
+ }
+
+ const segs = url.pathname.slice(1).split("/");
const filename = segs.pop()!;
const saveroot = "docs/public";
const saveto = path.join(saveroot, ...segs, filename);
@@ -42,7 +89,7 @@ async function mp3Downloader(
fs.mkdirSync(path.join(saveroot, ...segs), { recursive: true });
- const r = await fetch(url, { redirect: "follow" });
+ const r = await fetch(downloadURL, { redirect: "follow" });
if (!r.ok) {
console.error(r);
return next();
diff --git a/docs/.vitepress/vite-plugins/tokenizer.ts b/docs/.vitepress/vite-plugins/tokenizer.ts
new file mode 100644
index 0000000..8d8be42
--- /dev/null
+++ b/docs/.vitepress/vite-plugins/tokenizer.ts
@@ -0,0 +1,22 @@
+import { MeCab } from "kuromoji-ko";
+
+export class Tokenizer {
+ static async init() {
+ return new Tokenizer(
+ await MeCab.create({
+ engine: "ko",
+ dictPath: "./node_modules/kuromoji-ko/dict",
+ }),
+ );
+ }
+
+ private constructor(public mecab: MeCab) {}
+
+ normalize(s: string) {
+ return this.mecab
+ .parse(s)
+ .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+ .map((t) => t.lemma)
+ .join(" ");
+ }
+}
diff --git a/docs/howtostudykorean/unit-1/09-conjugate-이다.md b/docs/howtostudykorean/unit-1/09-conjugate-이다.md
index 7912907..9879308 100644
--- a/docs/howtostudykorean/unit-1/09-conjugate-이다.md
+++ b/docs/howtostudykorean/unit-1/09-conjugate-이다.md
@@ -197,7 +197,7 @@ vocabulary:
examples:
- ko: 저는 어렸을 때부터 **야구**를 좋아했어요
en: I've liked **baseball** since I was young
- - ko: "**야구선수**는 공을 세게 던졌어요"
+ - ko: "**야구**선수는 공을 세게 던졌어요"
en: The **baseball** player threw the ball hard
- ko: 저는 **야구**를 잘해요
en: I play **baseball** well
@@ -1162,7 +1162,7 @@ Common Usages:
Example:
- 저는 어렸을 때부터 **야구**를 좋아했어요 = I’ve liked **baseball** since I was young
-- **야구선수**는 공을 세게 던졌어요 = The **baseball** player threw the ball hard
+- **야구**선수는 공을 세게 던졌어요 = The **baseball** player threw the ball hard
- 저는 **야구**를 잘해요 = I play **baseball** well
- **야구**는 좋은 스포츠야 = **Baseball** is a good sport
- 우리는 **야구**를 1시에 할 거예요 = We will play **baseball** at 1:00
diff --git a/package.json b/package.json
index d7a6c31..3d3c4fc 100644
--- a/package.json
+++ b/package.json
@@ -27,11 +27,13 @@
"dependencies": {
"@supabase/supabase-js": "^2.110.5",
"@tauri-apps/plugin-http": "~2.5.9",
+ "@types/better-sqlite3": "^9.6.0",
"@types/markdown-it": "^14.2.0",
"@types/node": "^24.13.3",
+ "better-sqlite3": "^13.0.3",
+ "kuromoji-ko": "^1.0.8",
"markdown-it": "^14.3.0",
"pdfjs-dist": "^5.7.284",
- "shell-quote": "^1.10.0",
"tauri-plugin-tts-api": "^0.1.11",
"vite": "^8.1.5",
"yaml": "^2.9.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a3911ca..14e9046 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -14,21 +14,27 @@ importers:
'@tauri-apps/plugin-http':
specifier: ~2.5.9
version: 2.5.9
+ '@types/better-sqlite3':
+ specifier: ^9.6.0
+ version: 9.6.0
'@types/markdown-it':
specifier: ^14.2.0
version: 14.2.0
'@types/node':
specifier: ^24.13.3
version: 24.13.3
+ better-sqlite3:
+ specifier: ^13.0.3
+ version: 13.0.3
+ kuromoji-ko:
+ specifier: ^1.0.8
+ version: 1.0.8
markdown-it:
specifier: ^14.3.0
version: 14.3.0
pdfjs-dist:
specifier: ^5.7.284
version: 5.7.284
- shell-quote:
- specifier: ^1.10.0
- version: 1.10.0
tauri-plugin-tts-api:
specifier: ^0.1.11
version: 0.1.11
@@ -710,6 +716,9 @@ packages:
'@tybys/wasm-util@0.10.3':
resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==}
+ '@types/better-sqlite3@9.6.0':
+ resolution: {integrity: sha512-ZEEwBSgMu7GYJOynoagg5X9JbxfL6dTJDsgViJIqh67jV44kyOr9RXfmFjLK5rzC4MWssP06t9hu/JwGDnUbCg==}
+
'@types/estree@1.0.9':
resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==}
@@ -843,6 +852,10 @@ packages:
argparse@2.0.1:
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
+ better-sqlite3@13.0.3:
+ resolution: {integrity: sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==}
+ engines: {node: '>=22'}
+
birpc@2.9.0:
resolution: {integrity: sha512-KrayHS5pBi69Xi9JmvoqrIgYGDkD6mcSe/i6YKi3w5kekCLzrX4+nawcXqrj2tIp50Kw/mT/s3p+GVK0A0sKxw==}
@@ -872,6 +885,9 @@ packages:
devlop@1.1.0:
resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==}
+ doublearray@0.0.2:
+ resolution: {integrity: sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw==}
+
entities@4.5.0:
resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
engines: {node: '>=0.12'}
@@ -921,6 +937,9 @@ packages:
resolution: {integrity: sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==}
engines: {node: '>=20.0.0'}
+ kuromoji-ko@1.0.8:
+ resolution: {integrity: sha512-w2QP8rCMsE7yGJkQiY7LHhlhARspO42nSEroMtyTPzL7su+qfNQLVu5VQntx1mN9K8ta/5Qd+n63TiBHI57YTw==}
+
lightningcss-android-arm64@1.32.0:
resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==}
engines: {node: '>= 12.0.0'}
@@ -1037,12 +1056,19 @@ packages:
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
hasBin: true
+ node-addon-api@8.9.2:
+ resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==}
+ engines: {node: ^18 || ^20 || >= 21}
+
oniguruma-parser@0.12.2:
resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==}
oniguruma-to-es@4.3.6:
resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==}
+ pako@2.2.0:
+ resolution: {integrity: sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==}
+
pdfjs-dist@5.7.284:
resolution: {integrity: sha512-h4EdYQczmGhbOlqc3PPZwxevn7ApdWPbovAuWXOB/DjIyigSnwfy2oze7c6mRcSr9XgLp3eN3EeL4DyySTPMFw==}
engines: {node: '>=22.13.0 || >=24'}
@@ -1091,10 +1117,6 @@ packages:
engines: {node: '>=18.0.0', npm: '>=8.0.0'}
hasBin: true
- shell-quote@1.10.0:
- resolution: {integrity: sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==}
- engines: {node: '>= 0.4'}
-
shiki@3.23.0:
resolution: {integrity: sha512-55Dj73uq9ZXL5zyeRPzHQsK7Nbyt6Y10k5s7OjuFZGMhpp4r/rsLBH0o/0fstIzX1Lep9VxefWljK/SKCzygIA==}
@@ -1702,6 +1724,10 @@ snapshots:
tslib: 2.8.1
optional: true
+ '@types/better-sqlite3@9.6.0':
+ dependencies:
+ '@types/node': 24.13.3
+
'@types/estree@1.0.9': {}
'@types/hast@3.0.4':
@@ -1827,6 +1853,10 @@ snapshots:
argparse@2.0.1: {}
+ better-sqlite3@13.0.3:
+ dependencies:
+ node-addon-api: 8.9.2
+
birpc@2.9.0: {}
ccount@2.0.1: {}
@@ -1847,6 +1877,8 @@ snapshots:
dependencies:
dequal: 2.0.3
+ doublearray@0.0.2: {}
+
entities@4.5.0: {}
entities@7.0.1: {}
@@ -1917,6 +1949,11 @@ snapshots:
iceberg-js@0.8.1: {}
+ kuromoji-ko@1.0.8:
+ dependencies:
+ doublearray: 0.0.2
+ pako: 2.2.0
+
lightningcss-android-arm64@1.32.0:
optional: true
@@ -2020,6 +2057,8 @@ snapshots:
nanoid@3.3.15: {}
+ node-addon-api@8.9.2: {}
+
oniguruma-parser@0.12.2: {}
oniguruma-to-es@4.3.6:
@@ -2028,6 +2067,8 @@ snapshots:
regex: 6.1.0
regex-recursion: 6.0.2
+ pako@2.2.0: {}
+
pdfjs-dist@5.7.284:
optionalDependencies:
'@napi-rs/canvas': 0.1.100
@@ -2116,8 +2157,6 @@ snapshots:
'@rollup/rollup-win32-x64-msvc': 4.62.2
fsevents: 2.3.3
- shell-quote@1.10.0: {}
-
shiki@3.23.0:
dependencies:
'@shikijs/core': 3.23.0
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index efc037a..f2721cf 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -1,2 +1,3 @@
onlyBuiltDependencies:
+ - better-sqlite3
- esbuild
diff --git a/test-cases.js b/test-cases.js
new file mode 100644
index 0000000..1ad3cd2
--- /dev/null
+++ b/test-cases.js
@@ -0,0 +1,35 @@
+// testCases.map(c=>mecab.parse(c.sentence).filter(t=>t.pos.some(p=>/^[NVM]/.test(p))).map(t=>t.lemma));
+export const testCases = [
+ {
+ sentence: "학교에서 공부했어요",
+ expected: ["학교", "공부"],
+ },
+ {
+ sentence: "맛있는 김치찌개를 먹었어요",
+ expected: ["맛있다", "김치찌개", "먹다"],
+ },
+ {
+ sentence: "사랑하는 가족에게 선물을 줬어요",
+ expected: ["사랑", "가족", "선물", "주다"],
+ },
+ {
+ sentence: "빨리 달려가자",
+ expected: ["빨리", "달려가다"],
+ },
+ {
+ sentence: "예쁜 꽃이 피었어요",
+ expected: ["예쁘다", "꽃", "피다"],
+ },
+ {
+ sentence: "그는 학교에 갔다",
+ expected: ["그", "학교", "가다"],
+ },
+ {
+ sentence: "열심히 공부하는 학생입니다",
+ expected: ["열심히", "공부", "학생", "이다"],
+ },
+ {
+ sentence: "어제 영화를 봤어요",
+ expected: ["어제", "영화", "보다"],
+ },
+];