sentence database builder script

docs/.vitepress/vite-plugins/gtts.ts

@@ -49,11 +49,17 @@ async function gtts(
     fileList = JSON.parse(fs.readFileSync(fileListJSON, "utf-8"));
   }
 
-  const fileNo = fileList[text] || Object.keys(fileList).length;
+  let fileNo = fileList[text];
+
+  let isFileNoExists = Number.isInteger(fileNo);
+  if (!isFileNoExists) {
+    fileNo = Math.max(0, ...Object.values(fileList)) + 1;
+  }
+
   const filename = `${fileNo}.mp3`;
   const saveto = path.join(saveroot, filename);
 
-  if (!fs.existsSync(saveto)) {
+  if (!isFileNoExists || !fs.existsSync(saveto)) {
     await new Promise((resolve, reject) => {
       const p = spawn(
         "gtts-cli",

docs/howtostudykorean/unit-1/10-numbers.md

@@ -175,111 +175,80 @@ vocabulary:
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v18.mp3
       en: four (4)
       examples:
-        - ko: 한국에는 **사** 계절이 뚜렷해요
+        - ko: 한국은 **사**계절이 있어요
-          en: The **four** seasons are distinct in Korea
+          en: There are **four** seasons in Korea
-          notes: Gemini
-        - ko: "**사** 월에 벚꽃이 펴요"
-          en: Cherry blossoms bloom in **April**
-          notes: Gemini
 
     - ko: 오
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v19.mp3
       en: five (5)
       examples:
-        - ko: 수업이 **오** 분 뒤에 끝나요
+        - ko: 저는 **5**년 이내에 외국어를 다섯 개 배우고 싶어요
-          en: Classes end in **five** minutes
+          en: I want to learn 5 languages within five years
-          notes: Gemini
-        - ko: "**오** 월은 가정의 달이에요"
-          en: "**May** is the month of family"
           notes: Gemini
+        - ko: 저는 그 회사에서 **5**월까지 일할 거예요
+          en: I will work at that company until **May**
 
     - ko: 육
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v20.mp3
       en: six (6)
       examples:
-        - ko: 올해 **육** 월에 졸업해요
+        - ko: 우리가 벌써 **6**월에 쓸 수 있는 인터넷 용량 제한을 넘었어요
-          en: I graduate in **June** of this year
+          en: We already went over the data/space limit (data cap) that we can use for **June**
-          notes: Gemini
+        - ko: 1분은 **6**0초입니다
-        - ko: 한국어 공부를 **육** 개월 동안 했어요
+          en: One minute is **six**ty seconds
-          en: I studied Korean for **six** months
-          notes: Gemini
 
     - ko: 칠
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v21.mp3
       en: seven (7)
       examples:
-        - ko: 여름 방학은 **칠** 월에 시작해요
+        - ko: "공, 공, **칠**, 빵! - 으악! - 공, 공, **칠**, 빵!"
-          en: Summer vacation starts in **July**
+          en: "007 bang. - Ugh. - 007 bang!"
-          notes: Gemini
+          notes: Weverse
-        - ko: 숙제를 **칠** 일 동안 했어요
+        - ko: 저는 **7**년 동안 한국에서 살았어요
-          en: I did homework for **seven** days
+          en: I lived in Korea for **seven** years
-          notes: Gemini
 
     - ko: 팔
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v22.mp3
       en: eight (8)
       examples:
-        - ko: "**팔** 월 날씨가 가장 더워요"
+        - ko: 저는 **8**월 15일에 도착할 거예요
-          en: The weather in **August** is the hottest
+          en: I will arrive on **August** 15th
-          notes: Gemini
-        - ko: 우리 사무실은 **팔** 층에 있어요
-          en: Our office is on the **eighth** floor
-          notes: Gemini
 
     - ko: 구
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v23.mp3
       en: nine (9)
       examples:
-        - ko: 새 학기가 **구** 월에 시작돼요
+        - ko: 저는 **9**월1일에 캐나다에 돌아갈 거예요
-          en: The new semester starts in **September**
+          en: I will go back to Canada on **September** 1st
-          notes: Gemini
-        - ko: 버스 **구** 번을 타야 해요
-          en: I have to take bus **number nine**
-          notes: Gemini
 
     - ko: 십
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v24.mp3
       en: ten (10)
       examples:
-        - ko: 김치냉장고를 산 지 **십** 년이 되었어요
+        - ko: 경찰관은 강아지의 주인에게 **십**만 원의 벌금을 물렸어요
-          en: It has been **ten** years since I bought the kimchi refrigerator
+          en: The police officers gave the owner of the dog a **100 000** won fine
-          notes: Gemini
-        - ko: "**십** 분만 기다려 주세요"
-          en: Please wait for just **ten** minutes
-          notes: Gemini
 
     - ko: 백
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v25.mp3
       en: one hundred (100)
       examples:
-        - ko: 시험에서 **백** 점을 맞았어요
+        - ko: 저는 **백** 번째 페이지를 읽었어요
-          en: I got **a hundred** points on the test
+          audio: https://www.howtostudykorean.com/wp-content/uploads/2015/02/Lesson10-Feb-10-17.mp3
-          notes: Gemini
+          en: I read the **100**th page
-        - ko: 이 사과는 **백** 원이에요
-          en: This apple costs **one hundred** won
-          notes: Gemini
 
     - ko: 천
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v26.mp3
       en: one thousand (1000)
       examples:
-        - ko: 이 볼펜은 **천** 원이에요
+        - ko: 저는 한국 단어 **천** 개를 알고 있어요
-          en: This ballpoint pen costs **one thousand** won
+          en: I know **1000** Korean words
-          notes: Gemini
-        - ko: "**천** 명의 관객이 모였어요"
-          en: "**One thousand** spectators gathered"
-          notes: Gemini
 
     - ko: 만
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v27.mp3
       en: ten thousand (10000)
       examples:
-        - ko: 신발이 **만** 원이에요
+        - ko: "아니요. 10**만**원 조금 넘어요"
-          en: These shoes cost **ten thousand** won
+          en: "No. A little more than 10**0,000** won"
-          notes: Gemini
-        - ko: 이 옷은 삼**만** 원이에요
-          en: This clothes cost **thirty thousand** won
-          notes: Gemini
 
     - ko: 영
       audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v28.mp3

package.json

@@ -2,6 +2,7 @@
   "name": "ko-grammar",
   "version": "1.0.0",
   "description": "",
+  "type": "module",
   "main": "index.js",
   "scripts": {
     "start": "tauri dev",

sentence-db.js

@@ -0,0 +1,98 @@
+import sqlite3 from "better-sqlite3";
+import { MeCab } from "kuromoji-ko";
+import fs from "node:fs/promises";
+import yaml from "yaml";
+
+const mecab = await MeCab.create({
+  engine: "ko",
+  dictPath: "./node_modules/kuromoji-ko/dict",
+});
+
+const db = sqlite3(".sentence.db");
+db.exec(/* sql */ `
+  CREATE TABLE IF NOT EXISTS sentence (
+    ko        TEXT NOT NULL,
+    en        TEXT,
+    original  TEXT NOT NULL,
+    [url]     TEXT NOT NULL,
+    notes     TEXT,
+    filepath  TEXT NOT NULL
+  );
+
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u ON sentence(original, [url]);
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
+    ko,
+    en,
+    original  UNINDEXED,
+    [url]     UNINDEXED,
+    notes     UNINDEXED,
+    filepath  UNINDEXED,
+    content='sentence',
+    tokenize='porter'
+  );
+
+  -- Triggers to keep the FTS index up to date.
+  CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
+    INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en);
+  END;
+  CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
+    INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en);
+  END;
+  CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
+    INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en);
+    INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en);
+  END;
+`);
+
+const stmt = db.prepare(/* sql */ `
+  INSERT OR IGNORE INTO sentence (ko, original, en, [url], notes, filepath)
+  VALUES (:parsed, :ko, :en, :audio, :notes, :filepath)
+  `);
+const tx = db.transaction((ss) => {
+  for (const s of ss) {
+    if (s.notes === "Gemini") continue;
+    stmt.run({
+      ko: s.ko.replaceAll("**", ""),
+      parsed: mecab
+        .parse(s.ko)
+        .filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
+        .map((t) => t.lemma)
+        .join(" "),
+      en: (s.en || "").replaceAll("**", ""),
+      audio: s.audio || "",
+      notes: Array.isArray(s.notes) ? s.notes.join("\n") : s.notes || null,
+      filepath: s.filepath,
+    });
+  }
+});
+
+const SEPARATOR = "---\n";
+
+for await (const md of fs.glob("docs/**/*.md")) {
+  let text = await fs.readFile(md, "utf-8");
+  if (!text.startsWith(SEPARATOR)) continue;
+  try {
+    const ss = [];
+
+    const { vocabulary, sentence } =
+      yaml.parse(text.slice(SEPARATOR.length).split(SEPARATOR, 1)[0]) || {};
+    if (!vocabulary) continue;
+
+    for (const { examples = [] } of Object.values(vocabulary).flat(1)) {
+      ss.push(...examples.map((o) => ({ ...o, filepath: md })));
+    }
+
+    if (sentence) {
+      ss.push(
+        ...Object.values(sentence)
+          .flat(1)
+          .map((o) => ({ ...o, filepath: md })),
+      );
+    }
+
+    tx(ss);
+  } catch (e) {
+    console.error(e, md);
+  }
+}