From cac640fdea57d0818ce596593351c2f886293d3d Mon Sep 17 00:00:00 2001 From: polv Date: Sat, 29 Aug 2026 11:06:38 +0700 Subject: [PATCH] sentence database builder script --- docs/.vitepress/vite-plugins/gtts.ts | 10 ++- docs/howtostudykorean/unit-1/10-numbers.md | 87 +++++++------------ package.json | 1 + sentence-db.js | 98 ++++++++++++++++++++++ 4 files changed, 135 insertions(+), 61 deletions(-) create mode 100644 sentence-db.js diff --git a/docs/.vitepress/vite-plugins/gtts.ts b/docs/.vitepress/vite-plugins/gtts.ts index 58339e6..9cb27a9 100644 --- a/docs/.vitepress/vite-plugins/gtts.ts +++ b/docs/.vitepress/vite-plugins/gtts.ts @@ -49,11 +49,17 @@ async function gtts( fileList = JSON.parse(fs.readFileSync(fileListJSON, "utf-8")); } - const fileNo = fileList[text] || Object.keys(fileList).length; + let fileNo = fileList[text]; + + let isFileNoExists = Number.isInteger(fileNo); + if (!isFileNoExists) { + fileNo = Math.max(0, ...Object.values(fileList)) + 1; + } + const filename = `${fileNo}.mp3`; const saveto = path.join(saveroot, filename); - if (!fs.existsSync(saveto)) { + if (!isFileNoExists || !fs.existsSync(saveto)) { await new Promise((resolve, reject) => { const p = spawn( "gtts-cli", diff --git a/docs/howtostudykorean/unit-1/10-numbers.md b/docs/howtostudykorean/unit-1/10-numbers.md index 79ad9fe..b381ba0 100644 --- a/docs/howtostudykorean/unit-1/10-numbers.md +++ b/docs/howtostudykorean/unit-1/10-numbers.md @@ -175,111 +175,80 @@ vocabulary: audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v18.mp3 en: four (4) examples: - - ko: 한국에는 **사** 계절이 뚜렷해요 - en: The **four** seasons are distinct in Korea - notes: Gemini - - ko: "**사** 월에 벚꽃이 펴요" - en: Cherry blossoms bloom in **April** - notes: Gemini + - ko: 한국은 **사**계절이 있어요 + en: There are **four** seasons in Korea - ko: 오 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v19.mp3 en: five (5) examples: - - ko: 수업이 **오** 분 뒤에 끝나요 - en: Classes end in **five** minutes - notes: Gemini - - ko: "**오** 월은 가정의 달이에요" - en: "**May** is the month of family" + - ko: 저는 **5**년 이내에 외국어를 다섯 개 배우고 싶어요 + en: I want to learn 5 languages within five years notes: Gemini + - ko: 저는 그 회사에서 **5**월까지 일할 거예요 + en: I will work at that company until **May** - ko: 육 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v20.mp3 en: six (6) examples: - - ko: 올해 **육** 월에 졸업해요 - en: I graduate in **June** of this year - notes: Gemini - - ko: 한국어 공부를 **육** 개월 동안 했어요 - en: I studied Korean for **six** months - notes: Gemini + - ko: 우리가 벌써 **6**월에 쓸 수 있는 인터넷 용량 제한을 넘었어요 + en: We already went over the data/space limit (data cap) that we can use for **June** + - ko: 1분은 **6**0초입니다 + en: One minute is **six**ty seconds - ko: 칠 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v21.mp3 en: seven (7) examples: - - ko: 여름 방학은 **칠** 월에 시작해요 - en: Summer vacation starts in **July** - notes: Gemini - - ko: 숙제를 **칠** 일 동안 했어요 - en: I did homework for **seven** days - notes: Gemini + - ko: "공, 공, **칠**, 빵! - 으악! - 공, 공, **칠**, 빵!" + en: "007 bang. - Ugh. - 007 bang!" + notes: Weverse + - ko: 저는 **7**년 동안 한국에서 살았어요 + en: I lived in Korea for **seven** years - ko: 팔 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v22.mp3 en: eight (8) examples: - - ko: "**팔** 월 날씨가 가장 더워요" - en: The weather in **August** is the hottest - notes: Gemini - - ko: 우리 사무실은 **팔** 층에 있어요 - en: Our office is on the **eighth** floor - notes: Gemini + - ko: 저는 **8**월 15일에 도착할 거예요 + en: I will arrive on **August** 15th - ko: 구 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v23.mp3 en: nine (9) examples: - - ko: 새 학기가 **구** 월에 시작돼요 - en: The new semester starts in **September** - notes: Gemini - - ko: 버스 **구** 번을 타야 해요 - en: I have to take bus **number nine** - notes: Gemini + - ko: 저는 **9**월1일에 캐나다에 돌아갈 거예요 + en: I will go back to Canada on **September** 1st - ko: 십 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v24.mp3 en: ten (10) examples: - - ko: 김치냉장고를 산 지 **십** 년이 되었어요 - en: It has been **ten** years since I bought the kimchi refrigerator - notes: Gemini - - ko: "**십** 분만 기다려 주세요" - en: Please wait for just **ten** minutes - notes: Gemini + - ko: 경찰관은 강아지의 주인에게 **십**만 원의 벌금을 물렸어요 + en: The police officers gave the owner of the dog a **100 000** won fine - ko: 백 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v25.mp3 en: one hundred (100) examples: - - ko: 시험에서 **백** 점을 맞았어요 - en: I got **a hundred** points on the test - notes: Gemini - - ko: 이 사과는 **백** 원이에요 - en: This apple costs **one hundred** won - notes: Gemini + - ko: 저는 **백** 번째 페이지를 읽었어요 + audio: https://www.howtostudykorean.com/wp-content/uploads/2015/02/Lesson10-Feb-10-17.mp3 + en: I read the **100**th page - ko: 천 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v26.mp3 en: one thousand (1000) examples: - - ko: 이 볼펜은 **천** 원이에요 - en: This ballpoint pen costs **one thousand** won - notes: Gemini - - ko: "**천** 명의 관객이 모였어요" - en: "**One thousand** spectators gathered" - notes: Gemini + - ko: 저는 한국 단어 **천** 개를 알고 있어요 + en: I know **1000** Korean words - ko: 만 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v27.mp3 en: ten thousand (10000) examples: - - ko: 신발이 **만** 원이에요 - en: These shoes cost **ten thousand** won - notes: Gemini - - ko: 이 옷은 삼**만** 원이에요 - en: This clothes cost **thirty thousand** won - notes: Gemini + - ko: "아니요. 10**만**원 조금 넘어요" + en: "No. A little more than 10**0,000** won" - ko: 영 audio: https://www.howtostudykorean.com/wp-content/uploads/2015/11/Lesson10v28.mp3 diff --git a/package.json b/package.json index 3d3c4fc..19ebbff 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "ko-grammar", "version": "1.0.0", "description": "", + "type": "module", "main": "index.js", "scripts": { "start": "tauri dev", diff --git a/sentence-db.js b/sentence-db.js new file mode 100644 index 0000000..b2fa6cb --- /dev/null +++ b/sentence-db.js @@ -0,0 +1,98 @@ +import sqlite3 from "better-sqlite3"; +import { MeCab } from "kuromoji-ko"; +import fs from "node:fs/promises"; +import yaml from "yaml"; + +const mecab = await MeCab.create({ + engine: "ko", + dictPath: "./node_modules/kuromoji-ko/dict", +}); + +const db = sqlite3(".sentence.db"); +db.exec(/* sql */ ` + CREATE TABLE IF NOT EXISTS sentence ( + ko TEXT NOT NULL, + en TEXT, + original TEXT NOT NULL, + [url] TEXT NOT NULL, + notes TEXT, + filepath TEXT NOT NULL + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u ON sentence(original, [url]); + + CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 ( + ko, + en, + original UNINDEXED, + [url] UNINDEXED, + notes UNINDEXED, + filepath UNINDEXED, + content='sentence', + tokenize='porter' + ); + + -- Triggers to keep the FTS index up to date. + CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN + INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en); + END; + CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN + INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en); + END; + CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN + INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en); + INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en); + END; +`); + +const stmt = db.prepare(/* sql */ ` + INSERT OR IGNORE INTO sentence (ko, original, en, [url], notes, filepath) + VALUES (:parsed, :ko, :en, :audio, :notes, :filepath) + `); +const tx = db.transaction((ss) => { + for (const s of ss) { + if (s.notes === "Gemini") continue; + stmt.run({ + ko: s.ko.replaceAll("**", ""), + parsed: mecab + .parse(s.ko) + .filter((t) => t.pos.some((p) => /^[NVM]/.test(p))) + .map((t) => t.lemma) + .join(" "), + en: (s.en || "").replaceAll("**", ""), + audio: s.audio || "", + notes: Array.isArray(s.notes) ? s.notes.join("\n") : s.notes || null, + filepath: s.filepath, + }); + } +}); + +const SEPARATOR = "---\n"; + +for await (const md of fs.glob("docs/**/*.md")) { + let text = await fs.readFile(md, "utf-8"); + if (!text.startsWith(SEPARATOR)) continue; + try { + const ss = []; + + const { vocabulary, sentence } = + yaml.parse(text.slice(SEPARATOR.length).split(SEPARATOR, 1)[0]) || {}; + if (!vocabulary) continue; + + for (const { examples = [] } of Object.values(vocabulary).flat(1)) { + ss.push(...examples.map((o) => ({ ...o, filepath: md }))); + } + + if (sentence) { + ss.push( + ...Object.values(sentence) + .flat(1) + .map((o) => ({ ...o, filepath: md })), + ); + } + + tx(ss); + } catch (e) { + console.error(e, md); + } +}