99 lines
2.7 KiB
JavaScript
99 lines
2.7 KiB
JavaScript
import sqlite3 from "better-sqlite3";
|
|
import { MeCab } from "kuromoji-ko";
|
|
import fs from "node:fs/promises";
|
|
import yaml from "yaml";
|
|
|
|
const mecab = await MeCab.create({
|
|
engine: "ko",
|
|
dictPath: "./node_modules/kuromoji-ko/dict",
|
|
});
|
|
|
|
const db = sqlite3(".sentence.db");
|
|
db.exec(/* sql */ `
|
|
CREATE TABLE IF NOT EXISTS sentence (
|
|
ko TEXT NOT NULL,
|
|
en TEXT,
|
|
original TEXT NOT NULL,
|
|
[url] TEXT NOT NULL,
|
|
notes TEXT,
|
|
filepath TEXT NOT NULL
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_sentence_u ON sentence(original, [url]);
|
|
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS sentence_fts USING fts5 (
|
|
ko,
|
|
en,
|
|
original UNINDEXED,
|
|
[url] UNINDEXED,
|
|
notes UNINDEXED,
|
|
filepath UNINDEXED,
|
|
content='sentence',
|
|
tokenize='porter'
|
|
);
|
|
|
|
-- Triggers to keep the FTS index up to date.
|
|
CREATE TRIGGER IF NOT EXISTS t1_ai AFTER INSERT ON sentence BEGIN
|
|
INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en);
|
|
END;
|
|
CREATE TRIGGER IF NOT EXISTS t1_ad AFTER DELETE ON sentence BEGIN
|
|
INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en);
|
|
END;
|
|
CREATE TRIGGER IF NOT EXISTS t1_au AFTER UPDATE ON sentence BEGIN
|
|
INSERT INTO sentence_fts(fts_idx, rowid, ko, en) VALUES('delete', old.rowid, old.ko, old.en);
|
|
INSERT INTO sentence_fts(rowid, ko, en) VALUES (new.rowid, new.ko, new.en);
|
|
END;
|
|
`);
|
|
|
|
const stmt = db.prepare(/* sql */ `
|
|
INSERT OR IGNORE INTO sentence (ko, original, en, [url], notes, filepath)
|
|
VALUES (:parsed, :ko, :en, :audio, :notes, :filepath)
|
|
`);
|
|
const tx = db.transaction((ss) => {
|
|
for (const s of ss) {
|
|
if (s.notes === "Gemini") continue;
|
|
stmt.run({
|
|
ko: s.ko.replaceAll("**", ""),
|
|
parsed: mecab
|
|
.parse(s.ko)
|
|
.filter((t) => t.pos.some((p) => /^[NVM]/.test(p)))
|
|
.map((t) => t.lemma)
|
|
.join(" "),
|
|
en: (s.en || "").replaceAll("**", ""),
|
|
audio: s.audio || "",
|
|
notes: Array.isArray(s.notes) ? s.notes.join("\n") : s.notes || null,
|
|
filepath: s.filepath,
|
|
});
|
|
}
|
|
});
|
|
|
|
const SEPARATOR = "---\n";
|
|
|
|
for await (const md of fs.glob("docs/**/*.md")) {
|
|
let text = await fs.readFile(md, "utf-8");
|
|
if (!text.startsWith(SEPARATOR)) continue;
|
|
try {
|
|
const ss = [];
|
|
|
|
const { vocabulary, sentence } =
|
|
yaml.parse(text.slice(SEPARATOR.length).split(SEPARATOR, 1)[0]) || {};
|
|
if (!vocabulary) continue;
|
|
|
|
for (const { examples = [] } of Object.values(vocabulary).flat(1)) {
|
|
ss.push(...examples.map((o) => ({ ...o, filepath: md })));
|
|
}
|
|
|
|
if (sentence) {
|
|
ss.push(
|
|
...Object.values(sentence)
|
|
.flat(1)
|
|
.map((o) => ({ ...o, filepath: md })),
|
|
);
|
|
}
|
|
|
|
tx(ss);
|
|
} catch (e) {
|
|
console.error(e, md);
|
|
}
|
|
}
|