try load to local mongo
This commit is contained in:
polv committed
1 parent
91623834d6
commit
a26d329d46
7 files changed
+705
-5
No files matched your search
@@ -0,0 +1,118 @@
|
||||
import { MongoClient } from "mongodb";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
interface IYomitanIndex {
|
||||
title: string;
|
||||
sequenced: boolean;
|
||||
format: number; // integer?
|
||||
url: string;
|
||||
revision: string;
|
||||
attribution: string;
|
||||
}
|
||||
|
||||
interface IYomitanTerm {
|
||||
term: string;
|
||||
reading: string;
|
||||
def_tags: any | null;
|
||||
rules: string;
|
||||
score: number;
|
||||
glossary: any; // structured?
|
||||
sequence: number;
|
||||
term_tags: string;
|
||||
}
|
||||
|
||||
interface IYomitanTermMeta {
|
||||
term: string;
|
||||
mode: number;
|
||||
data: {
|
||||
reading: string;
|
||||
} | null;
|
||||
}
|
||||
|
||||
interface IYomitanTag {
|
||||
name: string;
|
||||
category: string;
|
||||
sort_order: number;
|
||||
notes: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
const client = new MongoClient("mongodb://root:example@localhost:27017");
|
||||
await client.connect();
|
||||
|
||||
const dbYomitan = client.db("yomitan");
|
||||
const colIndex = dbYomitan.collection<IYomitanIndex>("index");
|
||||
|
||||
// await loadFolderToYomitan("tmp/yomitan/jitenon-kotowaza");
|
||||
await client.close();
|
||||
|
||||
async function loadFolderToYomitan(folderpath: string) {
|
||||
const dbx = client.db(path.parse(folderpath).base);
|
||||
|
||||
for (const filename of await fs.readdir(folderpath)) {
|
||||
if (!filename.endsWith(".json")) continue;
|
||||
|
||||
console.log(filename);
|
||||
|
||||
if (filename === "index.json") {
|
||||
await colIndex.insertOne(
|
||||
JSON.parse(await fs.readFile(path.join(folderpath, filename), "utf-8")),
|
||||
);
|
||||
} else if (filename.startsWith("term_bank_")) {
|
||||
const terms: any[][] = JSON.parse(
|
||||
await fs.readFile(path.join(folderpath, filename), "utf-8"),
|
||||
);
|
||||
|
||||
const fields = [
|
||||
"term",
|
||||
"reading",
|
||||
"def_tags",
|
||||
"rules",
|
||||
"score",
|
||||
"glossary",
|
||||
"structured",
|
||||
"sequence",
|
||||
"term_tags",
|
||||
];
|
||||
|
||||
await dbx
|
||||
.collection<IYomitanTerm>("term")
|
||||
.insertMany(
|
||||
terms.map((rs) =>
|
||||
Object.fromEntries(rs.map((r, i) => [fields[i], r])),
|
||||
) as IYomitanTerm[],
|
||||
);
|
||||
} else if (filename.startsWith("tag_bank_")) {
|
||||
const terms: any[][] = JSON.parse(
|
||||
await fs.readFile(path.join(folderpath, filename), "utf-8"),
|
||||
);
|
||||
|
||||
const fields = [`name`, `category`, `sort_order`, `notes`, `score`];
|
||||
|
||||
await dbx
|
||||
.collection<IYomitanTag>("tag")
|
||||
.insertMany(
|
||||
terms.map((rs) =>
|
||||
Object.fromEntries(rs.map((r, i) => [fields[i], r])),
|
||||
) as IYomitanTag[],
|
||||
);
|
||||
} else if (filename.startsWith("term_meta_bank_")) {
|
||||
const terms: any[][] = JSON.parse(
|
||||
await fs.readFile(path.join(folderpath, filename), "utf-8"),
|
||||
);
|
||||
|
||||
const fields = ["term", "mode", "data"];
|
||||
|
||||
await dbx
|
||||
.collection<IYomitanTermMeta>("term_meta")
|
||||
.insertMany(
|
||||
terms.map((rs) =>
|
||||
Object.fromEntries(rs.map((r, i) => [fields[i], r])),
|
||||
) as IYomitanTermMeta[],
|
||||
);
|
||||
} else {
|
||||
console.error(filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
import sqlite3 from "better-sqlite3";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
const mainDB = sqlite3("tmp/yomitan/index.db");
|
||||
mainDB.exec(/* sql */ `
|
||||
CREATE TABLE IF NOT EXISTS dictionaries (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
revision TEXT NOT NULL,
|
||||
author TEXT,
|
||||
[url] TEXT,
|
||||
[description] TEXT,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
installed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
lang TEXT NOT NULL, -- custom
|
||||
[data] TEXT NOT NULL, -- JSON
|
||||
UNIQUE (title, revision)
|
||||
);
|
||||
`);
|
||||
|
||||
async function loadFolder(folderpath: string, lang = "ja") {
|
||||
const db = sqlite3(path.join(folderpath, "index.db"));
|
||||
|
||||
db.exec(/* sql */ `
|
||||
CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
|
||||
CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
|
||||
CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE);
|
||||
CREATE TABLE IF NOT EXISTS terms (
|
||||
id INTEGER PRIMARY KEY,
|
||||
term TEXT NOT NULL,
|
||||
reading TEXT NOT NULL,
|
||||
def_tags_id INTEGER,
|
||||
rules_id INTEGER,
|
||||
score INTEGER,
|
||||
glossary TEXT NOT NULL, -- JSON
|
||||
glossary_id INTEGER, -- NOT NULL REFERENCES glossaries(id),
|
||||
[sequence] INTEGER,
|
||||
term_tags_id INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS term_meta (
|
||||
id INTEGER PRIMARY KEY,
|
||||
term TEXT NOT NULL,
|
||||
mode TEXT NOT NULL,
|
||||
reading TEXT,
|
||||
[data] TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id INTEGER PRIMARY KEY,
|
||||
[name] TEXT NOT NULL,
|
||||
category TEXT,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
notes TEXT,
|
||||
score INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE ([name])
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS kanji (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kanji TEXT NOT NULL,
|
||||
onyomi TEXT,
|
||||
kunyomi TEXT,
|
||||
tags TEXT,
|
||||
meanings TEXT NOT NULL DEFAULT '[]', -- JSON
|
||||
[stats] TEXT NOT NULL DEFAULT '{}' -- JSON
|
||||
);
|
||||
`);
|
||||
|
||||
db.exec(/* sql */ `
|
||||
CREATE INDEX IF NOT EXISTS terms_term_reading_score ON terms(term, reading, score DESC);
|
||||
`);
|
||||
|
||||
const stmt = (() => {
|
||||
let fields = ["title", "revision", "author", "url", "description", "lang"];
|
||||
const index = mainDB.prepare(/* sql */ `
|
||||
INSERT INTO dictionaries (${fields.map((f) => `[${f}]`).join(",")}, [data])
|
||||
VALUES (${fields.map((f) => ":${f}").join(",")}, :data)
|
||||
`);
|
||||
|
||||
fields = [
|
||||
"term",
|
||||
"reading",
|
||||
"def_tags",
|
||||
"rules",
|
||||
"score",
|
||||
"glossary",
|
||||
"sequence",
|
||||
"term_tags",
|
||||
];
|
||||
const term = db.prepare(/* sql */ `
|
||||
INSERT INTO term (${fields.map((f) => `[${f}]`).join(",")})
|
||||
VALUES (${fields.map(() => "?").join(",")})
|
||||
`);
|
||||
|
||||
return {
|
||||
index,
|
||||
term,
|
||||
};
|
||||
})();
|
||||
|
||||
for (const filename of await fs.readdir(folderpath)) {
|
||||
try {
|
||||
if (!filename.endsWith(".json")) continue;
|
||||
|
||||
if (filename === "index.json") {
|
||||
const obj = JSON.parse(
|
||||
await fs.readFile(path.join(folderpath, filename), "utf-8"),
|
||||
);
|
||||
|
||||
stmt.index.run({ lang, data: JSON.stringify(obj), ...obj });
|
||||
} else if (filename.startsWith("term_bank_")) {
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(filename);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user