node/mongo server

This commit is contained in:
polv committed 2026-03-02 23:51:25 +07:00
1 parent 5b8a728db4
commit bedb810119
16 files changed
+1558 -8

No files matched your search

+202
View File
@@ -0,0 +1,202 @@
import "dotenv/config";
import AdmZip from "adm-zip";
import { readdirSync } from "fs";
import { MongoClient, ObjectId } from "mongodb";
import { join as joinPath } from "path";
import { Kanji, KanjiEntry } from "./kanji";
import { KanjiMeta, KanjiMetaEntry } from "./kanji-meta";
import { Tag, TagEntry } from "./tag";
import { Term, TermEntry } from "./term";
import { TermMeta, TermMetaEntry } from "./term-meta";
if (require.main === module) {
const client = new MongoClient(process.env.MONGO_URI!);
(async () => {
await client.connect();
const db = client.db("yomitan");
const colMeta = db.collection<DictionaryIndex>("meta");
await colMeta.createIndex({ title: 1 }, { unique: true });
const colKanjiMeta = db.collection<KanjiMeta & { meta_id: ObjectId }>(
"kanji-meta",
);
await colKanjiMeta.createIndex({ meta_id: 1 });
const colKanji = db.collection<Kanji & { meta_id: ObjectId }>("kanji");
await colKanji.createIndex({ meta_id: 1 });
const colTag = db.collection<Tag & { meta_id: ObjectId }>("tag");
await colTag.createIndex({ meta_id: 1 });
const colTermMeta = db.collection<TermMeta & { meta_id: ObjectId }>(
"term-meta",
);
await colTermMeta.createIndex({ meta_id: 1 });
const colTerm = db.collection<Term & { meta_id: ObjectId }>("term");
await colTerm.createIndex({ meta_id: 1 });
const zipPath = "../src-tauri/resources/yomitan/ja";
for (const f of readdirSync(zipPath)) {
if (!f.endsWith(".zip")) continue;
const zipFile = new AdmZip(joinPath(zipPath, f));
const idx = zipFile.getEntry("index.json");
if (!idx) continue;
const meta: DictionaryIndex = JSON.parse(idx.getData().toString("utf-8"));
const title = meta.title.replace(/\[.+?\]$/, "").trimEnd();
if (title !== meta.title) {
console.log(meta.title);
meta.title = title;
}
console.log(meta.title);
const it = await colMeta.insertOne(meta).catch((e) => {
console.error(e);
return null;
});
if (!it) continue;
const meta_id = it.insertedId;
for (const et of zipFile.getEntries()) {
const f = et.entryName;
if (f.startsWith("kanji_meta_bank")) {
// yet empty
await colKanjiMeta.insertMany(
JSON.parse(et.getData().toString("utf-8")).map((t: any) => ({
meta_id,
...KanjiMetaEntry.fromBank(t).json,
})),
);
} else if (f.startsWith("kanji_bank")) {
// yet empty
await colKanji.insertMany(
JSON.parse(et.getData().toString("utf-8")).map((t: any) => ({
meta_id,
...KanjiEntry.fromBank(t).json,
})),
);
} else if (f.startsWith("tag_bank")) {
// 5 so far
await colTag.insertMany(
JSON.parse(et.getData().toString("utf-8")).map((t: any) => ({
meta_id,
...TagEntry.fromBank(t).json,
})),
);
} else if (f.startsWith("term_bank")) {
// lots
await colTerm.insertMany(
JSON.parse(et.getData().toString("utf-8")).map((t: any) => ({
meta_id,
...TermEntry.fromBank(t).json,
})),
);
} else if (f.startsWith("term_meta_bank")) {
// yet empty
await colTermMeta.insertMany(
JSON.parse(et.getData().toString("utf-8")).map((t: any) => ({
meta_id,
...TermMetaEntry.fromBank(t).json,
})),
);
}
}
}
})().finally(async () => {
await client.close();
});
}
/**
* The schema for the index.json file that contains metadata about the dictionary.
* PLEASE ALWAYS PUT AS MUCH DETAIL IN THIS AS POSSIBLE.
*
* Note that this information can be displayed in Yomichan by going to the dictionaries overview page
* and clicking the three dots, then Details....
*
* "isoLanguageCode": {
* "type": "string",
* "description": "ISO language code (ISO 639-1 where possible, ISO 639-3 otherwise).",
* "pattern": "^[a-z]{2,3}$"
* }
* "required": [
* "title",
* "revision"
* ],
* "anyOf": [
* {
* "required": ["format"]
* },
* {
* "required": ["version"]
* }
* ],
* "dependencies": {
* "isUpdatable": ["indexUrl", "downloadUrl"]
* }
*
* @see https://github.com/yomidevs/yomitan/blob/master/ext/data/schemas/dictionary-index-schema.json
*/
interface DictionaryIndex {
title: string;
// "description": "Title of the dictionary."
revision: string;
// "description": "Revision of the dictionary. This value is displayed, and used to check for dictionary updates."
minimumYomitanVersion?: string;
// "description": "Minimum version of Yomitan that is compatible with this dictionary."
sequenced?: boolean;
// "default": false,
// "description": "Whether or not this dictionary contains sequencing information for related terms."
format?: 1 | 2 | 3;
// "type": "integer",
// "description": "Format of data found in the JSON data files.",
// "enum": [1, 2, 3]
version?: 1 | 2 | 3;
// "type": "integer",
// "description": "Alias for format.",
// "enum": [1, 2, 3]
author?: string;
// "description": "Creator of the dictionary."
isUpdatable?: boolean;
// "const": true,
// "description": "Whether this dictionary contains links to its latest version."
indexUrl?: string;
// "description": "URL for the index file of the latest revision of the dictionary, used to check for updates."
downloadUrl?: string;
// "description": "URL for the download of the latest revision of the dictionary."
url?: string;
// "description": "URL for the source of the dictionary, displayed in the dictionary details."
description?: string;
// "description": "Description of the dictionary data."
attribution?: string;
// "description": "Attribution information for the dictionary data."
sourceLanguage?: string;
// "$ref": "#/definitions/isoLanguageCode",
// "description": "Language of the terms in the dictionary."
targetLanguage?: string;
// "$ref": "#/definitions/isoLanguageCode",
// "description": "Main language of the definitions in the dictionary."
frequencyMode?: "occurrence-based" | "rank-based";
tagMeta?: {
// "description": "Tag information for terms and kanji. This object is obsolete and individual tag files should be used instead.",
// "description": "Information about a single tag. The object key is the name of the tag.",
category?: string;
// "description": "Category for the tag."
order?: number;
// "description": "Sorting order for the tag."
notes?: string;
// "description": "Notes for the tag."
score?: number;
// "description": "Score used to determine popularity. Negative values are more rare and positive values are more frequent. This score is also used to sort search results."
};
}
+62
View File
@@ -0,0 +1,62 @@
export interface KanjiMeta {
character: string;
type: string;
freqNumber?: number;
freqString?: string;
}
export class KanjiMetaEntry {
static fromBank([character, type, freq]: DictionaryKanjiMeta) {
let freqNumber = 0;
let freqString = "";
if (freq) {
if (typeof freq === "string") {
freqString = freq;
} else if (typeof freq === "number") {
freqNumber = freq;
} else {
freqString = freq.displayValue || "";
freqNumber = freq.value;
}
}
return new KanjiMetaEntry({
character,
type,
freqNumber: freqNumber || undefined,
freqString: freqString || undefined,
});
}
constructor(public json: KanjiMeta) {}
}
/**
* kanji_meta_bank_${number}.json
*
* The meta bank for kanji information. Right now, this is only used to store kanji frequency data.
*
* @see https://github.com/themoeway/yomitan/tree/master/ext/data/schemas/dictionary-kanji-meta-bank-v3-schema.json
*/
type DictionaryKanjiMetaBankV3 = DictionaryKanjiMeta[];
type DictionaryKanjiMeta = [
// "description": "Metadata about a single kanji character.",
// "minItems": 3,
// "maxItems": 3,
string,
// "minLength": 1
string,
// "const": "freq",
// "description": "Type of data. \"freq\" corresponds to frequency information."
Frequency,
];
export type Frequency = string | number | FrequencyObject;
interface FrequencyObject {
value: number;
displayValue?: string;
}
+63
View File
@@ -0,0 +1,63 @@
export interface Kanji {
character: string;
onyomi?: string[];
kunyomi?: string[];
tags?: string[];
meanings?: string[];
stats?: Record<string, string>;
}
export class KanjiEntry {
static fromBank([
character,
onyomi,
kunyomi,
tags,
meanings,
stats,
]: DictionaryKanji) {
return new KanjiEntry({
character,
onyomi: onyomi ? onyomi.split(" ") : undefined,
kunyomi: kunyomi ? kunyomi.split(" ") : undefined,
tags: tags ? tags.split(" ") : undefined,
meanings: meanings.length ? meanings : undefined,
stats,
});
}
constructor(public json: Kanji) {}
}
/**
* kanji_bank_${number}.json
*
* Contains information used in the kanji viewer - meaning, readings, statistics, and codepoints.
* Unfortunately a lot of the structuring is hardcoded and can't be customized nearly as much as with term definitions.
*
* @see https://github.com/yomidevs/yomitan/blob/master/ext/data/schemas/dictionary-kanji-bank-v3-schema.json
*/
type DictionaryKanjiBankV3 = DictionaryKanji[];
type DictionaryKanji = [
// "description": "Information about a single kanji character.",
// "minItems": 6,
// "maxItems": 6,
string,
// "description": "Kanji character.",
// "minLength": 1
string,
// "description": "String of space-separated onyomi readings for the kanji character. An empty string is treated as no readings."
string,
// "description": "String of space-separated kunyomi readings for the kanji character. An empty string is treated as no readings."
string,
// "description": "String of space-separated tags for the kanji character. An empty string is treated as no tags."
string[],
// "description": "Array of meanings for the kanji character.",
{
// "description": "Various stats for the kanji character.",
[k: string]: string;
},
];
+52
View File
@@ -0,0 +1,52 @@
export interface Tag {
tag: string;
category: string;
sorting: number;
notes: string;
popularity: number;
}
export class TagEntry {
static fromBank([tag, category, sorting, notes, popularity]: DictionaryTag) {
return new TagEntry({
tag,
category,
sorting,
notes,
popularity,
});
}
constructor(public json: Tag) {}
}
/**
* tag_bank_${number}.json
*
* The tag bank for term information.
*
* This is where you'll define tags for kanji and term dictionaries, like for example specifying parts of speech or kanken level.
* These are generally displayed in Yomichan as grey tags next to the dictionary name.
*
* @see https://github.com/yomidevs/yomitan/blob/master/ext/data/schemas/dictionary-tag-bank-v3-schema.json
*/
type DictionaryTagBankV3 = DictionaryTag[];
type DictionaryTag = [
// "description": "Information about a single tag.",
// "minItems": 5,
// "maxItems": 5,
string,
// "description": "Tag name."
string,
// "description": "Category for the tag."
number,
// "description": "Sorting order for the tag."
string,
// "description": "Notes for the tag."
number,
// "description": "Score used to determine popularity.
// Negative values are more rare and positive values are more frequent.
// This score is also used to sort search results."
];
+154
View File
@@ -0,0 +1,154 @@
import { Frequency } from "./kanji-meta";
export interface TermFreq {
text: string;
type: "freq";
data: {
reading?: string;
freqNumber?: number;
freqString?: string;
};
}
export interface TermPitch {
text: string;
type: "pitch";
data: {
pitches: PitchEntry[];
};
}
export interface TermIPA {
text: string;
type: "ipa";
data: {
reading: string;
transcriptions: IPATranscription[];
};
}
export type TermMeta = TermFreq | TermPitch | TermIPA;
export class TermMetaEntry {
static fromBank([text, type, data]: DictionaryTermMeta) {
switch (type) {
case "freq":
const freq = data;
let reading = "";
let freqNumber = 0;
let freqString = "";
if (freq) {
if (typeof freq === "string") {
freqString = freq;
} else if (typeof freq === "number") {
freqNumber = freq;
} else if ("reading" in freq) {
reading = freq.reading;
const f = freq.frequency;
if (typeof f === "string") {
freqString = f;
} else if (typeof f === "number") {
freqNumber = f;
} else {
freqString = f.displayValue || "";
freqNumber = f.value;
}
} else {
freqString = freq.displayValue || "";
freqNumber = freq.value;
}
}
return new TermMetaEntry({
text,
type,
data: {
reading: reading || undefined,
freqNumber: freqNumber || undefined,
freqString: freqString || undefined,
},
});
case "pitch":
return new TermMetaEntry({ text, type, data });
case "ipa":
return new TermMetaEntry({ text, type, data });
}
return new TermMetaEntry({
text,
type,
data: data as any,
});
}
constructor(public json: TermMeta) {}
}
type DictionaryTermMetaBankV3 = DictionaryTermMeta[];
type DictionaryTermMeta =
| DictionaryTermMeta_freq
| DictionaryTermMeta_pitch
| DictionaryTermMeta_ipa;
type DictionaryTermMeta_freq = [string, "freq", Frequency | ReadingFrequency];
type DictionaryTermMeta_pitch = [string, "pitch", Pitch];
type DictionaryTermMeta_ipa = [string, "ipa", IPA];
interface ReadingFrequency {
reading: string;
// "description": "Reading for the term."
frequency: Frequency;
}
interface Pitch {
reading: string;
pitches: PitchEntry[];
}
interface PitchEntry {
position: PitchPosition_integer | PitchPosition_string;
nasal?: PitchNasal | PitchNasal[];
devoice?: PitchDevoice | PitchDevoice[];
tags?: string[];
// "description": "Tag for this pitch accent. This typically corresponds to a certain type of part of speech."
}
type PitchPosition_integer = number;
// "type": "integer",
// "description": "Mora position of the pitch accent downstep. A value of 0 indicates that the word does not have a downstep (heiban).",
// "minimum": 0
type PitchPosition_string = string;
// "type": "string",
// "description": "Pitch level of each mora with H representing high and L representing low. For example: HHLL for a 4 mora word. Add an additional pitch level at the end to explicitly define the suffix.",
// "pattern": "^[HL]+$"
type PitchNasal = number;
// "type": "integer",
// "description": "Position of a mora with nasal sound.",
// "minimum": 0
type PitchDevoice = number;
// "type": "integer",
// "description": "Position of a mora with devoiced sound.",
// "minimum": 0
interface IPA {
// "description": "IPA transcription information for the term.",
reading: string;
// "description": "Reading for the term."
transcriptions: IPATranscription[];
}
interface IPATranscription {
ipa?: string;
// "description": "IPA transcription for the term."
tags: string[];
// "description": "Tag for this IPA transcription."
}
+334
View File
@@ -0,0 +1,334 @@
export interface Term {
text: string;
reading?: string;
defTags?: string[];
rules?: string[];
popularity?: number;
definition: DictionaryDefinition;
sequence?: number;
tags?: string[];
}
export class TermEntry {
static fromBank([
text,
reading,
defTags,
rules,
popularity,
definition,
sequence,
tags,
]: DictionaryTerm) {
return new TermEntry({
text,
reading,
defTags: defTags ? defTags.split(" ") : undefined,
rules: rules ? rules.split(" ") : undefined,
popularity,
definition,
sequence,
tags: tags ? tags.split(" ") : undefined,
});
}
constructor(public json: Term) {}
}
/**
* term_bank_${number}.json
*
* The term bank for term information. This is where dictionary readings, definitions, and such are stored.
*
* @see https://github.com/themoeway/yomitan/tree/master/ext/data/schemas/dictionary-term-bank-v3-schema.json
*/
type DictionaryTermBankV3 = DictionaryTerm[];
type DictionaryTerm = [
// "description": "Information about a single term.",
// "minItems": 8,
// "maxItems": 8,
string,
// "description": "The text for the term."
// "description": "Reading of the term, or an empty string if the reading is the same as the term."
string,
string | null,
// "description": "String of space-separated tags for the definition. An empty string is treated as no tags."
string,
// "description": "String of space-separated rule identifiers for the definition which is used to validate deinflection.
// An empty string should be used for words which aren't inflected."
number,
// "description": "Score used to determine popularity.
// Negative values are more rare and positive values are more frequent.
// This score is also used to sort search results."
DictionaryDefinition,
number,
// "type": "integer",
// "description": "Sequence number for the term.
// Terms with the same sequence number can be shown together when the \"resultOutputMode\" option is set to \"merge\"."
string,
// "description": "String of space-separated tags for the term. An empty string is treated as no tags."
];
type DictionaryDefinition =
| string
| DictionaryDefinitionObject
| DictionaryDefinitionInflected;
type DictionaryDefinitionObject =
| DictionaryDefinition_text
| DictionaryDefinition_structured
| DictionaryDefinition_img;
type DictionaryDefinitionInflected = [
// "description": "Deinflection of the term to an uninflected term.",
// "minItems": 2,
// "maxItems": 2,
string,
// "description": "The uninflected term."
string[],
// "description": "A chain of inflection rules that produced the inflected term",
// "description": "A single inflection rule."
];
interface DictionaryDefinition_text {
type: "text";
text: string;
// "description": "Single definition for the term."
}
interface DictionaryDefinition_structured {
type: "structured-content";
content: StructuredContent;
}
interface DictionaryDefinition_img {
type: "image";
path: string;
width?: number;
// "type": "integer",
// "description": "Preferred width of the image.",
// "minimum": 1
height?: number;
// "type": "integer",
// "description": "Preferred height of the image.",
// "minimum": 1
title?: string;
// "description": "Hover text for the image."
alt?: string;
// "description": "Alt text for the image."
description?: string;
// "description": "Description of the image."
pixelated?: boolean;
// "description": "Whether or not the image should appear pixelated at sizes larger than the image's native resolution.",
// "default": false
imageRendering?: "auto" | "pixelated" | "crisp-edges";
// "description": "Controls how the image is rendered. The value of this field supersedes the pixelated field.",
// "default": "auto"
appearance?: "auto" | "monochrome";
// "description": "Controls the appearance of the image.
// The \"monochrome\" value will mask the opaque parts of the image using the current text color.",
// "default": "auto"
background?: boolean;
// "description": "Whether or not a background color is displayed behind the image.",
// "default": true
collapsed?: boolean;
// "description": "Whether or not the image is collapsed by default.",
// "default": false
collapsible?: boolean;
// "description": "Whether or not the image can be collapsed.",
// "default": true
}
type StructuredContent =
| string
| StructuredContent[]
| StructuredContent_br
| StructuredContent_container
| StructuredContent_table
| StructuredContent_styled_container
| StructuredContent_img
| StructuredContent_link;
interface StructuredContent_br {
tag: "br";
data?: StructuredContentData;
}
interface StructuredContent_container {
tag: "ruby" | "rt" | "rp" | "table" | "thead" | "tbody" | "tfoot" | "tr";
content?: StructuredContent;
data?: StructuredContentData;
lang?: string;
}
interface StructuredContent_table {
tag: "td" | "th";
content?: StructuredContent;
data?: StructuredContentData;
colSpan?: number;
// "type": "integer",
// "minimum": 1
rowSpan?: number;
// "type": "integer",
// "minimum": 1
lang?: string;
}
interface StructuredContent_styled_container {
tag: "span" | "div" | "ol" | "ul" | "li" | "details" | "summary";
content?: StructuredContent;
data?: StructuredContentData;
style: StructuredContentStyle;
title?: string;
// "description": "Hover text for the element."
open?: boolean;
// "description": "Whether or not the details element is open by default."
lang?: string;
}
interface StructuredContent_img {
tag: "img";
data?: StructuredContentData;
path: string;
// "description": "Path to the image file in the archive."
width?: number;
// "description": "Preferred width of the image.",
// "minimum": 0
height?: number;
// "description": "Preferred height of the image.",
// "minimum": 0
title?: string;
// "description": "Hover text for the image."
alt?: string;
// "description": "Alt text for the image."
description?: string;
// "description": "Description of the image."
pixelated?: boolean;
// "description": "Whether or not the image should appear pixelated at sizes larger than the image's native resolution.",
// "default": false
imageRendering?: "auto" | "pixelated" | "crisp-edges";
// "description": "Controls how the image is rendered. The value of this field supersedes the pixelated field.",
// "default": "auto"
appearance?: "auto" | "monochrome";
// "description": "Controls the appearance of the image. The \"monochrome\" value will mask the opaque parts of the image using the current text color.",
// "default": "auto"
background?: boolean;
// "description": "Whether or not a background color is displayed behind the image.",
// "default": true
collapsed?: boolean;
// "description": "Whether or not the image is collapsed by default.",
// "default": false
collapsible?: boolean;
// "description": "Whether or not the image can be collapsed.",
// "default": false
verticalAlign:
| "baseline"
| "sub"
| "super"
| "text-top"
| "text-bottom"
| "middle"
| "top"
| "bottom";
// "description": "The vertical alignment of the image.",
border?: string;
// "description": "Shorthand for border width, style, and color."
borderRadius?: string;
// "description": "Roundness of the corners of the image's outer border edge."
sizeUnits: "px" | "em";
// "description": "The units for the width and height.",
}
interface StructuredContent_link {
tag: "a";
content?: StructuredContent;
href: string;
// "description": "The URL for the link. URLs starting with a ? are treated as internal links to other dictionary content.",
// "pattern": "^(?:https?:|\\?)[\\w\\W]*"
lang?: string;
}
interface StructuredContentData {
// "description": "Generic data attributes that should be added to the element.",
[key: string]: string;
}
interface StructuredContentStyle {
fontStyle?: "normal" | "italic";
// "default": "normal"
fontWeight?: "normal" | "bold";
// "default": "normal"
fontSize?: string;
// "default": "medium"
color?: string;
background?: string;
backgroundColor?: string;
textDecorationLine?:
| "none"
| "underline"
| "overline"
| "line-through"
| StructuredContentStyle_textDecorationLine[];
// "default": "none"
textDecorationStyle?: "solid" | "double" | "dotted" | "dashed" | "wavy";
// "default": "solid"
textDecorationColor?: string;
borderColor?: string;
borderStyle?: string;
borderRadius?: string;
borderWidth?: string;
clipPath?: string;
verticalAlign?:
| "baseline"
| "sub"
| "super"
| "text-top"
| "text-bottom"
| "middle"
| "top"
| "bottom";
// "default": "baseline"
textAlign?:
| "start"
| "end"
| "left"
| "right"
| "center"
| "justify"
| "justify-all"
| "match-parent";
// "default": "start"
textEmphasis?: string;
textShadow?: string;
margin?: string;
marginTop?: number | string;
// "default": 0
marginLeft?: number | string;
// "default": 0
marginRight?: number | string;
// "default": 0
marginBottom?: number | string;
// "default": 0
padding?: string;
paddingTop?: string;
paddingLeft?: string;
paddingRight?: string;
paddingBottom?: string;
wordBreak?: "normal" | "break-all" | "keep-all";
// "default": "normal"
whiteSpace?: string;
// "default": "normal"
cursor?: string;
// "default": "auto"
listStyleType?: string;
// "default": "disc"
}
type StructuredContentStyle_textDecorationLine =
| "underline"
| "overline"
| "line-through";
// "default": "none"