You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
742 lines
19 KiB
742 lines
19 KiB
import { existsSync, readFileSync, writeFileSync } from 'fs';
|
|
|
|
import axios, { AxiosInstance } from 'axios';
|
|
import yaml from 'js-yaml';
|
|
|
|
import { soundTag } from './anki';
|
|
import { AnkiConnect, IAnkiConnectActions } from './ankiconnect';
|
|
import { ILevelLabel, ILevelMap } from './level';
|
|
import { logger } from './logger';
|
|
|
|
export class WaniKani {
|
|
$axios: AxiosInstance;
|
|
|
|
static files = {
|
|
wanikani: 'cache/wanikani.json',
|
|
beyond: 'assets/beyond.yaml',
|
|
};
|
|
|
|
constructor(public apiKey = process.env['WANIKANI_API_KEY']) {
|
|
this.$axios = axios.create({
|
|
baseURL: 'https://api.wanikani.com/v2/',
|
|
headers: {
|
|
Authorization: apiKey ? `Bearer ${apiKey}` : '',
|
|
},
|
|
});
|
|
}
|
|
|
|
static getLevelLabel(level: number): ILevelLabel | null {
|
|
if (level < 11) {
|
|
return {
|
|
range: '01-10',
|
|
ja: '快',
|
|
en: 'PLEASANT',
|
|
};
|
|
} else if (level < 21) {
|
|
return {
|
|
range: '11-20',
|
|
ja: '苦',
|
|
en: 'PAINFUL',
|
|
};
|
|
} else if (level < 31) {
|
|
return {
|
|
range: '21-30',
|
|
ja: '死',
|
|
en: 'DEATH',
|
|
};
|
|
} else if (level < 41) {
|
|
return {
|
|
range: '31-40',
|
|
ja: '地獄',
|
|
en: 'HELL',
|
|
};
|
|
} else if (level < 51) {
|
|
return {
|
|
range: '41-50',
|
|
ja: '天国',
|
|
en: 'PARADISE',
|
|
};
|
|
} else if (level < 61) {
|
|
return {
|
|
range: '51-60',
|
|
ja: '現実',
|
|
en: 'REALITY',
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async subjects({
|
|
force,
|
|
}: {
|
|
force?: boolean;
|
|
} = {}) {
|
|
const subjects = {
|
|
data: [] as ISubject[],
|
|
filename: WaniKani.files.wanikani,
|
|
load() {
|
|
this.data = existsSync(this.filename)
|
|
? JSON.parse(readFileSync(this.filename, 'utf-8'))
|
|
: [];
|
|
return this.data;
|
|
},
|
|
dump(d: ISubject[]) {
|
|
this.data = d;
|
|
this.finalize();
|
|
},
|
|
finalize() {
|
|
writeFileSync(this.filename, JSON.stringify(this.data));
|
|
},
|
|
};
|
|
|
|
let data = subjects.load();
|
|
if (data.length && !force) {
|
|
return data;
|
|
}
|
|
data = [];
|
|
|
|
let nextURL = '/subjects';
|
|
while (nextURL) {
|
|
const r = await this.$axios
|
|
.get<{
|
|
pages: {
|
|
next_url?: string;
|
|
};
|
|
data: ISubject[];
|
|
}>(nextURL)
|
|
.then((r) => r.data);
|
|
data.push(...r.data);
|
|
logger('WaniKani API:', r.pages.next_url);
|
|
nextURL = r.pages.next_url || '';
|
|
}
|
|
|
|
subjects.dump(data);
|
|
return data;
|
|
}
|
|
|
|
async sortByLevels(
|
|
deck: string,
|
|
fields: { ja: string },
|
|
opts: {
|
|
/**
|
|
* Whether to use Level > 60, i.e. outside WaniKani
|
|
*
|
|
* @link https://community.wanikani.com/t/fake-levels-61-70-or-%E7%84%A1%E9%99%90-infinity/16399
|
|
*/
|
|
useBeyond?: boolean;
|
|
} = {},
|
|
) {
|
|
const anki = new AnkiConnect();
|
|
|
|
const deckConfig = await anki.api('getDeckConfig', {
|
|
deck,
|
|
});
|
|
|
|
const deckQuery = `deck:${deck} -deck:${deck}::*`;
|
|
|
|
const wkLevelMap: ILevelMap = {};
|
|
const wkKanji = await this.subjects().then(
|
|
(vs) => vs.filter((v) => v.object === 'kanji') as IKanji[],
|
|
);
|
|
|
|
for (const k of wkKanji) {
|
|
const { level, characters } = k.data;
|
|
const label = WaniKani.getLevelLabel(level);
|
|
if (!label) throw new Error(`Invalid level: ${level}`);
|
|
|
|
const category = `${label.range}: ${
|
|
label.ja
|
|
} ${label.en.toLocaleUpperCase()}`;
|
|
|
|
const catMap = wkLevelMap[category] || {};
|
|
const levelString = level.toString().padStart(2, '0');
|
|
|
|
const ks = catMap[levelString] || [];
|
|
ks.push(characters);
|
|
catMap[levelString] = ks;
|
|
|
|
wkLevelMap[category] = catMap;
|
|
}
|
|
|
|
const beyond = opts.useBeyond
|
|
? (yaml.load(readFileSync(WaniKani.files.beyond, 'utf-8')) as ILevelMap)
|
|
: {};
|
|
|
|
const kanjiLevels = new Map<
|
|
string,
|
|
{
|
|
level: number;
|
|
deckName: string;
|
|
}
|
|
>();
|
|
|
|
const setKanjiToLevel = (lvMap: ILevelMap) => {
|
|
Object.entries(lvMap).map(([cat, map]) => {
|
|
Object.entries(map).map(([levelString, ks]) => {
|
|
const level = Number(levelString);
|
|
ks.map((k) => {
|
|
let prev = kanjiLevels.get(k);
|
|
if (!prev || prev.level > level) {
|
|
prev = {
|
|
level,
|
|
deckName: cat + '::' + levelString,
|
|
};
|
|
kanjiLevels.set(k, prev);
|
|
}
|
|
});
|
|
});
|
|
});
|
|
};
|
|
setKanjiToLevel(wkLevelMap);
|
|
setKanjiToLevel(beyond);
|
|
|
|
await anki
|
|
.api('findCards', {
|
|
query: deckQuery,
|
|
})
|
|
.then((cards) => anki.api('cardsInfo', { cards }))
|
|
.then(async (rs) => {
|
|
const modelToSubdecks = new Map<
|
|
string,
|
|
{
|
|
[subdeck: string]: number[];
|
|
}
|
|
>();
|
|
rs.map((r) => {
|
|
const vs = modelToSubdecks.get(r.modelName) || {};
|
|
const { value } = r.fields[fields.ja] || {};
|
|
if (value) {
|
|
// Get the subdeck names from Kanji
|
|
let subdeck = '';
|
|
let level = 0;
|
|
|
|
Array.from(value).map((k) => {
|
|
const m = kanjiLevels.get(k);
|
|
if (m && m.level > level) {
|
|
level = m.level;
|
|
subdeck = m.deckName;
|
|
}
|
|
});
|
|
|
|
if (subdeck) {
|
|
const cardIds = vs[subdeck] || [];
|
|
cardIds.push(r.cardId);
|
|
vs[subdeck] = cardIds;
|
|
|
|
modelToSubdecks.set(r.modelName, vs);
|
|
}
|
|
}
|
|
});
|
|
|
|
return Promise.all(
|
|
Array.from(modelToSubdecks).map(([modelName, subdecks]) =>
|
|
anki.api('modelTemplates', { modelName }).then((templates) =>
|
|
Object.entries(subdecks).map(([subdeck, cardIds]) => ({
|
|
modelName,
|
|
templateNames: Object.keys(templates),
|
|
subdeck,
|
|
cardIdsSet: new Set(cardIds),
|
|
})),
|
|
),
|
|
),
|
|
);
|
|
})
|
|
.then(async (rs) => {
|
|
const remaining: {
|
|
[templateName: string]: Set<number>;
|
|
} = {};
|
|
|
|
const cardsByTemplateName = {
|
|
data: new Map<string, number[]>(),
|
|
async get(templateName: string) {
|
|
let v = this.data.get(templateName);
|
|
if (!v) {
|
|
v = await anki.api('findCards', {
|
|
query: `${deckQuery} card:${templateName}`,
|
|
});
|
|
remaining[templateName] = new Set(v);
|
|
}
|
|
this.data.set(templateName, v);
|
|
return v;
|
|
},
|
|
};
|
|
|
|
const toChangeDeck: IAnkiConnectActions['changeDeck']['params'][] = [];
|
|
|
|
for (const { templateName, subdeck, cardIdsSet } of rs
|
|
.flat()
|
|
.flatMap((r) =>
|
|
r.templateNames.map((templateName) => ({
|
|
...r,
|
|
templateName,
|
|
})),
|
|
)) {
|
|
const cards = await cardsByTemplateName.get(templateName);
|
|
|
|
toChangeDeck.push({
|
|
cards: cards.filter((c) => {
|
|
const r = remaining[templateName];
|
|
if (r) {
|
|
r.delete(c);
|
|
}
|
|
return cardIdsSet.has(c);
|
|
}),
|
|
deck: `${deck}::${subdeck}::${templateName}`,
|
|
});
|
|
}
|
|
|
|
Object.entries(remaining).map(([templateName, cards]) => {
|
|
if (cards.size) {
|
|
toChangeDeck.push({
|
|
cards: [...cards],
|
|
deck: `${deck}::独習::${templateName}`,
|
|
});
|
|
}
|
|
});
|
|
|
|
if (toChangeDeck.length) {
|
|
const batchSize = 20;
|
|
for (let i = 0; i < toChangeDeck.length; i += batchSize) {
|
|
await Promise.all(
|
|
toChangeDeck
|
|
.slice(i, i + batchSize)
|
|
.map((params) => anki.api('changeDeck', params)),
|
|
);
|
|
}
|
|
|
|
await anki.api('setDeckConfigId', {
|
|
decks: toChangeDeck.map((r) => r.deck),
|
|
configId: deckConfig.id,
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
async populateSound(
|
|
query: string,
|
|
fields: {
|
|
ja: string;
|
|
audio: string;
|
|
},
|
|
opts: {
|
|
mode?: {
|
|
online?: boolean;
|
|
};
|
|
} = {},
|
|
) {
|
|
const subjects = await this.subjects();
|
|
|
|
const vocabularies = subjects.filter(
|
|
(s) => s.object === 'vocabulary',
|
|
) as IVocabulary[];
|
|
const audioMap = new Map<
|
|
string,
|
|
{
|
|
url: string;
|
|
filename: string;
|
|
}
|
|
>();
|
|
vocabularies.map((v) => {
|
|
if (audioMap.has(v.data.characters)) return;
|
|
const audio = v.data.pronunciation_audios[0];
|
|
if (!audio) return;
|
|
audioMap.set(v.data.characters, {
|
|
url: audio.url,
|
|
filename: `wanikani_${v.data.characters}_${audio.metadata.source_id}${
|
|
audio.content_type === 'audio/ogg' ? '.ogg' : '.mp3'
|
|
}`,
|
|
});
|
|
});
|
|
if (!audioMap.size) return;
|
|
|
|
query += ` -${fields.ja}: ${fields.audio}:`;
|
|
|
|
const anki = new AnkiConnect();
|
|
anki
|
|
.api('findNotes', {
|
|
query,
|
|
})
|
|
.then((notes) => anki.api('notesInfo', { notes }))
|
|
.then(async (notes) => {
|
|
const notesToUpdate: IAnkiConnectActions['updateNoteFields']['params']['note'][] =
|
|
[];
|
|
|
|
for (const n of notes) {
|
|
const { value: ja } = n.fields[fields.ja] || {};
|
|
if (ja) {
|
|
// const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '')
|
|
const cleanJa = ja;
|
|
const audio = audioMap.get(cleanJa);
|
|
if (audio) {
|
|
notesToUpdate.push({
|
|
id: n.noteId,
|
|
...(opts.mode?.online
|
|
? {
|
|
fields: {
|
|
[fields.audio]: soundTag.make(audio.url),
|
|
},
|
|
}
|
|
: {
|
|
audio: [
|
|
{
|
|
url: audio.url,
|
|
filename: audio.filename,
|
|
fields: [fields.audio],
|
|
},
|
|
],
|
|
}),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!notesToUpdate.length) return;
|
|
|
|
while (notesToUpdate.length) {
|
|
await anki.multi<'updateNoteFields'[]>({
|
|
actions: notesToUpdate.splice(0, 100).map((note) => ({
|
|
action: 'updateNoteFields',
|
|
params: {
|
|
note,
|
|
},
|
|
})),
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
async populateSentence(
|
|
query: string,
|
|
fields: {
|
|
vocabJa: string;
|
|
sentenceJa: string;
|
|
sentenceAudio?: string;
|
|
sentenceEn: string;
|
|
sentenceCloze?: string;
|
|
},
|
|
opts: {
|
|
prependEnglishCloze?: boolean;
|
|
overwrite?: boolean;
|
|
} = {},
|
|
) {
|
|
const subjects = await this.subjects();
|
|
|
|
const vocabularies = subjects.filter(
|
|
(s) => s.object === 'vocabulary',
|
|
) as IVocabulary[];
|
|
const sentenceMap = new Map<
|
|
string,
|
|
{
|
|
ja: string;
|
|
en: string;
|
|
sentences: {
|
|
ja: string;
|
|
en: string;
|
|
}[];
|
|
meaning: string;
|
|
}
|
|
>();
|
|
vocabularies.map((v) => {
|
|
if (sentenceMap.has(v.data.characters)) return;
|
|
const sent = v.data.context_sentences[0];
|
|
if (!sent || !sent.ja.trim()) return;
|
|
sentenceMap.set(v.data.characters, {
|
|
...sent,
|
|
sentences: v.data.context_sentences,
|
|
meaning: v.data.meanings
|
|
.sort((m1, m2) => Number(m2.primary) - Number(m1.primary))
|
|
.map((m) => m.meaning)
|
|
.join('; '),
|
|
});
|
|
});
|
|
if (!sentenceMap.size) return;
|
|
|
|
query += ` -${fields.vocabJa}:`;
|
|
if (fields.sentenceAudio) {
|
|
query += ` ${fields.sentenceAudio}:`;
|
|
}
|
|
if (!opts.overwrite) {
|
|
query += ` -${fields.sentenceJa}: -${fields.sentenceEn}:`;
|
|
}
|
|
|
|
const anki = new AnkiConnect();
|
|
anki
|
|
.api('findNotes', {
|
|
query,
|
|
})
|
|
.then((notes) => anki.api('notesInfo', { notes }))
|
|
.then(async (notes) => {
|
|
const notesToUpdate: IAnkiConnectActions['updateNoteFields']['params']['note'][] =
|
|
[];
|
|
|
|
for (const n of notes) {
|
|
const { value: ja } = n.fields[fields.vocabJa] || {};
|
|
if (ja) {
|
|
// const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '');
|
|
const cleanJa = ja;
|
|
const sent = sentenceMap.get(cleanJa);
|
|
if (sent) {
|
|
const fieldUpdate = {
|
|
[fields.sentenceJa]: sent.ja,
|
|
[fields.sentenceEn]: sent.en,
|
|
};
|
|
|
|
if (fields.sentenceCloze) {
|
|
let clozeSent = n.fields[fields.sentenceCloze]?.value || '';
|
|
|
|
if (!clozeSent && sent.sentences.length) {
|
|
const clozeChar = '__';
|
|
let newSent = sent.sentences
|
|
.map(({ ja, en }) => `${ja}<br/>${en}`)
|
|
.join('<br/>')
|
|
.replace(cleanJa, clozeChar);
|
|
|
|
cleanJa.split(/[\p{sc=Hiragana}]+/gu).map((c) => {
|
|
if (c) {
|
|
newSent = newSent.replace(c, clozeChar);
|
|
}
|
|
});
|
|
|
|
fieldUpdate[fields.sentenceCloze] = newSent;
|
|
clozeSent = newSent;
|
|
}
|
|
|
|
const notJa = '[^\\p{sc=Han}\\p{sc=Katakana}\\p{sc=Hiragana}]+';
|
|
|
|
if (
|
|
opts.prependEnglishCloze &&
|
|
clozeSent &&
|
|
!new RegExp(
|
|
`(^${notJa}<br/?>|<br/?><br/?>${notJa}$)`,
|
|
'u',
|
|
).test(clozeSent)
|
|
) {
|
|
fieldUpdate[fields.sentenceCloze] =
|
|
sent.meaning + '<br><br>' + clozeSent;
|
|
}
|
|
}
|
|
|
|
notesToUpdate.push({
|
|
id: n.noteId,
|
|
fields: fieldUpdate,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!notesToUpdate.length) return;
|
|
|
|
await anki.multi<'updateNoteFields'[]>({
|
|
actions: notesToUpdate.map((note) => ({
|
|
action: 'updateNoteFields',
|
|
params: {
|
|
note,
|
|
},
|
|
})),
|
|
});
|
|
});
|
|
}
|
|
|
|
async addTags(
|
|
query: string,
|
|
fields: {
|
|
ja: string;
|
|
},
|
|
) {
|
|
const subjects = await this.subjects();
|
|
|
|
const vocabularies = subjects.filter(
|
|
(s) => s.object === 'vocabulary',
|
|
) as IVocabulary[];
|
|
const vocabMap = new Map<
|
|
string,
|
|
{
|
|
level: number;
|
|
label: ILevelLabel;
|
|
}
|
|
>();
|
|
vocabularies.map((v) => {
|
|
if (vocabMap.has(v.data.characters)) return;
|
|
const level = v.data.level;
|
|
const label = WaniKani.getLevelLabel(level);
|
|
if (!label) throw new Error(`Invalid level: ${level}`);
|
|
|
|
vocabMap.set(v.data.characters, {
|
|
level,
|
|
label,
|
|
});
|
|
});
|
|
if (!vocabMap.size) return;
|
|
|
|
query += ` -tag:wanikani -${fields.ja}:`;
|
|
|
|
const anki = new AnkiConnect();
|
|
anki
|
|
.api('findNotes', {
|
|
query,
|
|
})
|
|
.then((notes) => anki.api('notesInfo', { notes }))
|
|
.then(async (notes) => {
|
|
const notesToUpdate: {
|
|
id: SubjectID;
|
|
tags: string[];
|
|
}[] = [];
|
|
|
|
for (const n of notes) {
|
|
const { value: ja } = n.fields[fields.ja] || {};
|
|
if (ja) {
|
|
// const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '')
|
|
const cleanJa = ja;
|
|
const vocab = vocabMap.get(cleanJa);
|
|
if (vocab) {
|
|
notesToUpdate.push({
|
|
id: n.noteId,
|
|
tags: [
|
|
'wanikani',
|
|
`level-${vocab.level}`,
|
|
`wanikani-level-${vocab.level}`,
|
|
`wanikani-level-${vocab.label.range}-${vocab.label.ja}`,
|
|
`wanikani-level-${
|
|
vocab.label.range
|
|
}-${vocab.label.en.toLocaleLowerCase()}`,
|
|
],
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!notesToUpdate.length) return;
|
|
|
|
const addTagsMap = new Map<
|
|
string,
|
|
{
|
|
notes: SubjectID[];
|
|
tags: string;
|
|
}
|
|
>();
|
|
|
|
notesToUpdate.map(({ id, tags }) => {
|
|
const identifier = JSON.stringify(tags);
|
|
const v = addTagsMap.get(identifier) || {
|
|
notes: [],
|
|
tags: tags.join(' '),
|
|
};
|
|
v.notes.push(id);
|
|
addTagsMap.set(identifier, v);
|
|
});
|
|
|
|
await anki.multi<'addTags'[]>({
|
|
actions: Array.from(addTagsMap.values()).map((params) => ({
|
|
action: 'addTags',
|
|
params,
|
|
})),
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
type WaniKaniDate = string;
|
|
type Integer = number;
|
|
type Level = Integer;
|
|
type SubjectID = Integer;
|
|
|
|
interface IMeaning {
|
|
meaning: string;
|
|
primary: boolean;
|
|
}
|
|
|
|
interface ISubjectBase<
|
|
T extends 'radical' | 'kanji' | 'vocabulary',
|
|
Data extends {},
|
|
> {
|
|
id: SubjectID;
|
|
object: T;
|
|
url: string;
|
|
data_updated_at: WaniKaniDate;
|
|
data: {
|
|
auxillary_meanings: IMeaning[];
|
|
characters: string;
|
|
created_at: WaniKaniDate;
|
|
document_url: string;
|
|
hidden_at: WaniKaniDate | null;
|
|
lesson_position: Integer;
|
|
level: Level;
|
|
meaning_mnemonic: string;
|
|
meanings: IMeaning[];
|
|
slug: string;
|
|
spaced_repetition_system_id: Integer;
|
|
} & Data;
|
|
}
|
|
|
|
export type IRadical = ISubjectBase<
|
|
'radical',
|
|
{
|
|
character_images: ({
|
|
url: string;
|
|
} & (
|
|
| {
|
|
metadata: {
|
|
inline_styles?: boolean;
|
|
dimensions?: string;
|
|
};
|
|
content_type: 'image/svg+xml';
|
|
}
|
|
| {
|
|
metadata: {
|
|
color: string;
|
|
dimensions: string;
|
|
style_name: string;
|
|
};
|
|
content_type: 'image/png';
|
|
}
|
|
))[];
|
|
amalgamation_subject_ids: SubjectID[];
|
|
}
|
|
>;
|
|
|
|
export type IKanji = ISubjectBase<
|
|
'kanji',
|
|
{
|
|
readings: {
|
|
reading: string;
|
|
primary: boolean;
|
|
accepted_answer: boolean;
|
|
type: 'kunyomi' | 'onyomi' | 'nanori';
|
|
}[];
|
|
component_subject_ids: SubjectID[];
|
|
visually_similar_subject_ids: SubjectID[];
|
|
}
|
|
>;
|
|
|
|
export type IVocabulary = ISubjectBase<
|
|
'vocabulary',
|
|
{
|
|
component_subject_ids: SubjectID[];
|
|
context_sentences: {
|
|
en: string;
|
|
ja: string;
|
|
}[];
|
|
part_of_speech: string[];
|
|
pronunciation_audios: {
|
|
url: string;
|
|
metadata: {
|
|
gender: 'male' | 'female';
|
|
source_id: Integer;
|
|
pronunciation: string;
|
|
voice_actor_id: Integer;
|
|
voice_actor_name: string;
|
|
voice_description: string;
|
|
};
|
|
content_type: 'audio/mpeg' | 'audio/ogg';
|
|
}[];
|
|
readings: {
|
|
accepted_answer: boolean;
|
|
primary: boolean;
|
|
reading: string;
|
|
}[];
|
|
reading_mnemonic: string;
|
|
}
|
|
>;
|
|
|
|
export type ISubject = IRadical | IKanji | IVocabulary;
|