import { existsSync, readFileSync, writeFileSync } from 'fs'; import axios, { AxiosInstance } from 'axios'; import yaml from 'js-yaml'; import { soundTag } from './anki'; import { AnkiConnect, IAnkiConnectActions } from './ankiconnect'; import { ILevelLabel, ILevelMap } from './level'; import { logger } from './logger'; export class WaniKani { $axios: AxiosInstance; static files = { wanikani: 'cache/wanikani.json', beyond: 'assets/beyond.yaml', }; constructor(public apiKey = process.env['WANIKANI_API_KEY']) { this.$axios = axios.create({ baseURL: 'https://api.wanikani.com/v2/', headers: { Authorization: apiKey ? `Bearer ${apiKey}` : '', }, }); } static getLevelLabel(level: number): ILevelLabel | null { if (level < 11) { return { range: '01-10', ja: '快', en: 'PLEASANT', }; } else if (level < 21) { return { range: '11-20', ja: '苦', en: 'PAINFUL', }; } else if (level < 31) { return { range: '21-30', ja: '死', en: 'DEATH', }; } else if (level < 41) { return { range: '31-40', ja: '地獄', en: 'HELL', }; } else if (level < 51) { return { range: '41-50', ja: '天国', en: 'PARADISE', }; } else if (level < 61) { return { range: '51-60', ja: '現実', en: 'REALITY', }; } return null; } async subjects({ force, }: { force?: boolean; } = {}) { const subjects = { data: [] as ISubject[], filename: WaniKani.files.wanikani, load() { this.data = existsSync(this.filename) ? JSON.parse(readFileSync(this.filename, 'utf-8')) : []; return this.data; }, dump(d: ISubject[]) { this.data = d; this.finalize(); }, finalize() { writeFileSync(this.filename, JSON.stringify(this.data)); }, }; let data = subjects.load(); if (data.length && !force) { return data; } data = []; let nextURL = '/subjects'; while (nextURL) { const r = await this.$axios .get<{ pages: { next_url?: string; }; data: ISubject[]; }>(nextURL) .then((r) => r.data); data.push(...r.data); logger('WaniKani API:', r.pages.next_url); nextURL = r.pages.next_url || ''; } subjects.dump(data); return data; } async sortByLevels( deck: string, fields: { ja: string }, opts: { /** * Whether to use Level > 60, i.e. outside WaniKani * * @link https://community.wanikani.com/t/fake-levels-61-70-or-%E7%84%A1%E9%99%90-infinity/16399 */ useBeyond?: boolean; } = {}, ) { const anki = new AnkiConnect(); const deckConfig = await anki.api('getDeckConfig', { deck, }); const deckQuery = `deck:${deck} -deck:${deck}::*`; const wkLevelMap: ILevelMap = {}; const wkKanji = await this.subjects().then( (vs) => vs.filter((v) => v.object === 'kanji') as IKanji[], ); for (const k of wkKanji) { const { level, characters } = k.data; const label = WaniKani.getLevelLabel(level); if (!label) throw new Error(`Invalid level: ${level}`); const category = `${label.range}: ${ label.ja } ${label.en.toLocaleUpperCase()}`; const catMap = wkLevelMap[category] || {}; const levelString = level.toString().padStart(2, '0'); const ks = catMap[levelString] || []; ks.push(characters); catMap[levelString] = ks; wkLevelMap[category] = catMap; } const beyond = opts.useBeyond ? (yaml.load(readFileSync(WaniKani.files.beyond, 'utf-8')) as ILevelMap) : {}; const kanjiLevels = new Map< string, { level: number; deckName: string; } >(); const setKanjiToLevel = (lvMap: ILevelMap) => { Object.entries(lvMap).map(([cat, map]) => { Object.entries(map).map(([levelString, ks]) => { const level = Number(levelString); ks.map((k) => { let prev = kanjiLevels.get(k); if (!prev || prev.level > level) { prev = { level, deckName: cat + '::' + levelString, }; kanjiLevels.set(k, prev); } }); }); }); }; setKanjiToLevel(wkLevelMap); setKanjiToLevel(beyond); await anki .api('findCards', { query: deckQuery, }) .then((cards) => anki.api('cardsInfo', { cards })) .then(async (rs) => { const modelToSubdecks = new Map< string, { [subdeck: string]: number[]; } >(); rs.map((r) => { const vs = modelToSubdecks.get(r.modelName) || {}; const { value } = r.fields[fields.ja] || {}; if (value) { // Get the subdeck names from Kanji let subdeck = ''; let level = 0; Array.from(value).map((k) => { const m = kanjiLevels.get(k); if (m && m.level > level) { level = m.level; subdeck = m.deckName; } }); if (subdeck) { const cardIds = vs[subdeck] || []; cardIds.push(r.cardId); vs[subdeck] = cardIds; modelToSubdecks.set(r.modelName, vs); } } }); return Promise.all( Array.from(modelToSubdecks).map(([modelName, subdecks]) => anki.api('modelTemplates', { modelName }).then((templates) => Object.entries(subdecks).map(([subdeck, cardIds]) => ({ modelName, templateNames: Object.keys(templates), subdeck, cardIdsSet: new Set(cardIds), })), ), ), ); }) .then(async (rs) => { const remaining: { [templateName: string]: Set; } = {}; const cardsByTemplateName = { data: new Map(), async get(templateName: string) { let v = this.data.get(templateName); if (!v) { v = await anki.api('findCards', { query: `${deckQuery} card:${templateName}`, }); remaining[templateName] = new Set(v); } this.data.set(templateName, v); return v; }, }; const toChangeDeck: IAnkiConnectActions['changeDeck']['params'][] = []; for (const { templateName, subdeck, cardIdsSet } of rs .flat() .flatMap((r) => r.templateNames.map((templateName) => ({ ...r, templateName, })), )) { const cards = await cardsByTemplateName.get(templateName); toChangeDeck.push({ cards: cards.filter((c) => { const r = remaining[templateName]; if (r) { r.delete(c); } return cardIdsSet.has(c); }), deck: `${deck}::${subdeck}::${templateName}`, }); } Object.entries(remaining).map(([templateName, cards]) => { if (cards.size) { toChangeDeck.push({ cards: [...cards], deck: `${deck}::独習::${templateName}`, }); } }); if (toChangeDeck.length) { const batchSize = 20; for (let i = 0; i < toChangeDeck.length; i += batchSize) { await Promise.all( toChangeDeck .slice(i, i + batchSize) .map((params) => anki.api('changeDeck', params)), ); } await anki.api('setDeckConfigId', { decks: toChangeDeck.map((r) => r.deck), configId: deckConfig.id, }); } }); } async populateSound( query: string, fields: { ja: string; audio: string; }, opts: { mode?: { online?: boolean; }; } = {}, ) { const subjects = await this.subjects(); const vocabularies = subjects.filter( (s) => s.object === 'vocabulary', ) as IVocabulary[]; const audioMap = new Map< string, { url: string; filename: string; } >(); vocabularies.map((v) => { if (audioMap.has(v.data.characters)) return; const audio = v.data.pronunciation_audios[0]; if (!audio) return; audioMap.set(v.data.characters, { url: audio.url, filename: `wanikani_${v.data.characters}_${audio.metadata.source_id}${ audio.content_type === 'audio/ogg' ? '.ogg' : '.mp3' }`, }); }); if (!audioMap.size) return; query += ` -${fields.ja}: ${fields.audio}:`; const anki = new AnkiConnect(); anki .api('findNotes', { query, }) .then((notes) => anki.api('notesInfo', { notes })) .then(async (notes) => { const notesToUpdate: IAnkiConnectActions['updateNoteFields']['params']['note'][] = []; for (const n of notes) { const { value: ja } = n.fields[fields.ja] || {}; if (ja) { // const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '') const cleanJa = ja; const audio = audioMap.get(cleanJa); if (audio) { notesToUpdate.push({ id: n.noteId, ...(opts.mode?.online ? { fields: { [fields.audio]: soundTag.make(audio.url), }, } : { audio: [ { url: audio.url, filename: audio.filename, fields: [fields.audio], }, ], }), }); } } } if (!notesToUpdate.length) return; while (notesToUpdate.length) { await anki.multi<'updateNoteFields'[]>({ actions: notesToUpdate.splice(0, 100).map((note) => ({ action: 'updateNoteFields', params: { note, }, })), }); } }); } async populateSentence( query: string, fields: { vocabJa: string; sentenceJa: string; sentenceAudio?: string; sentenceEn: string; sentenceCloze?: string; }, opts: { prependEnglishCloze?: boolean; overwrite?: boolean; } = {}, ) { const subjects = await this.subjects(); const vocabularies = subjects.filter( (s) => s.object === 'vocabulary', ) as IVocabulary[]; const sentenceMap = new Map< string, { ja: string; en: string; sentences: { ja: string; en: string; }[]; meaning: string; } >(); vocabularies.map((v) => { if (sentenceMap.has(v.data.characters)) return; const sent = v.data.context_sentences[0]; if (!sent || !sent.ja.trim()) return; sentenceMap.set(v.data.characters, { ...sent, sentences: v.data.context_sentences, meaning: v.data.meanings .sort((m1, m2) => Number(m2.primary) - Number(m1.primary)) .map((m) => m.meaning) .join('; '), }); }); if (!sentenceMap.size) return; query += ` -${fields.vocabJa}:`; if (fields.sentenceAudio) { query += ` ${fields.sentenceAudio}:`; } if (!opts.overwrite) { query += ` -${fields.sentenceJa}: -${fields.sentenceEn}:`; } const anki = new AnkiConnect(); anki .api('findNotes', { query, }) .then((notes) => anki.api('notesInfo', { notes })) .then(async (notes) => { const notesToUpdate: IAnkiConnectActions['updateNoteFields']['params']['note'][] = []; for (const n of notes) { const { value: ja } = n.fields[fields.vocabJa] || {}; if (ja) { // const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, ''); const cleanJa = ja; const sent = sentenceMap.get(cleanJa); if (sent) { const fieldUpdate = { [fields.sentenceJa]: sent.ja, [fields.sentenceEn]: sent.en, }; if (fields.sentenceCloze) { let clozeSent = n.fields[fields.sentenceCloze]?.value || ''; if (!clozeSent && sent.sentences.length) { const clozeChar = '__'; let newSent = sent.sentences .map(({ ja, en }) => `${ja}
${en}`) .join('
') .replace(cleanJa, clozeChar); cleanJa.split(/[\p{sc=Hiragana}]+/gu).map((c) => { if (c) { newSent = newSent.replace(c, clozeChar); } }); fieldUpdate[fields.sentenceCloze] = newSent; clozeSent = newSent; } const notJa = '[^\\p{sc=Han}\\p{sc=Katakana}\\p{sc=Hiragana}]+'; if ( opts.prependEnglishCloze && clozeSent && !new RegExp( `(^${notJa}
|

${notJa}$)`, 'u', ).test(clozeSent) ) { fieldUpdate[fields.sentenceCloze] = sent.meaning + '

' + clozeSent; } } notesToUpdate.push({ id: n.noteId, fields: fieldUpdate, }); } } } if (!notesToUpdate.length) return; await anki.multi<'updateNoteFields'[]>({ actions: notesToUpdate.map((note) => ({ action: 'updateNoteFields', params: { note, }, })), }); }); } async addTags( query: string, fields: { ja: string; }, ) { const subjects = await this.subjects(); const vocabularies = subjects.filter( (s) => s.object === 'vocabulary', ) as IVocabulary[]; const vocabMap = new Map< string, { level: number; label: ILevelLabel; } >(); vocabularies.map((v) => { if (vocabMap.has(v.data.characters)) return; const level = v.data.level; const label = WaniKani.getLevelLabel(level); if (!label) throw new Error(`Invalid level: ${level}`); vocabMap.set(v.data.characters, { level, label, }); }); if (!vocabMap.size) return; query += ` -tag:wanikani -${fields.ja}:`; const anki = new AnkiConnect(); anki .api('findNotes', { query, }) .then((notes) => anki.api('notesInfo', { notes })) .then(async (notes) => { const notesToUpdate: { id: SubjectID; tags: string[]; }[] = []; for (const n of notes) { const { value: ja } = n.fields[fields.ja] || {}; if (ja) { // const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '') const cleanJa = ja; const vocab = vocabMap.get(cleanJa); if (vocab) { notesToUpdate.push({ id: n.noteId, tags: [ 'wanikani', `level-${vocab.level}`, `wanikani-level-${vocab.level}`, `wanikani-level-${vocab.label.range}-${vocab.label.ja}`, `wanikani-level-${ vocab.label.range }-${vocab.label.en.toLocaleLowerCase()}`, ], }); } } } if (!notesToUpdate.length) return; const addTagsMap = new Map< string, { notes: SubjectID[]; tags: string; } >(); notesToUpdate.map(({ id, tags }) => { const identifier = JSON.stringify(tags); const v = addTagsMap.get(identifier) || { notes: [], tags: tags.join(' '), }; v.notes.push(id); addTagsMap.set(identifier, v); }); await anki.multi<'addTags'[]>({ actions: Array.from(addTagsMap.values()).map((params) => ({ action: 'addTags', params, })), }); }); } } type WaniKaniDate = string; type Integer = number; type Level = Integer; type SubjectID = Integer; interface IMeaning { meaning: string; primary: boolean; } interface ISubjectBase< T extends 'radical' | 'kanji' | 'vocabulary', Data extends {}, > { id: SubjectID; object: T; url: string; data_updated_at: WaniKaniDate; data: { auxillary_meanings: IMeaning[]; characters: string; created_at: WaniKaniDate; document_url: string; hidden_at: WaniKaniDate | null; lesson_position: Integer; level: Level; meaning_mnemonic: string; meanings: IMeaning[]; slug: string; spaced_repetition_system_id: Integer; } & Data; } export type IRadical = ISubjectBase< 'radical', { character_images: ({ url: string; } & ( | { metadata: { inline_styles?: boolean; dimensions?: string; }; content_type: 'image/svg+xml'; } | { metadata: { color: string; dimensions: string; style_name: string; }; content_type: 'image/png'; } ))[]; amalgamation_subject_ids: SubjectID[]; } >; export type IKanji = ISubjectBase< 'kanji', { readings: { reading: string; primary: boolean; accepted_answer: boolean; type: 'kunyomi' | 'onyomi' | 'nanori'; }[]; component_subject_ids: SubjectID[]; visually_similar_subject_ids: SubjectID[]; } >; export type IVocabulary = ISubjectBase< 'vocabulary', { component_subject_ids: SubjectID[]; context_sentences: { en: string; ja: string; }[]; part_of_speech: string[]; pronunciation_audios: { url: string; metadata: { gender: 'male' | 'female'; source_id: Integer; pronunciation: string; voice_actor_id: Integer; voice_actor_name: string; voice_description: string; }; content_type: 'audio/mpeg' | 'audio/ogg'; }[]; readings: { accepted_answer: boolean; primary: boolean; reading: string; }[]; reading_mnemonic: string; } >; export type ISubject = IRadical | IKanji | IVocabulary;