wk-extra/src/wanikani.ts

import { existsSync, readFileSync, writeFileSync } from 'fs';

import axios, { AxiosInstance } from 'axios';
import yaml from 'js-yaml';

import { soundTag } from './anki';
import { AnkiConnect, IAnkiConnectActions } from './ankiconnect';
import { ILevelLabel, ILevelMap } from './level';
import { logger } from './logger';

export class WaniKani {
  $axios: AxiosInstance;

  static files = {
    wanikani: 'cache/wanikani.json',
    beyond: 'assets/beyond.yaml',
  };

  constructor(public apiKey = process.env['WANIKANI_API_KEY']) {
    this.$axios = axios.create({
      baseURL: 'https://api.wanikani.com/v2/',
      headers: {
        Authorization: apiKey ? `Bearer ${apiKey}` : '',
      },
    });
  }

  static getLevelLabel(level: number): ILevelLabel | null {
    if (level < 11) {
      return {
        range: '01-10',
        ja: '快',
        en: 'PLEASANT',
      };
    } else if (level < 21) {
      return {
        range: '11-20',
        ja: '苦',
        en: 'PAINFUL',
      };
    } else if (level < 31) {
      return {
        range: '21-30',
        ja: '死',
        en: 'DEATH',
      };
    } else if (level < 41) {
      return {
        range: '31-40',
        ja: '地獄',
        en: 'HELL',
      };
    } else if (level < 51) {
      return {
        range: '41-50',
        ja: '天国',
        en: 'PARADISE',
      };
    } else if (level < 61) {
      return {
        range: '51-60',
        ja: '現実',
        en: 'REALITY',
      };
    }

    return null;
  }

  async subjects({
    force,
  }: {
    force?: boolean;
  } = {}) {
    const subjects = {
      data: [] as ISubject[],
      filename: WaniKani.files.wanikani,
      load() {
        this.data = existsSync(this.filename)
          ? JSON.parse(readFileSync(this.filename, 'utf-8'))
          : [];
        return this.data;
      },
      dump(d: ISubject[]) {
        this.data = d;
        this.finalize();
      },
      finalize() {
        writeFileSync(this.filename, JSON.stringify(this.data));
      },
    };

    let data = subjects.load();
    if (data.length && !force) {
      return data;
    }
    data = [];

    let nextURL = '/subjects';
    while (nextURL) {
      const r = await this.$axios
        .get<{
          pages: {
            next_url?: string;
          };
          data: ISubject[];
        }>(nextURL)
        .then((r) => r.data);
      data.push(...r.data);
      logger('WaniKani API:', r.pages.next_url);
      nextURL = r.pages.next_url || '';
    }

    subjects.dump(data);
    return data;
  }

  async sortByLevels(
    deck: string,
    fields: { ja: string },
    opts: {
      /**
       * Whether to use Level > 60, i.e. outside WaniKani
       *
       * @link https://community.wanikani.com/t/fake-levels-61-70-or-%E7%84%A1%E9%99%90-infinity/16399
       */
      useBeyond?: boolean;
    } = {},
  ) {
    const anki = new AnkiConnect();

    const deckConfig = await anki.api('getDeckConfig', {
      deck,
    });

    const deckQuery = `deck:${deck} -deck:${deck}::*`;

    const wkLevelMap: ILevelMap = {};
    const wkKanji = await this.subjects().then(
      (vs) => vs.filter((v) => v.object === 'kanji') as IKanji[],
    );

    for (const k of wkKanji) {
      const { level, characters } = k.data;
      const label = WaniKani.getLevelLabel(level);
      if (!label) throw new Error(`Invalid level: ${level}`);

      const category = `${label.range}: ${
        label.ja
      } ${label.en.toLocaleUpperCase()}`;

      const catMap = wkLevelMap[category] || {};
      const levelString = level.toString().padStart(2, '0');

      const ks = catMap[levelString] || [];
      ks.push(characters);
      catMap[levelString] = ks;

      wkLevelMap[category] = catMap;
    }

    const beyond = opts.useBeyond
      ? (yaml.load(readFileSync(WaniKani.files.beyond, 'utf-8')) as ILevelMap)
      : {};

    const kanjiLevels = new Map<
      string,
      {
        level: number;
        deckName: string;
      }
    >();

    const setKanjiToLevel = (lvMap: ILevelMap) => {
      Object.entries(lvMap).map(([cat, map]) => {
        Object.entries(map).map(([levelString, ks]) => {
          const level = Number(levelString);
          ks.map((k) => {
            let prev = kanjiLevels.get(k);
            if (!prev || prev.level > level) {
              prev = {
                level,
                deckName: cat + '::' + levelString,
              };
              kanjiLevels.set(k, prev);
            }
          });
        });
      });
    };
    setKanjiToLevel(wkLevelMap);
    setKanjiToLevel(beyond);

    await anki
      .api('findCards', {
        query: deckQuery,
      })
      .then((cards) => anki.api('cardsInfo', { cards }))
      .then(async (rs) => {
        const modelToSubdecks = new Map<
          string,
          {
            [subdeck: string]: number[];
          }
        >();
        rs.map((r) => {
          const vs = modelToSubdecks.get(r.modelName) || {};
          const { value } = r.fields[fields.ja] || {};
          if (value) {
            // Get the subdeck names from Kanji
            let subdeck = '';
            let level = 0;

            Array.from(value).map((k) => {
              const m = kanjiLevels.get(k);
              if (m && m.level > level) {
                level = m.level;
                subdeck = m.deckName;
              }
            });

            if (subdeck) {
              const cardIds = vs[subdeck] || [];
              cardIds.push(r.cardId);
              vs[subdeck] = cardIds;

              modelToSubdecks.set(r.modelName, vs);
            }
          }
        });

        return Promise.all(
          Array.from(modelToSubdecks).map(([modelName, subdecks]) =>
            anki.api('modelTemplates', { modelName }).then((templates) =>
              Object.entries(subdecks).map(([subdeck, cardIds]) => ({
                modelName,
                templateNames: Object.keys(templates),
                subdeck,
                cardIdsSet: new Set(cardIds),
              })),
            ),
          ),
        );
      })
      .then(async (rs) => {
        const remaining: {
          [templateName: string]: Set<number>;
        } = {};

        const cardsByTemplateName = {
          data: new Map<string, number[]>(),
          async get(templateName: string) {
            let v = this.data.get(templateName);
            if (!v) {
              v = await anki.api('findCards', {
                query: `${deckQuery} card:${templateName}`,
              });
              remaining[templateName] = new Set(v);
            }
            this.data.set(templateName, v);
            return v;
          },
        };

        const toChangeDeck: IAnkiConnectActions['changeDeck']['params'][] = [];

        for (const { templateName, subdeck, cardIdsSet } of rs
          .flat()
          .flatMap((r) =>
            r.templateNames.map((templateName) => ({
              ...r,
              templateName,
            })),
          )) {
          const cards = await cardsByTemplateName.get(templateName);

          toChangeDeck.push({
            cards: cards.filter((c) => {
              const r = remaining[templateName];
              if (r) {
                r.delete(c);
              }
              return cardIdsSet.has(c);
            }),
            deck: `${deck}::${subdeck}::${templateName}`,
          });
        }

        Object.entries(remaining).map(([templateName, cards]) => {
          if (cards.size) {
            toChangeDeck.push({
              cards: [...cards],
              deck: `${deck}::独習::${templateName}`,
            });
          }
        });

        if (toChangeDeck.length) {
          const batchSize = 20;
          for (let i = 0; i < toChangeDeck.length; i += batchSize) {
            await Promise.all(
              toChangeDeck
                .slice(i, i + batchSize)
                .map((params) => anki.api('changeDeck', params)),
            );
          }

          await anki.api('setDeckConfigId', {
            decks: toChangeDeck.map((r) => r.deck),
            configId: deckConfig.id,
          });
        }
      });
  }

  async populateSound(
    query: string,
    fields: {
      ja: string;
      audio: string;
    },
    opts: {
      mode?: {
        online?: boolean;
      };
    } = {},
  ) {
    const subjects = await this.subjects();

    const vocabularies = subjects.filter(
      (s) => s.object === 'vocabulary',
    ) as IVocabulary[];
    const audioMap = new Map<
      string,
      {
        url: string;
        filename: string;
      }
    >();
    vocabularies.map((v) => {
      if (audioMap.has(v.data.characters)) return;
      const audio = v.data.pronunciation_audios[0];
      if (!audio) return;
      audioMap.set(v.data.characters, {
        url: audio.url,
        filename: `wanikani_${v.data.characters}_${audio.metadata.source_id}${
          audio.content_type === 'audio/ogg' ? '.ogg' : '.mp3'
        }`,
      });
    });
    if (!audioMap.size) return;

    query += ` -${fields.ja}: ${fields.audio}:`;

    const anki = new AnkiConnect();
    anki
      .api('findNotes', {
        query,
      })
      .then((notes) => anki.api('notesInfo', { notes }))
      .then(async (notes) => {
        const notesToUpdate: IAnkiConnectActions['updateNoteFields']['params']['note'][] =
          [];

        for (const n of notes) {
          const { value: ja } = n.fields[fields.ja] || {};
          if (ja) {
            // const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '')
            const cleanJa = ja;
            const audio = audioMap.get(cleanJa);
            if (audio) {
              notesToUpdate.push({
                id: n.noteId,
                ...(opts.mode?.online
                  ? {
                      fields: {
                        [fields.audio]: soundTag.make(audio.url),
                      },
                    }
                  : {
                      audio: [
                        {
                          url: audio.url,
                          filename: audio.filename,
                          fields: [fields.audio],
                        },
                      ],
                    }),
              });
            }
          }
        }

        if (!notesToUpdate.length) return;

        while (notesToUpdate.length) {
          await anki.multi<'updateNoteFields'[]>({
            actions: notesToUpdate.splice(0, 100).map((note) => ({
              action: 'updateNoteFields',
              params: {
                note,
              },
            })),
          });
        }
      });
  }

  async populateSentence(
    query: string,
    fields: {
      vocabJa: string;
      sentenceJa: string;
      sentenceAudio?: string;
      sentenceEn: string;
      sentenceCloze?: string;
    },
    opts: {
      prependEnglishCloze?: boolean;
      overwrite?: boolean;
    } = {},
  ) {
    const subjects = await this.subjects();

    const vocabularies = subjects.filter(
      (s) => s.object === 'vocabulary',
    ) as IVocabulary[];
    const sentenceMap = new Map<
      string,
      {
        ja: string;
        en: string;
        sentences: {
          ja: string;
          en: string;
        }[];
        meaning: string;
      }
    >();
    vocabularies.map((v) => {
      if (sentenceMap.has(v.data.characters)) return;
      const sent = v.data.context_sentences[0];
      if (!sent || !sent.ja.trim()) return;
      sentenceMap.set(v.data.characters, {
        ...sent,
        sentences: v.data.context_sentences,
        meaning: v.data.meanings
          .sort((m1, m2) => Number(m2.primary) - Number(m1.primary))
          .map((m) => m.meaning)
          .join('; '),
      });
    });
    if (!sentenceMap.size) return;

    query += ` -${fields.vocabJa}:`;
    if (fields.sentenceAudio) {
      query += ` ${fields.sentenceAudio}:`;
    }
    if (!opts.overwrite) {
      query += ` -${fields.sentenceJa}: -${fields.sentenceEn}:`;
    }

    const anki = new AnkiConnect();
    anki
      .api('findNotes', {
        query,
      })
      .then((notes) => anki.api('notesInfo', { notes }))
      .then(async (notes) => {
        const notesToUpdate: IAnkiConnectActions['updateNoteFields']['params']['note'][] =
          [];

        for (const n of notes) {
          const { value: ja } = n.fields[fields.vocabJa] || {};
          if (ja) {
            // const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '');
            const cleanJa = ja;
            const sent = sentenceMap.get(cleanJa);
            if (sent) {
              const fieldUpdate = {
                [fields.sentenceJa]: sent.ja,
                [fields.sentenceEn]: sent.en,
              };

              if (fields.sentenceCloze) {
                let clozeSent = n.fields[fields.sentenceCloze]?.value || '';

                if (!clozeSent && sent.sentences.length) {
                  const clozeChar = '__';
                  let newSent = sent.sentences
                    .map(({ ja, en }) => `${ja}<br/>${en}`)
                    .join('<br/>')
                    .replace(cleanJa, clozeChar);

                  cleanJa.split(/[\p{sc=Hiragana}]+/gu).map((c) => {
                    if (c) {
                      newSent = newSent.replace(c, clozeChar);
                    }
                  });

                  fieldUpdate[fields.sentenceCloze] = newSent;
                  clozeSent = newSent;
                }

                const notJa = '[^\\p{sc=Han}\\p{sc=Katakana}\\p{sc=Hiragana}]+';

                if (
                  opts.prependEnglishCloze &&
                  clozeSent &&
                  !new RegExp(
                    `(^${notJa}<br/?>|<br/?><br/?>${notJa}$)`,
                    'u',
                  ).test(clozeSent)
                ) {
                  fieldUpdate[fields.sentenceCloze] =
                    sent.meaning + '<br><br>' + clozeSent;
                }
              }

              notesToUpdate.push({
                id: n.noteId,
                fields: fieldUpdate,
              });
            }
          }
        }

        if (!notesToUpdate.length) return;

        await anki.multi<'updateNoteFields'[]>({
          actions: notesToUpdate.map((note) => ({
            action: 'updateNoteFields',
            params: {
              note,
            },
          })),
        });
      });
  }

  async addTags(
    query: string,
    fields: {
      ja: string;
    },
  ) {
    const subjects = await this.subjects();

    const vocabularies = subjects.filter(
      (s) => s.object === 'vocabulary',
    ) as IVocabulary[];
    const vocabMap = new Map<
      string,
      {
        level: number;
        label: ILevelLabel;
      }
    >();
    vocabularies.map((v) => {
      if (vocabMap.has(v.data.characters)) return;
      const level = v.data.level;
      const label = WaniKani.getLevelLabel(level);
      if (!label) throw new Error(`Invalid level: ${level}`);

      vocabMap.set(v.data.characters, {
        level,
        label,
      });
    });
    if (!vocabMap.size) return;

    query += ` -tag:wanikani -${fields.ja}:`;

    const anki = new AnkiConnect();
    anki
      .api('findNotes', {
        query,
      })
      .then((notes) => anki.api('notesInfo', { notes }))
      .then(async (notes) => {
        const notesToUpdate: {
          id: SubjectID;
          tags: string[];
        }[] = [];

        for (const n of notes) {
          const { value: ja } = n.fields[fields.ja] || {};
          if (ja) {
            // const cleanJa = ja.replace(/\[.+?\]/g, '').replace(/ /g, '')
            const cleanJa = ja;
            const vocab = vocabMap.get(cleanJa);
            if (vocab) {
              notesToUpdate.push({
                id: n.noteId,
                tags: [
                  'wanikani',
                  `level-${vocab.level}`,
                  `wanikani-level-${vocab.level}`,
                  `wanikani-level-${vocab.label.range}-${vocab.label.ja}`,
                  `wanikani-level-${
                    vocab.label.range
                  }-${vocab.label.en.toLocaleLowerCase()}`,
                ],
              });
            }
          }
        }

        if (!notesToUpdate.length) return;

        const addTagsMap = new Map<
          string,
          {
            notes: SubjectID[];
            tags: string;
          }
        >();

        notesToUpdate.map(({ id, tags }) => {
          const identifier = JSON.stringify(tags);
          const v = addTagsMap.get(identifier) || {
            notes: [],
            tags: tags.join(' '),
          };
          v.notes.push(id);
          addTagsMap.set(identifier, v);
        });

        await anki.multi<'addTags'[]>({
          actions: Array.from(addTagsMap.values()).map((params) => ({
            action: 'addTags',
            params,
          })),
        });
      });
  }
}

type WaniKaniDate = string;
type Integer = number;
type Level = Integer;
type SubjectID = Integer;

interface IMeaning {
  meaning: string;
  primary: boolean;
}

interface ISubjectBase<
  T extends 'radical' | 'kanji' | 'vocabulary',
  Data extends {},
> {
  id: SubjectID;
  object: T;
  url: string;
  data_updated_at: WaniKaniDate;
  data: {
    auxillary_meanings: IMeaning[];
    characters: string;
    created_at: WaniKaniDate;
    document_url: string;
    hidden_at: WaniKaniDate | null;
    lesson_position: Integer;
    level: Level;
    meaning_mnemonic: string;
    meanings: IMeaning[];
    slug: string;
    spaced_repetition_system_id: Integer;
  } & Data;
}

export type IRadical = ISubjectBase<
  'radical',
  {
    character_images: ({
      url: string;
    } & (
      | {
          metadata: {
            inline_styles?: boolean;
            dimensions?: string;
          };
          content_type: 'image/svg+xml';
        }
      | {
          metadata: {
            color: string;
            dimensions: string;
            style_name: string;
          };
          content_type: 'image/png';
        }
    ))[];
    amalgamation_subject_ids: SubjectID[];
  }
>;

export type IKanji = ISubjectBase<
  'kanji',
  {
    readings: {
      reading: string;
      primary: boolean;
      accepted_answer: boolean;
      type: 'kunyomi' | 'onyomi' | 'nanori';
    }[];
    component_subject_ids: SubjectID[];
    visually_similar_subject_ids: SubjectID[];
  }
>;

export type IVocabulary = ISubjectBase<
  'vocabulary',
  {
    component_subject_ids: SubjectID[];
    context_sentences: {
      en: string;
      ja: string;
    }[];
    part_of_speech: string[];
    pronunciation_audios: {
      url: string;
      metadata: {
        gender: 'male' | 'female';
        source_id: Integer;
        pronunciation: string;
        voice_actor_id: Integer;
        voice_actor_name: string;
        voice_description: string;
      };
      content_type: 'audio/mpeg' | 'audio/ogg';
    }[];
    readings: {
      accepted_answer: boolean;
      primary: boolean;
      reading: string;
    }[];
    reading_mnemonic: string;
  }
>;

export type ISubject = IRadical | IKanji | IVocabulary;