use async generator

scripts/all.ts

@@ -0,0 +1,54 @@
+import { makeWanikani } from '@/.'
+import sqlite3 from 'better-sqlite3'
+
+async function main() {
+  const db = sqlite3('cache/wanikani.db')
+  const wkApi = makeWanikani()
+
+  db.exec(/* sql */ `
+  CREATE TABLE IF NOT EXISTS [subjects] (
+    id          INT PRIMARY KEY NOT NULL,
+    updated_at  TIMESTAMP NOT NULL,
+    [object]    TEXT NOT NULL,
+    [url]       TEXT NOT NULL,
+    [data]      JSON NOT NULL
+  );
+
+  CREATE INDEX idx_subjects_object ON [subjects] ([object]);
+  `)
+
+  const stmt = db.prepare(/* sql */ `
+  INSERT OR REPLACE INTO [subjects] (id, updated_at, [object], [url], [data])
+  VALUES (@id, @updatedAt, @object, @url, @data)
+  `)
+
+  const batch: any[] = []
+
+  for await (const d of wkApi.subjects()) {
+    batch.push({
+      id: d.id,
+      updatedAt: d.data_updated_at,
+      object: d.object,
+      url: d.url,
+      data: JSON.stringify(d.data),
+    })
+
+    if (batch.length > 500) {
+      db.transaction(() => {
+        batch.splice(0, 500).map((d) => stmt.run(d))
+      })()
+    }
+  }
+
+  if (batch.length) {
+    db.transaction(() => {
+      batch.map((d) => stmt.run(d))
+    })()
+  }
+
+  db.close()
+}
+
+if (require.main === module) {
+  main()
+}