add kanjidic and radicals

scripts/radical.ts

@@ -0,0 +1,98 @@
+import createConnectionPool, { ConnectionPool, sql } from '@databases/pg'
+import sqlite3 from 'better-sqlite3'
+import { PythonShell } from 'python-shell'
+
+export async function main(db: ConnectionPool) {
+  const s3 = sqlite3('./assets/radical.db')
+
+  const rows = s3
+    .prepare(
+      /* sql */ `
+  SELECT "entry", "sub", "sup", "var" FROM "radical"
+  `
+    )
+    .all()
+
+  const fPrev = new Set<string>(
+    await db
+      .query(
+        sql`
+        SELECT "entry" FROM "frequency"
+        `
+      )
+      .then((rs) => rs.map((r) => r.entry))
+  )
+
+  const py = new PythonShell('./scripts/freq.py', {
+    mode: 'text',
+    pythonOptions: ['-u'],
+  })
+
+  const fMap = new Map<string, number>()
+  py.on('message', (msg: string) => {
+    const [w = '', f] = msg.split('=')
+    fMap.set(w, Number(f))
+  })
+  ;[
+    ...new Set(
+      rows.flatMap((r) =>
+        [
+          ...((r.entry + r.sub + r.sup + r.var) as string).matchAll(
+            /\p{sc=Han}/gu
+          ),
+        ].map((m) => m[0]!)
+      )
+    ),
+  ]
+    .filter((v) => !fPrev.has(v))
+    .map((v) => py.send(v))
+
+  await new Promise<void>((resolve, reject) => {
+    py.end((err) => (err ? reject(err) : resolve()))
+  })
+
+  const batchSize = 5000
+
+  if (fMap.size) {
+    const fRow = [...fMap]
+    for (let i = 0; i < fRow.length; i += batchSize) {
+      console.log(i)
+
+      await db.query(sql`
+      INSERT INTO "frequency" ("entry", "frequency")
+      VALUES ${sql.join(
+        fRow.slice(i, i + batchSize).map(([k, v]) => sql`(${k}, ${v})`),
+        ','
+      )}
+      ON CONFLICT ("entry")
+      DO UPDATE SET "frequency" = EXCLUDED."frequency";
+      `)
+    }
+  }
+
+  for (let i = 0; i < rows.length; i += batchSize) {
+    console.log(i)
+    const sublot = rows.slice(i, i + batchSize)
+
+    await db.query(sql`
+    INSERT INTO "character" ("entry", "sub", "sup", "var")
+    VALUES ${sql.join(
+      sublot.map((r) => {
+        return sql`(${r.entry}, ${[
+          ...(r.sub as string).matchAll(/\p{sc=Han}/gu),
+        ].map((m) => m[0]!)}, ${[
+          ...(r.sup as string).matchAll(/\p{sc=Han}/gu),
+        ].map((m) => m[0]!)}, ${[
+          ...(r.var as string).matchAll(/\p{sc=Han}/gu),
+        ].map((m) => m[0]!)})`
+      }),
+      ','
+    )}
+    ON CONFLICT DO NOTHING;
+    `)
+  }
+}
+
+if (require.main === module) {
+  main(createConnectionPool({ bigIntMode: 'number' }))
+}