try create a database

.dockerignore

@@ -3,3 +3,5 @@
 !/src/
 !package.json
 !yarn.lock
+!/cache
+!/wk-api

.gitignore

@@ -256,3 +256,5 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+/pgdata
+/wk-api

Dockerfile

@@ -7,9 +7,9 @@ RUN apt-get update
 # pgroonga extension
 # Does not officially support alpine
 RUN apt-get install -y curl
-RUN curl -O https://packages.groonga.org/debian/groonga-apt-source-latest-buster.deb
-RUN apt-get install -y ./groonga-apt-source-latest-buster.deb
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN curl -O https://packages.groonga.org/debian/groonga-apt-source-latest-bullseye.deb
+RUN apt-get install -y ./groonga-apt-source-latest-bullseye.deb
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 RUN apt-get update
 RUN apt-get install -y postgresql-12-pgdg-pgroonga
@@ -25,14 +25,15 @@ RUN cd postgres-json-schema && make install
 # nodejs14
 # Install nodejs 14
 WORKDIR /app
-RUN apt-get install -y dirmngr apt-transport-https lsb-release ca-certificates jq
+RUN apt-get install -y dirmngr apt-transport-https lsb-release ca-certificates
 RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash -
 RUN apt-get install -y nodejs gcc g++ make
 
 RUN npm i -g yarn
+RUN yarn
 COPY package.json yarn.lock ./
 RUN yarn --frozen-lockfile
-# COPY . .
+COPY . .
 # RUN yarn build
-# RUN jq 'del(.devDependencies)' package.json > tmp.json && mv tmp.json package.json
+# RUN node -e 'const pkg=require("./package.json");delete pkg.devDependencies;require("fs").writeFileSync("package.json",JSON.stringify(pkg))'
 # RUN yarn --frozen-lockfile

docker-compose.yml

@@ -5,11 +5,15 @@ services:
     build: .
     environment:
       POSTGRES_USER: &pguser postgres
-      POSTGRES_PASSWORD: &pgpass postgres-passwd
+      POSTGRES_PASSWORD: &pgpass postgress
       POSTGRES_DB: &pgdb jaquiz
 
       TZ: &tz Asia/Bangkok
       PGTZ: *tz
+
+      PGUSER: *pguser
+      PGPASSWORD: *pgpass
+      PGDATABASE: *pgdb
     ports:
       - 5433:5432 # pgAdmin connection port
 
@@ -17,6 +21,8 @@ services:
       - ./pgdata:/var/lib/postgresql/data
 
       - ./initdb.d:/docker-entrypoint-initdb.d
+      # - ./node_modules:/app/node_modules
       - ./cache:/app/cache
       - ./assets:/app/assets
+      - ./scripts:/app/scripts
       - ./src:/app/src

initdb.d/10-wanikani.sql

@@ -0,0 +1,15 @@
+CREATE TABLE "wanikani.subjects" (
+  "id"          INT PRIMARY KEY NOT NULL,
+  "updatedAt"   TIMESTAMPTZ NOT NULL,
+  "object"      TEXT NOT NULL,
+  "url"         TEXT NOT NULL,
+  "data"        JSONB NOT NULL
+);
+
+CREATE INDEX "idx_wanikani.subjects_object" ON "wanikani.subjects" ("object");
+CREATE INDEX "idx_wanikani.subjects_data" ON "wanikani.subjects"
+  USING pgroonga("data")
+  WITH (
+    tokenizer='TokenMecab',
+    normalizer='NormalizerNFKC100("unify_kana", true)'
+  );

initdb.d/20-user.sql

@@ -4,7 +4,7 @@ CREATE TABLE "user" (
     "updatedAt"                 TIMESTAMPTZ DEFAULT now(),
     "identifier"                TEXT UNIQUE NOT NULL,
     "level.min"                 INT DEFAULT 1,
-    "level.max"                 INT NOT NULL,
+    "level"                     INT NOT NULL,
     "level.vocabulary.showing"  TEXT[],
     "quiz.settings"             JSONB
 );
@@ -17,6 +17,6 @@ CREATE TRIGGER "t_user_updatedAt"
 CREATE INDEX "idx_user_updatedAt" ON "user" ("updatedAt");
 CREATE INDEX "idx_user_identifier" ON "user" ("identifier");
 
-INSERT INTO "user" ("id", "identifier") VALUES
+INSERT INTO "user" ("id", "identifier", "level") VALUES
 (uuid_nil(), '', 1),
 ('00000000-0000-0000-0000-000000000001', 'wanikani', 1);

initdb.d/22-entry.sql

@@ -45,14 +45,14 @@ CREATE TABLE "entry.reading" (
     "id"            UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(),
     "createdAt"     TIMESTAMPTZ DEFAULT now(),
     "updatedAt"     TIMESTAMPTZ DEFAULT now(),
-    "userId"        UUID REFERENCES "user"("id") ON DELETE CASCADE,
-    "entryId"       UUID REFERENCES "entry"("id") ON DELETE CASCADE
+    "userId"        UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE,
+    "entryId"       UUID NOT NULL REFERENCES "entry"("id") ON DELETE CASCADE,
     "type"          TEXT NOT NULL,
     "reading"       TEXT[] NOT NULL DEFAULT '{}' CHECK ("reading"[1] IS NOT NULL)
 );
 
 CREATE INDEX "idx_entry.reading_updatedAt" ON "entry.reading" ("updatedAt");
-CREATE INDEX "idx_entry.reading_userId" ON "entry.reading" ("userId")
+CREATE INDEX "idx_entry.reading_userId" ON "entry.reading" ("userId");
 CREATE INDEX "idx_entry.reading_type" ON "entry.reading" ("type");
 CREATE INDEX "idx_entry.reading_reading" ON "entry.reading"
     USING pgroonga (normalize_reading("reading"))
@@ -65,15 +65,15 @@ CREATE TABLE "entry.translation" (
     "id"            UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(),
     "createdAt"     TIMESTAMPTZ DEFAULT now(),
     "updatedAt"     TIMESTAMPTZ DEFAULT now(),
-    "userId"        UUID REFERENCES "user"("id") ON DELETE CASCADE,
-    "entryId"       UUID REFERENCES "entry"("id") ON DELETE CASCADE
+    "userId"        UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE,
+    "entryId"       UUID NOT NULL REFERENCES "entry"("id") ON DELETE CASCADE,
     "language"      TEXT NOT NULL,
     "translation"   TEXT[] NOT NULL DEFAULT '{}' CHECK ("translation"[1] IS NOT NULL)
 );
 
 CREATE INDEX "idx_entry.translation_updatedAt" ON "entry.translation" ("updatedAt");
-CREATE INDEX "idx_entry.translation_userId" ON "entry.translation" ("userId")
-CREATE INDEX "idx_entry.translation_language" ON "entry.trnalsation" ("language");
+CREATE INDEX "idx_entry.translation_userId" ON "entry.translation" ("userId");
+CREATE INDEX "idx_entry.translation_language" ON "entry.translation" ("language");
 CREATE INDEX "idx_entry.translation_translation" ON "entry.translation"
     USING pgroonga("translation")
     WITH (plugins='token_filters/stem', token_filters='TokenFilterStem');

initdb.d/26-radical.sql → initdb.d/26-character.sql

@@ -1,15 +1,18 @@
-CREATE TABLE radical (
+CREATE TABLE "character" (
     "entry"         TEXT NOT NULL PRIMARY KEY,
     "sub"           TEXT[] NOT NULL,
     "sup"           TEXT[] NOT NULL,
-    "var"           TEXT[] NOT NULL
+    "var"           TEXT[] NOT NULL,
+    "amalgamation"  TEXT[] NOT NULL,
+    "component"     TEXT[] NOT NULL,
+    "visual"        TEXT[] NOT NULL
 );
 
-CREATE INDEX idx_radical_search ON radical
+CREATE INDEX idx_character_search ON >"character"
     USING pgroonga ("entry", "sub", "sup", "var")
     WITH (tokenizer='TokenUnigram');
 
-CREATE INDEX idx_radical_synonyms ON radical
+CREATE INDEX idx_character_synonyms ON >"character"
     USING pgroonga ("entry" pgroonga_text_term_search_ops_v2)
     WITH (tokenizer='TokenUnigram');
 

initdb.d/99-init.sh

@@ -5,7 +5,7 @@ pg_ctl -o "-c listen_addresses='localhost'" -w restart
 cd /app
 
 if [[ -d "./lib" ]]; then
-    node ./lib/init.js
+    node ./lib/index.js
 else
-    yarn ts ./src/init.ts
+    yarn ts ./src/index.ts
 fi

src/index.ts

@@ -0,0 +1,10 @@
+import createConnectionPool from '@databases/pg'
+
+import { makeWanikani } from './wanikani'
+
+if (require.main === module) {
+  ;(async function () {
+    const db = createConnectionPool({ bigIntMode: 'number' })
+    await makeWanikani(db)
+  })()
+}

src/wanikani.ts

@@ -0,0 +1,51 @@
+import path from 'path'
+
+import { ConnectionPool, sql } from '@databases/pg'
+import sqlite3 from 'better-sqlite3'
+
+export async function makeWanikani(
+  db: ConnectionPool,
+  srcdb = path.join(__dirname, '../cache/wanikani.db')
+) {
+  const s3 = sqlite3(srcdb, {
+    readonly: true,
+  })
+
+  await db.tx(async (db) => {
+    const batchSize = 10000
+
+    const lots = s3
+      .prepare(
+        /* sql */ `
+    SELECT "id", "updated_at", "object", "url", "data"
+    FROM "subjects"
+    `
+      )
+      .all()
+
+    for (let i = 0; i < lots.length; i += batchSize) {
+      console.log(i)
+      await db.query(sql`
+        INSERT INTO "wanikani.subjects" ("id", "updatedAt", "object", "url", "data")
+        VALUES ${sql.join(
+          lots
+            .slice(i, i + batchSize)
+            .map(
+              (p) =>
+                sql`(${p.id}, ${new Date(p.updated_at)}, ${p.object}, ${
+                  p.url
+                }, ${JSON.parse(p.data)})`
+            ),
+          ','
+        )}
+        ON CONFLICT ("id") DO UPDATE SET
+          "updatedAt" = EXCLUDED."updatedAt",
+          "object"    = EXCLUDED."object",
+          "url"       = EXCLUDED."url",
+          "data"      = EXCLUDED."data"
+      `)
+    }
+  })
+
+  s3.close()
+}