From 792fa600305fa013e2516a1b3b31d790a1ac424a Mon Sep 17 00:00:00 2001 From: Pacharapol Withayasakpunt Date: Sun, 17 Oct 2021 09:36:24 +0700 Subject: [PATCH] try create a database --- .dockerignore | 2 + .gitignore | 2 + Dockerfile | 13 ++--- cache/.gitignore | 0 docker-compose.yml | 8 ++- initdb.d/00-extension.sql | 0 initdb.d/01-function.sql | 0 initdb.d/10-wanikani.sql | 15 ++++++ initdb.d/20-user.sql | 4 +- initdb.d/21-quiz.sql | 0 initdb.d/22-entry.sql | 14 ++--- initdb.d/23-library.sql | 0 initdb.d/24-preset.sql | 0 initdb.d/{26-radical.sql => 26-character.sql} | 11 ++-- initdb.d/50-session.sql | 0 initdb.d/99-init.sh | 4 +- src/index.ts | 10 ++++ src/init.ts | 0 src/wanikani.ts | 51 +++++++++++++++++++ 19 files changed, 112 insertions(+), 22 deletions(-) mode change 100644 => 100755 cache/.gitignore mode change 100644 => 100755 initdb.d/00-extension.sql mode change 100644 => 100755 initdb.d/01-function.sql create mode 100755 initdb.d/10-wanikani.sql mode change 100644 => 100755 initdb.d/20-user.sql mode change 100644 => 100755 initdb.d/21-quiz.sql mode change 100644 => 100755 initdb.d/22-entry.sql mode change 100644 => 100755 initdb.d/23-library.sql mode change 100644 => 100755 initdb.d/24-preset.sql rename initdb.d/{26-radical.sql => 26-character.sql} (68%) mode change 100644 => 100755 mode change 100644 => 100755 initdb.d/50-session.sql create mode 100755 src/index.ts delete mode 100644 src/init.ts create mode 100755 src/wanikani.ts diff --git a/.dockerignore b/.dockerignore index 067a162..d3bcdc0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,5 @@ !/src/ !package.json !yarn.lock +!/cache +!/wk-api diff --git a/.gitignore b/.gitignore index 97d71b2..f549ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -256,3 +256,5 @@ dmypy.json # Cython debug symbols cython_debug/ +/pgdata +/wk-api diff --git a/Dockerfile b/Dockerfile index 501803f..1a6e2ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,9 +7,9 @@ RUN apt-get update # pgroonga extension # Does not officially support alpine RUN apt-get install -y curl -RUN curl -O https://packages.groonga.org/debian/groonga-apt-source-latest-buster.deb -RUN apt-get install -y ./groonga-apt-source-latest-buster.deb -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN curl -O https://packages.groonga.org/debian/groonga-apt-source-latest-bullseye.deb +RUN apt-get install -y ./groonga-apt-source-latest-bullseye.deb +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN apt-get update RUN apt-get install -y postgresql-12-pgdg-pgroonga @@ -25,14 +25,15 @@ RUN cd postgres-json-schema && make install # nodejs14 # Install nodejs 14 WORKDIR /app -RUN apt-get install -y dirmngr apt-transport-https lsb-release ca-certificates jq +RUN apt-get install -y dirmngr apt-transport-https lsb-release ca-certificates RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get install -y nodejs gcc g++ make RUN npm i -g yarn +RUN yarn COPY package.json yarn.lock ./ RUN yarn --frozen-lockfile -# COPY . . +COPY . . # RUN yarn build -# RUN jq 'del(.devDependencies)' package.json > tmp.json && mv tmp.json package.json +# RUN node -e 'const pkg=require("./package.json");delete pkg.devDependencies;require("fs").writeFileSync("package.json",JSON.stringify(pkg))' # RUN yarn --frozen-lockfile diff --git a/cache/.gitignore b/cache/.gitignore old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml index 9d2bf8d..410b55e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,11 +5,15 @@ services: build: . environment: POSTGRES_USER: &pguser postgres - POSTGRES_PASSWORD: &pgpass postgres-passwd + POSTGRES_PASSWORD: &pgpass postgress POSTGRES_DB: &pgdb jaquiz TZ: &tz Asia/Bangkok PGTZ: *tz + + PGUSER: *pguser + PGPASSWORD: *pgpass + PGDATABASE: *pgdb ports: - 5433:5432 # pgAdmin connection port @@ -17,6 +21,8 @@ services: - ./pgdata:/var/lib/postgresql/data - ./initdb.d:/docker-entrypoint-initdb.d + # - ./node_modules:/app/node_modules - ./cache:/app/cache - ./assets:/app/assets + - ./scripts:/app/scripts - ./src:/app/src diff --git a/initdb.d/00-extension.sql b/initdb.d/00-extension.sql old mode 100644 new mode 100755 diff --git a/initdb.d/01-function.sql b/initdb.d/01-function.sql old mode 100644 new mode 100755 diff --git a/initdb.d/10-wanikani.sql b/initdb.d/10-wanikani.sql new file mode 100755 index 0000000..3b10325 --- /dev/null +++ b/initdb.d/10-wanikani.sql @@ -0,0 +1,15 @@ +CREATE TABLE "wanikani.subjects" ( + "id" INT PRIMARY KEY NOT NULL, + "updatedAt" TIMESTAMPTZ NOT NULL, + "object" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" JSONB NOT NULL +); + +CREATE INDEX "idx_wanikani.subjects_object" ON "wanikani.subjects" ("object"); +CREATE INDEX "idx_wanikani.subjects_data" ON "wanikani.subjects" + USING pgroonga("data") + WITH ( + tokenizer='TokenMecab', + normalizer='NormalizerNFKC100("unify_kana", true)' + ); diff --git a/initdb.d/20-user.sql b/initdb.d/20-user.sql old mode 100644 new mode 100755 index c12a8a5..e912fdf --- a/initdb.d/20-user.sql +++ b/initdb.d/20-user.sql @@ -4,7 +4,7 @@ CREATE TABLE "user" ( "updatedAt" TIMESTAMPTZ DEFAULT now(), "identifier" TEXT UNIQUE NOT NULL, "level.min" INT DEFAULT 1, - "level.max" INT NOT NULL, + "level" INT NOT NULL, "level.vocabulary.showing" TEXT[], "quiz.settings" JSONB ); @@ -17,6 +17,6 @@ CREATE TRIGGER "t_user_updatedAt" CREATE INDEX "idx_user_updatedAt" ON "user" ("updatedAt"); CREATE INDEX "idx_user_identifier" ON "user" ("identifier"); -INSERT INTO "user" ("id", "identifier") VALUES +INSERT INTO "user" ("id", "identifier", "level") VALUES (uuid_nil(), '', 1), ('00000000-0000-0000-0000-000000000001', 'wanikani', 1); diff --git a/initdb.d/21-quiz.sql b/initdb.d/21-quiz.sql old mode 100644 new mode 100755 diff --git a/initdb.d/22-entry.sql b/initdb.d/22-entry.sql old mode 100644 new mode 100755 index b7b8cb8..e4a759b --- a/initdb.d/22-entry.sql +++ b/initdb.d/22-entry.sql @@ -45,14 +45,14 @@ CREATE TABLE "entry.reading" ( "id" UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMPTZ DEFAULT now(), "updatedAt" TIMESTAMPTZ DEFAULT now(), - "userId" UUID REFERENCES "user"("id") ON DELETE CASCADE, - "entryId" UUID REFERENCES "entry"("id") ON DELETE CASCADE + "userId" UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE, + "entryId" UUID NOT NULL REFERENCES "entry"("id") ON DELETE CASCADE, "type" TEXT NOT NULL, "reading" TEXT[] NOT NULL DEFAULT '{}' CHECK ("reading"[1] IS NOT NULL) ); CREATE INDEX "idx_entry.reading_updatedAt" ON "entry.reading" ("updatedAt"); -CREATE INDEX "idx_entry.reading_userId" ON "entry.reading" ("userId") +CREATE INDEX "idx_entry.reading_userId" ON "entry.reading" ("userId"); CREATE INDEX "idx_entry.reading_type" ON "entry.reading" ("type"); CREATE INDEX "idx_entry.reading_reading" ON "entry.reading" USING pgroonga (normalize_reading("reading")) @@ -65,15 +65,15 @@ CREATE TABLE "entry.translation" ( "id" UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMPTZ DEFAULT now(), "updatedAt" TIMESTAMPTZ DEFAULT now(), - "userId" UUID REFERENCES "user"("id") ON DELETE CASCADE, - "entryId" UUID REFERENCES "entry"("id") ON DELETE CASCADE + "userId" UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE, + "entryId" UUID NOT NULL REFERENCES "entry"("id") ON DELETE CASCADE, "language" TEXT NOT NULL, "translation" TEXT[] NOT NULL DEFAULT '{}' CHECK ("translation"[1] IS NOT NULL) ); CREATE INDEX "idx_entry.translation_updatedAt" ON "entry.translation" ("updatedAt"); -CREATE INDEX "idx_entry.translation_userId" ON "entry.translation" ("userId") -CREATE INDEX "idx_entry.translation_language" ON "entry.trnalsation" ("language"); +CREATE INDEX "idx_entry.translation_userId" ON "entry.translation" ("userId"); +CREATE INDEX "idx_entry.translation_language" ON "entry.translation" ("language"); CREATE INDEX "idx_entry.translation_translation" ON "entry.translation" USING pgroonga("translation") WITH (plugins='token_filters/stem', token_filters='TokenFilterStem'); diff --git a/initdb.d/23-library.sql b/initdb.d/23-library.sql old mode 100644 new mode 100755 diff --git a/initdb.d/24-preset.sql b/initdb.d/24-preset.sql old mode 100644 new mode 100755 diff --git a/initdb.d/26-radical.sql b/initdb.d/26-character.sql old mode 100644 new mode 100755 similarity index 68% rename from initdb.d/26-radical.sql rename to initdb.d/26-character.sql index 7bd697e..2c53fb1 --- a/initdb.d/26-radical.sql +++ b/initdb.d/26-character.sql @@ -1,15 +1,18 @@ -CREATE TABLE radical ( +CREATE TABLE "character" ( "entry" TEXT NOT NULL PRIMARY KEY, "sub" TEXT[] NOT NULL, "sup" TEXT[] NOT NULL, - "var" TEXT[] NOT NULL + "var" TEXT[] NOT NULL, + "amalgamation" TEXT[] NOT NULL, + "component" TEXT[] NOT NULL, + "visual" TEXT[] NOT NULL ); -CREATE INDEX idx_radical_search ON radical +CREATE INDEX idx_character_search ON "character" USING pgroonga ("entry", "sub", "sup", "var") WITH (tokenizer='TokenUnigram'); -CREATE INDEX idx_radical_synonyms ON radical +CREATE INDEX idx_character_synonyms ON "character" USING pgroonga ("entry" pgroonga_text_term_search_ops_v2) WITH (tokenizer='TokenUnigram'); diff --git a/initdb.d/50-session.sql b/initdb.d/50-session.sql old mode 100644 new mode 100755 diff --git a/initdb.d/99-init.sh b/initdb.d/99-init.sh index adc010b..c9adce7 100755 --- a/initdb.d/99-init.sh +++ b/initdb.d/99-init.sh @@ -5,7 +5,7 @@ pg_ctl -o "-c listen_addresses='localhost'" -w restart cd /app if [[ -d "./lib" ]]; then - node ./lib/init.js + node ./lib/index.js else - yarn ts ./src/init.ts + yarn ts ./src/index.ts fi diff --git a/src/index.ts b/src/index.ts new file mode 100755 index 0000000..935176c --- /dev/null +++ b/src/index.ts @@ -0,0 +1,10 @@ +import createConnectionPool from '@databases/pg' + +import { makeWanikani } from './wanikani' + +if (require.main === module) { + ;(async function () { + const db = createConnectionPool({ bigIntMode: 'number' }) + await makeWanikani(db) + })() +} diff --git a/src/init.ts b/src/init.ts deleted file mode 100644 index e69de29..0000000 diff --git a/src/wanikani.ts b/src/wanikani.ts new file mode 100755 index 0000000..36e0ca4 --- /dev/null +++ b/src/wanikani.ts @@ -0,0 +1,51 @@ +import path from 'path' + +import { ConnectionPool, sql } from '@databases/pg' +import sqlite3 from 'better-sqlite3' + +export async function makeWanikani( + db: ConnectionPool, + srcdb = path.join(__dirname, '../cache/wanikani.db') +) { + const s3 = sqlite3(srcdb, { + readonly: true, + }) + + await db.tx(async (db) => { + const batchSize = 10000 + + const lots = s3 + .prepare( + /* sql */ ` + SELECT "id", "updated_at", "object", "url", "data" + FROM "subjects" + ` + ) + .all() + + for (let i = 0; i < lots.length; i += batchSize) { + console.log(i) + await db.query(sql` + INSERT INTO "wanikani.subjects" ("id", "updatedAt", "object", "url", "data") + VALUES ${sql.join( + lots + .slice(i, i + batchSize) + .map( + (p) => + sql`(${p.id}, ${new Date(p.updated_at)}, ${p.object}, ${ + p.url + }, ${JSON.parse(p.data)})` + ), + ',' + )} + ON CONFLICT ("id") DO UPDATE SET + "updatedAt" = EXCLUDED."updatedAt", + "object" = EXCLUDED."object", + "url" = EXCLUDED."url", + "data" = EXCLUDED."data" + `) + } + }) + + s3.close() +}