From 8bb5d996b9128c0e75d34882acd2ba15cb97e529 Mon Sep 17 00:00:00 2001 From: Pacharapol Withayasakpunt Date: Tue, 19 Oct 2021 13:27:06 +0700 Subject: [PATCH] add edict --- .dockerignore | 1 + .gitignore | 1 + initdb.d/01-function.sql | 7 +--- initdb.d/22-entry.sql | 54 +++++++------------------ initdb.d/24-preset.sql | 12 +++--- initdb.d/27-freq.sql | 7 ++++ package.json | 3 +- scripts/.gitkeep | 0 scripts/edict.ts | 87 ++++++++++++++++++++++++++++++++++++++++ scripts/freq.py | 7 ++++ yarn.lock | 5 +++ 11 files changed, 132 insertions(+), 52 deletions(-) create mode 100644 initdb.d/27-freq.sql delete mode 100644 scripts/.gitkeep create mode 100644 scripts/edict.ts create mode 100644 scripts/freq.py diff --git a/.dockerignore b/.dockerignore index d3bcdc0..fa19333 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,4 @@ !yarn.lock !/cache !/wk-api +!/scripts/ diff --git a/.gitignore b/.gitignore index f549ceb..d6207ce 100644 --- a/.gitignore +++ b/.gitignore @@ -258,3 +258,4 @@ cython_debug/ /pgdata /wk-api +/assets/ diff --git a/initdb.d/01-function.sql b/initdb.d/01-function.sql index 2f967d3..0c7c470 100755 --- a/initdb.d/01-function.sql +++ b/initdb.d/01-function.sql @@ -10,15 +10,12 @@ CREATE FUNCTION array_distinct(anyarray) RETURNS anyarray AS $f$ SELECT array_agg(DISTINCT x) FROM unnest($1) t(x); $f$ LANGUAGE SQL IMMUTABLE; -CREATE OR REPLACE FUNCTION normalize_reading (TEXT[]) RETURNS TEXT[] AS +CREATE OR REPLACE FUNCTION normalize_reading (TEXT) RETURNS TEXT[] AS $func$ DECLARE s TEXT; new_arr TEXT[] := '{}'; BEGIN - FOREACH s IN ARRAY $1||'{}'::text[] LOOP - new_arr := new_arr||ARRAY[split_part(s,'.',1), replace(s,'.','')]; - END LOOP; - RETURN new_arr; + RETURN ARRAY[split_part($1,'.',1), replace($1,'.','')]; END; $func$ LANGUAGE plpgsql IMMUTABLE; diff --git a/initdb.d/22-entry.sql b/initdb.d/22-entry.sql index e4a759b..5fcaf77 100755 --- a/initdb.d/22-entry.sql +++ b/initdb.d/22-entry.sql @@ -1,17 +1,20 @@ CREATE TABLE "entry" ( - "id" UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(), + "id" UUID NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMPTZ DEFAULT now(), "updatedAt" TIMESTAMPTZ DEFAULT now(), "userId" UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE, "type" TEXT NOT NULL, "entry" TEXT[] NOT NULL CHECK ("entry"[1] IS NOT NULL), - -- "reading" -- backref "entry.reading" - -- "translation" -- backref "entry.translation" + "reading" JSONB NOT NULL DEFAULT '{}', -- TODO: jsonschema + "translation" JSONB NOT NULL DEFAULT '{}', -- TODO: jsonschema "description" TEXT NOT NULL DEFAULT '', "tag" TEXT[] NOT NULL DEFAULT '{}', - "level" FLOAT NOT NULL, - "level.kanji" INT NOT NULL, - "frequency" FLOAT + "level" FLOAT, + "level.kanji" INT, + "frequency" FLOAT, + "key" TEXT UNIQUE, + "_meta" JSONB, -- TODO: jsonschema + PRIMARY KEY ("id") ); CREATE TRIGGER "t_entry_updatedAt" @@ -19,8 +22,6 @@ CREATE TRIGGER "t_entry_updatedAt" FOR EACH ROW EXECUTE PROCEDURE "f_updatedAt"(); -CREATE UNIQUE INDEX idx_entry_u ON "entry" (("entry"[1]), "type", "userId"); - CREATE INDEX "idx_entry_updatedAt" ON "entry" ("updatedAt"); CREATE INDEX "idx_entry_userId" ON "entry" ("userId"); CREATE INDEX "idx_entry_type" ON "entry" ("type"); @@ -41,39 +42,12 @@ CREATE INDEX "idx_entry_entry" ON "entry" CREATE INDEX "idx_entry_entry_gin" ON "entry" USING GIN("entry"); CREATE INDEX "idx_entry_tag" ON "entry" USING pgroonga ("tag"); -CREATE TABLE "entry.reading" ( - "id" UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(), - "createdAt" TIMESTAMPTZ DEFAULT now(), - "updatedAt" TIMESTAMPTZ DEFAULT now(), - "userId" UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE, - "entryId" UUID NOT NULL REFERENCES "entry"("id") ON DELETE CASCADE, - "type" TEXT NOT NULL, - "reading" TEXT[] NOT NULL DEFAULT '{}' CHECK ("reading"[1] IS NOT NULL) -); - -CREATE INDEX "idx_entry.reading_updatedAt" ON "entry.reading" ("updatedAt"); -CREATE INDEX "idx_entry.reading_userId" ON "entry.reading" ("userId"); -CREATE INDEX "idx_entry.reading_type" ON "entry.reading" ("type"); -CREATE INDEX "idx_entry.reading_reading" ON "entry.reading" - USING pgroonga (normalize_reading("reading")) +CREATE INDEX "idx_entry_meta" ON "entry" USING pgroonga("_meta"); +CREATE INDEX "idx_entry_reading" ON "entry" + USING pgroonga("reading") WITH ( tokenizer='TokenMecab', normalizer='NormalizerNFKC100("unify_kana", true)' ); - -CREATE TABLE "entry.translation" ( - "id" UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(), - "createdAt" TIMESTAMPTZ DEFAULT now(), - "updatedAt" TIMESTAMPTZ DEFAULT now(), - "userId" UUID NOT NULL REFERENCES "user"("id") ON DELETE CASCADE, - "entryId" UUID NOT NULL REFERENCES "entry"("id") ON DELETE CASCADE, - "language" TEXT NOT NULL, - "translation" TEXT[] NOT NULL DEFAULT '{}' CHECK ("translation"[1] IS NOT NULL) -); - -CREATE INDEX "idx_entry.translation_updatedAt" ON "entry.translation" ("updatedAt"); -CREATE INDEX "idx_entry.translation_userId" ON "entry.translation" ("userId"); -CREATE INDEX "idx_entry.translation_language" ON "entry.translation" ("language"); -CREATE INDEX "idx_entry.translation_translation" ON "entry.translation" - USING pgroonga("translation") - WITH (plugins='token_filters/stem', token_filters='TokenFilterStem'); +CREATE INDEX "idx_entry_translation" ON "entry" + USING pgroonga("translation"); diff --git a/initdb.d/24-preset.sql b/initdb.d/24-preset.sql index e1e2ab2..0b3f93c 100755 --- a/initdb.d/24-preset.sql +++ b/initdb.d/24-preset.sql @@ -1,4 +1,4 @@ -CREATE TABLE "quiz_preset" ( +CREATE TABLE "preset.quiz" ( "id" UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMPTZ DEFAULT now(), "updatedAt" TIMESTAMPTZ DEFAULT now(), @@ -7,12 +7,12 @@ CREATE TABLE "quiz_preset" ( "settings" JSONB NOT NULL ); -CREATE TRIGGER "t_quiz_preset_updatedAt" - BEFORE UPDATE ON "quiz_preset" +CREATE TRIGGER "t_preset.quiz_updatedAt" + BEFORE UPDATE ON "preset.quiz" FOR EACH ROW EXECUTE PROCEDURE "f_updatedAt"(); -CREATE INDEX "idx_quiz_preset_updatedAt" ON "quiz_preset" ("updatedAt"); -CREATE INDEX "idx_quiz_preset_userId" ON "quiz_preset" ("userId"); -CREATE INDEX "idx_quiz_preset_name" ON "quiz_preset" +CREATE INDEX "idx_preset.quiz_updatedAt" ON "preset.quiz" ("updatedAt"); +CREATE INDEX "idx_preset.quiz_userId" ON "preset.quiz" ("userId"); +CREATE INDEX "idx_preset.quiz_name" ON "preset.quiz" USING pgroonga ("name"); diff --git a/initdb.d/27-freq.sql b/initdb.d/27-freq.sql new file mode 100644 index 0000000..6212051 --- /dev/null +++ b/initdb.d/27-freq.sql @@ -0,0 +1,7 @@ +CREATE TABLE "frequency" ( + "entry" TEXT NOT NULL, + "frequency" FLOAT NOT NULL, + PRIMARY KEY ("entry") +); + +CREATE INDEX "idx_frequency_frequency" ON "frequency"("frequency"); diff --git a/package.json b/package.json index 0163921..ce8648d 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "better-sqlite3": "^7.4.3", "fast-glob": "^3.2.7", "js-yaml": "^4.1.0", - "jsonschema-definer": "^1.3.2" + "jsonschema-definer": "^1.3.2", + "python-shell": "^3.0.1" }, "devDependencies": { "@types/better-sqlite3": "^7.4.0", diff --git a/scripts/.gitkeep b/scripts/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/edict.ts b/scripts/edict.ts new file mode 100644 index 0000000..b659e1e --- /dev/null +++ b/scripts/edict.ts @@ -0,0 +1,87 @@ +import createConnectionPool, { ConnectionPool, sql } from '@databases/pg' +import sqlite3 from 'better-sqlite3' +import { PythonShell } from 'python-shell' + +async function main(db: ConnectionPool) { + const s3 = sqlite3('./assets/edict.db') + + const rows = s3 + .prepare( + /* sql */ ` + SELECT "id", "entry", "reading", "english" FROM "edict" + ` + ) + .all() + + const fPrev = new Set( + await db + .query( + sql` + SELECT "entry" FROM "frequency" + ` + ) + .then((rs) => rs.map((r) => r.entry)) + ) + + const py = new PythonShell('./scripts/freq.py', { + mode: 'text', + pythonOptions: ['-u'], + }) + + const fMap = new Map() + py.on('message', (msg: string) => { + const [w = '', f] = msg.split('=') + fMap.set(w, Number(f)) + }) + ;[...new Set(rows.flatMap((r) => JSON.parse(r.entry)))] + .filter((v) => !fPrev.has(v)) + .map((v) => py.send(v)) + + await new Promise((resolve, reject) => { + py.end((err) => (err ? reject(err) : resolve())) + }) + + const batchSize = 5000 + + if (fMap.size) { + const fRow = [...fMap] + for (let i = 0; i < fRow.length; i += batchSize) { + console.log(i) + + await db.query(sql` + INSERT INTO "frequency" ("entry", "frequency") + VALUES ${sql.join( + fRow.slice(i, i + batchSize).map(([k, v]) => sql`(${k}, ${v})`), + ',' + )} + ON CONFLICT ("entry") + DO UPDATE SET "frequency" = EXCLUDED."frequency"; + `) + } + } + + for (let i = 0; i < rows.length; i += batchSize) { + console.log(i) + const sublot = rows.slice(i, i + batchSize) + + await db.query(sql` + INSERT INTO "entry" ("userId", "type", "entry", "key", "reading", "translation") + VALUES ${sql.join( + sublot.map( + (r) => + sql`(uuid_nil(), 'vocabulary', ${JSON.parse( + r.entry + )}, ${`edict:${r.id}`}, ${{ + _: JSON.parse(r.reading), + }}, ${{ en: JSON.parse(r.english) }})` + ), + ',' + )} + ON CONFLICT DO NOTHING; + `) + } +} + +if (require.main === module) { + main(createConnectionPool({ bigIntMode: 'number' })) +} diff --git a/scripts/freq.py b/scripts/freq.py new file mode 100644 index 0000000..e428f83 --- /dev/null +++ b/scripts/freq.py @@ -0,0 +1,7 @@ +import sys + +from wordfreq import zipf_frequency + +for s in sys.stdin: + w = s.rstrip() + print(f"{w}={zipf_frequency(w, 'ja')}") diff --git a/yarn.lock b/yarn.lock index 6353d95..7f93131 100644 --- a/yarn.lock +++ b/yarn.lock @@ -918,6 +918,11 @@ punycode@^2.1.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +python-shell@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/python-shell/-/python-shell-3.0.1.tgz#c3d3b11536e6ebdb8d6a2602482f7180d940bb13" + integrity sha512-TWeotuxe1auhXa5bGRScxnc2J+0r41NBntSa6RYZtMBLtAEsvCboKrEbW6DvASosWQepVkhZZlT3B5Ei766G+Q== + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"