From fc7831b04e28773cdfb54bbb45d9a14531b1b567 Mon Sep 17 00:00:00 2001 From: polv Date: Sun, 30 Aug 2026 12:52:08 +0700 Subject: [PATCH] bundle tokenizer --- src-tauri/src/yomitan/search.rs | 124 ++++++++++++++++---------------- src-tauri/tauri.conf.json5 | 3 +- 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/src-tauri/src/yomitan/search.rs b/src-tauri/src/yomitan/search.rs index 638095e..dba2d44 100644 --- a/src-tauri/src/yomitan/search.rs +++ b/src-tauri/src/yomitan/search.rs @@ -4,72 +4,72 @@ pub struct YomitanSearch; impl YomitanSearch { pub fn migrations() -> Vec { - vec![ - Migration { - version: 1, - description: "create yomitan dict-version tables", - sql: " - CREATE TABLE dict ( - id INTEGER PRIMARY KEY, - title TEXT NOT NULL, - revision TEXT NOT NULL, - author TEXT, - url TEXT, - description TEXT, - sort_order INTEGER NOT NULL DEFAULT 0, - installed_at TEXT NOT NULL DEFAULT (datetime('now')), - bundle_name TEXT, -- custom - lang TEXT NOT NULL, -- custom - asset_count INTEGER, -- custom - source TEXT, -- custom - UNIQUE (title, revision) - ); - ", - kind: MigrationKind::Up, - }, - Migration { - version: 1, - description: "create yomitan ja query tables", - sql: " - CREATE TABLE ja_search ( - id INTEGER NOT NULL, -- yomitan.terms.id OR yomitan dups OR -user.terms.id - dict_id INTEGER NOT NULL REFERENCES dict(id) ON DELETE CASCADE, - term TEXT NOT NULL, -- normalized - term_seg TEXT, -- segmented and filtered for fillers - reading TEXT NOT NULL, -- normalized - frequency INTEGER, - PRIMARY KEY (id, dict_id) - ); + vec![ + Migration { + version: 1, + description: "create yomitan dict-version tables", + sql: " + CREATE TABLE dict ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + revision TEXT NOT NULL, + author TEXT, + url TEXT, + description TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + installed_at TEXT NOT NULL DEFAULT (datetime('now')), + bundle_name TEXT, -- custom + lang TEXT NOT NULL, -- custom + asset_count INTEGER, -- custom + source TEXT, -- custom + UNIQUE (title, revision) + ); + ", + kind: MigrationKind::Up, + }, + Migration { + version: 1, + description: "create yomitan ja query tables", + sql: " + CREATE TABLE ja_search ( + id INTEGER NOT NULL, -- yomitan.terms.id OR yomitan dups OR -user.terms.id + dict_id INTEGER NOT NULL REFERENCES dict(id) ON DELETE CASCADE, + term TEXT NOT NULL, -- normalized + term_seg TEXT, -- segmented and filtered for fillers + reading TEXT NOT NULL, -- normalized + frequency INTEGER, + PRIMARY KEY (id, dict_id) + ); - CREATE INDEX ja_search_term ON ja_search (term); - CREATE INDEX ja_search_reading ON ja_search (reading); - CREATE INDEX ja_search_frequency ON ja_search (frequency DESC); + CREATE INDEX ja_search_term ON ja_search (term); + CREATE INDEX ja_search_reading ON ja_search (reading); + CREATE INDEX ja_search_frequency ON ja_search (frequency DESC); - CREATE VIRTUAL TABLE IF NOT EXISTS ja_search_ft USING fts5 ( - term_seg, - content='ja_search', - content_rowid='rowid', - tokenize=unicode61 - ); + CREATE VIRTUAL TABLE IF NOT EXISTS ja_search_ft USING fts5 ( + term_seg, + content='ja_search', + content_rowid='rowid', + tokenize=unicode61 + ); - CREATE TRIGGER t_ja_search_ft_after_insert - AFTER INSERT ON ja_search BEGIN - INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg); - END; + CREATE TRIGGER t_ja_search_ft_after_insert + AFTER INSERT ON ja_search BEGIN + INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg); + END; - CREATE TRIGGER t_ja_search_ft_after_delete - AFTER DELETE ON ja_search BEGIN - INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg); - END; + CREATE TRIGGER t_ja_search_ft_after_delete + AFTER DELETE ON ja_search BEGIN + INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg); + END; - CREATE TRIGGER t_ja_search_ft_after_update - AFTER UPDATE ON ja_search BEGIN - INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg); - INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg); - END; - ", - kind: MigrationKind::Up - } - ] + CREATE TRIGGER t_ja_search_ft_after_update + AFTER UPDATE ON ja_search BEGIN + INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg); + INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg); + END; + ", + kind: MigrationKind::Up + } + ] } } diff --git a/src-tauri/tauri.conf.json5 b/src-tauri/tauri.conf.json5 index 5da1f30..c5b73f8 100644 --- a/src-tauri/tauri.conf.json5 +++ b/src-tauri/tauri.conf.json5 @@ -31,6 +31,7 @@ "icons/128x128@2x.png", "icons/icon.icns", "icons/icon.ico" - ] + ], + "resources": ["resources/mecab-ipadic/system.dic.zst"] } }