diff --git a/Cargo.lock b/Cargo.lock index 4435cf5..1aa4dc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -543,6 +543,7 @@ version = "0.1.0" dependencies = [ "anyhow", "blake3", + "regex", "reqwest", "rusqlite", "serde", diff --git a/src-core/Cargo.toml b/src-core/Cargo.toml index d1c73fa..99ec03f 100644 --- a/src-core/Cargo.toml +++ b/src-core/Cargo.toml @@ -16,3 +16,4 @@ walkdir = "2.5.0" reqwest = "0.13.2" blake3 = "1.8.3" anyhow = "1.0.102" +regex = "1.12.3" diff --git a/src-core/examples/yomitan_update_schema.rs b/src-core/examples/yomitan_update_schema.rs new file mode 100644 index 0000000..2e2d858 --- /dev/null +++ b/src-core/examples/yomitan_update_schema.rs @@ -0,0 +1,19 @@ + + +use std::time::Instant; + +use anyhow::Ok; +use cjdic2_core::AppService; + +mod common; +use common::get_db_dir; + +fn main() -> Result<(), anyhow::Error> { + let service = AppService::new(get_db_dir())?; + + let start = Instant::now(); + service.get_yomitan_writer()?; + println!("[{:.2?}]", start.elapsed()); + + Ok(()) +} diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs index 8c642ea..bf42824 100644 --- a/src-core/src/db/yomitan.rs +++ b/src-core/src/db/yomitan.rs @@ -1,7 +1,10 @@ use rusqlite::{Result, params}; use serde::Serialize; -use crate::{CJDicError, db::Database}; +use crate::{ + CJDicError, + db::{Database, yomitan_writer::normalize_term}, +}; #[derive(Serialize, Debug)] pub struct YomitanRow { @@ -34,6 +37,12 @@ impl YomitanDatabase { ) -> Result, CJDicError> { let and_or = if q_term == q_reading { "OR" } else { "AND" }; + let q_term_norm = if q_term.ends_with("*") { + normalize_term(&q_term[..q_term.len() - 1]) + "*" + } else { + normalize_term(q_term) + }; + // TODO: consider using = if not GLOB string let sql = format!( r#" @@ -48,13 +57,14 @@ impl YomitanDatabase { COALESCE(tt.tags, '') AS term_tags, d.title AS dict_title FROM yomitan.terms t + JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading JOIN yomitan.glossaries g ON g.id = t.glossary_id JOIN yomitan.dictionaries d ON d.id = t.dict_id LEFT JOIN yomitan.def_tag_sets dt ON dt.id = t.def_tags_id LEFT JOIN yomitan.rule_sets r ON r.id = t.rules_id LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id - WHERE LOWER(t.term) GLOB ?1 {} t.reading GLOB ?2 - ORDER BY t.score DESC + WHERE t.term_norm GLOB ?1 {} t.reading GLOB ?2 + ORDER BY tr.max_score DESC LIMIT ?3 OFFSET ?4 "#, and_or @@ -63,22 +73,19 @@ impl YomitanDatabase { let conn = self.db.conn.lock().unwrap(); let mut stmt = conn.prepare(&sql)?; - let rows = stmt.query_map( - params![q_term.to_ascii_lowercase(), q_reading, limit, offset], - |r| { - Ok(YomitanRow { - term: r.get(0)?, - reading: r.get(1)?, - def_tags: r.get(2)?, - rules: r.get(3)?, - score: r.get(4)?, - glossary_json: r.get(5)?, - sequence: r.get(6)?, - term_tags: r.get(7)?, - dict_title: r.get(8)?, - }) - }, - )?; + let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| { + Ok(YomitanRow { + term: r.get(0)?, + reading: r.get(1)?, + def_tags: r.get(2)?, + rules: r.get(3)?, + score: r.get(4)?, + glossary_json: r.get(5)?, + sequence: r.get(6)?, + term_tags: r.get(7)?, + dict_title: r.get(8)?, + }) + })?; let mut out = Vec::new(); for r in rows { diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs index 06ca771..3db47ee 100644 --- a/src-core/src/db/yomitan_writer.rs +++ b/src-core/src/db/yomitan_writer.rs @@ -12,6 +12,13 @@ fn blake3_hex(s: &str) -> String { format!("{}", hash(s.as_bytes())) } +pub(super) fn normalize_term(s: &str) -> String { + s.chars() + .filter(|c| c.is_alphabetic()) + .flat_map(|c| c.to_lowercase()) + .collect() +} + #[derive(Serialize, Debug)] pub struct YomitanZipImportResult { pub exists: bool, @@ -38,7 +45,7 @@ impl YomitanWriter { Ok(Self { conn }) } - pub fn create_schema(&self) -> Result<()> { + pub fn create_schema(&mut self) -> Result<()> { self.conn.execute_batch( " CREATE TABLE IF NOT EXISTS schema_meta ( @@ -54,7 +61,7 @@ impl YomitanWriter { description TEXT, sort_order INTEGER NOT NULL DEFAULT 0, installed_at TEXT NOT NULL DEFAULT (datetime('now')), - lang TEXT NOT NULL, + lang TEXT NOT NULL, -- custom UNIQUE (title, revision) ); CREATE TABLE IF NOT EXISTS glossaries ( @@ -75,7 +82,8 @@ impl YomitanWriter { score INTEGER NOT NULL DEFAULT 0, glossary_id INTEGER NOT NULL REFERENCES glossaries(id), sequence INTEGER, - term_tags_id INTEGER + term_tags_id INTEGER, + term_norm TEXT -- custom ); CREATE TABLE IF NOT EXISTS term_meta ( id INTEGER PRIMARY KEY, @@ -115,6 +123,7 @@ impl YomitanWriter { CREATE INDEX IF NOT EXISTS terms_dict_id ON terms (dict_id); CREATE INDEX IF NOT EXISTS terms_term_LOWER ON terms (LOWER(term)); CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading); + CREATE INDEX IF NOT EXISTS terms_term_reading_score ON terms(term, reading, score DESC); CREATE INDEX IF NOT EXISTS term_meta_dict_id ON term_meta (dict_id); @@ -124,6 +133,8 @@ impl YomitanWriter { ", )?; + self.create_materialized_views()?; + let mut stmt = self .conn .prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?; @@ -141,6 +152,83 @@ impl YomitanWriter { Ok(()) } + fn create_materialized_views(&mut self) -> Result<()> { + self.conn.execute_batch( + " + CREATE TABLE IF NOT EXISTS view_terms_term_rank ( + term TEXT NOT NULL, + reading TEXT NOT NULL, + max_score INTEGER NOT NULL, + PRIMARY KEY (term, reading) + ); + ", + )?; + + self.conn.execute_batch( + " + CREATE TRIGGER IF NOT EXISTS terms_max_score_after_insert + AFTER INSERT ON terms + WHEN NEW.score > 0 + -- This automatically excludes NULL (since NULL > 0 is false) + BEGIN + INSERT INTO term_rank (term, reading, max_score) + VALUES (NEW.term, NEW.reading, NEW.score) + ON CONFLICT(term, reading) + DO UPDATE SET max_score = + CASE + WHEN NEW.score > max_score + THEN NEW.score + ELSE max_score + END; + END; + ", + )?; + // no AFTER DELETE trigger yet. existence of max score is more important. + + { + let mut stmt = self.conn.prepare("SELECT 1 FROM view_terms_term_rank")?; + if !(stmt.exists([])?) { + self.conn.execute_batch( + " + INSERT INTO view_terms_term_rank + SELECT term, reading, MAX(score) + FROM terms + GROUP BY term, reading + ", + )?; + // 1.9 sec + } + } + + let tx = self.conn.transaction()?; + { + let mut stmt = tx.prepare("SELECT rowid, term FROM terms WHERE term_norm IS NULL")?; + + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + + let mut update_stmt = tx.prepare("UPDATE terms SET term_norm = ?1 WHERE rowid = ?2")?; + + for row in rows { + let (rowid, term) = row?; + let norm = normalize_term(&term); // your Unicode normalizer + update_stmt.execute((&norm, rowid))?; + } + } + tx.commit()?; + // 5 sec + + self.conn.execute_batch( + " + CREATE INDEX IF NOT EXISTS terms_term_norm ON terms (term_norm); + ", + )?; + // 1 sec + + Ok(()) + } + pub fn import_dictionary_zip_file( &mut self, zip_file: PathBuf, diff --git a/src-core/src/service.rs b/src-core/src/service.rs index cd13864..feb0191 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -43,7 +43,7 @@ impl AppService { pub fn get_yomitan_writer(&self) -> Result { let conn = Connection::open(self.db.dir.join("yomitan.db"))?; - let writer = YomitanWriter::new(conn)?; + let mut writer = YomitanWriter::new(conn)?; writer.create_schema()?; Ok(writer) }