sort by max score

Cargo.lock

@@ -543,6 +543,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "blake3",
+ "regex",
  "reqwest",
  "rusqlite",
  "serde",

src-core/Cargo.toml

@@ -16,3 +16,4 @@ walkdir = "2.5.0"
 reqwest = "0.13.2"
 blake3 = "1.8.3"
 anyhow = "1.0.102"
+regex = "1.12.3"

src-core/examples/yomitan_update_schema.rs

@@ -0,0 +1,19 @@
+
+
+use std::time::Instant;
+
+use anyhow::Ok;
+use cjdic2_core::AppService;
+
+mod common;
+use common::get_db_dir;
+
+fn main() -> Result<(), anyhow::Error> {
+    let service = AppService::new(get_db_dir())?;
+
+    let start = Instant::now();
+    service.get_yomitan_writer()?;
+    println!("[{:.2?}]", start.elapsed());
+
+    Ok(())
+}

src-core/src/db/yomitan.rs

@@ -1,7 +1,10 @@
 use rusqlite::{Result, params};
 use serde::Serialize;
 
-use crate::{CJDicError, db::Database};
+use crate::{
+    CJDicError,
+    db::{Database, yomitan_writer::normalize_term},
+};
 
 #[derive(Serialize, Debug)]
 pub struct YomitanRow {
@@ -34,6 +37,12 @@ impl YomitanDatabase {
     ) -> Result<Vec<YomitanRow>, CJDicError> {
         let and_or = if q_term == q_reading { "OR" } else { "AND" };
 
+        let q_term_norm = if q_term.ends_with("*") {
+            normalize_term(&q_term[..q_term.len() - 1]) + "*"
+        } else {
+            normalize_term(q_term)
+        };
+
         // TODO: consider using = if not GLOB string
         let sql = format!(
             r#"
@@ -48,13 +57,14 @@ impl YomitanDatabase {
             COALESCE(tt.tags,  '')  AS term_tags,
             d.title                 AS dict_title
         FROM yomitan.terms              t
+        JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
         JOIN yomitan.glossaries         g  ON g.id  = t.glossary_id
         JOIN yomitan.dictionaries       d  ON d.id  = t.dict_id
         LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
         LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
         LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id
-        WHERE LOWER(t.term) GLOB ?1 {} t.reading GLOB ?2
+        WHERE t.term_norm GLOB ?1 {} t.reading GLOB ?2
-        ORDER BY t.score DESC
+        ORDER BY tr.max_score DESC
         LIMIT ?3 OFFSET ?4
         "#,
             and_or
@@ -63,22 +73,19 @@ impl YomitanDatabase {
         let conn = self.db.conn.lock().unwrap();
 
         let mut stmt = conn.prepare(&sql)?;
-        let rows = stmt.query_map(
+        let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| {
-            params![q_term.to_ascii_lowercase(), q_reading, limit, offset],
+            Ok(YomitanRow {
-            |r| {
+                term: r.get(0)?,
-                Ok(YomitanRow {
+                reading: r.get(1)?,
-                    term: r.get(0)?,
+                def_tags: r.get(2)?,
-                    reading: r.get(1)?,
+                rules: r.get(3)?,
-                    def_tags: r.get(2)?,
+                score: r.get(4)?,
-                    rules: r.get(3)?,
+                glossary_json: r.get(5)?,
-                    score: r.get(4)?,
+                sequence: r.get(6)?,
-                    glossary_json: r.get(5)?,
+                term_tags: r.get(7)?,
-                    sequence: r.get(6)?,
+                dict_title: r.get(8)?,
-                    term_tags: r.get(7)?,
+            })
-                    dict_title: r.get(8)?,
+        })?;
-                })
-            },
-        )?;
 
         let mut out = Vec::new();
         for r in rows {

src-core/src/db/yomitan_writer.rs

@@ -12,6 +12,13 @@ fn blake3_hex(s: &str) -> String {
     format!("{}", hash(s.as_bytes()))
 }
 
+pub(super) fn normalize_term(s: &str) -> String {
+    s.chars()
+        .filter(|c| c.is_alphabetic())
+        .flat_map(|c| c.to_lowercase())
+        .collect()
+}
+
 #[derive(Serialize, Debug)]
 pub struct YomitanZipImportResult {
     pub exists: bool,
@@ -38,7 +45,7 @@ impl YomitanWriter {
         Ok(Self { conn })
     }
 
-    pub fn create_schema(&self) -> Result<()> {
+    pub fn create_schema(&mut self) -> Result<()> {
         self.conn.execute_batch(
         "
             CREATE TABLE IF NOT EXISTS schema_meta (
@@ -54,7 +61,7 @@ impl YomitanWriter {
                 description  TEXT,
                 sort_order   INTEGER NOT NULL DEFAULT 0,
                 installed_at TEXT    NOT NULL DEFAULT (datetime('now')),
-                lang         TEXT    NOT NULL,
+                lang         TEXT    NOT NULL,      -- custom
                 UNIQUE (title, revision)
             );
             CREATE TABLE IF NOT EXISTS glossaries (
@@ -75,7 +82,8 @@ impl YomitanWriter {
                 score INTEGER NOT NULL DEFAULT 0,
                 glossary_id INTEGER NOT NULL REFERENCES glossaries(id),
                 sequence INTEGER,
-                term_tags_id INTEGER
+                term_tags_id INTEGER,
+                term_norm TEXT                      -- custom
             );
             CREATE TABLE IF NOT EXISTS term_meta (
                 id INTEGER PRIMARY KEY,
@@ -115,6 +123,7 @@ impl YomitanWriter {
             CREATE INDEX IF NOT EXISTS terms_dict_id ON terms (dict_id);
             CREATE INDEX IF NOT EXISTS terms_term_LOWER ON terms (LOWER(term));
             CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading);
+            CREATE INDEX IF NOT EXISTS terms_term_reading_score ON terms(term, reading, score DESC);
 
             CREATE INDEX IF NOT EXISTS term_meta_dict_id ON term_meta (dict_id);
 
@@ -124,6 +133,8 @@ impl YomitanWriter {
             ",
         )?;
 
+        self.create_materialized_views()?;
+
         let mut stmt = self
             .conn
             .prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?;
@@ -141,6 +152,83 @@ impl YomitanWriter {
         Ok(())
     }
 
+    fn create_materialized_views(&mut self) -> Result<()> {
+        self.conn.execute_batch(
+            "
+            CREATE TABLE IF NOT EXISTS view_terms_term_rank (
+                term        TEXT NOT NULL,
+                reading     TEXT NOT NULL,
+                max_score   INTEGER NOT NULL,
+                PRIMARY KEY (term, reading)
+            );
+            ",
+        )?;
+
+        self.conn.execute_batch(
+            "
+            CREATE TRIGGER IF NOT EXISTS terms_max_score_after_insert
+            AFTER INSERT ON terms
+            WHEN NEW.score > 0
+            -- This automatically excludes NULL (since NULL > 0 is false)
+            BEGIN
+                INSERT INTO term_rank (term, reading, max_score)
+                VALUES (NEW.term, NEW.reading, NEW.score)
+                ON CONFLICT(term, reading)
+                DO UPDATE SET max_score =
+                    CASE
+                        WHEN NEW.score > max_score
+                        THEN NEW.score
+                        ELSE max_score
+                    END;
+            END;
+            ",
+        )?;
+        // no AFTER DELETE trigger yet. existence of max score is more important.
+
+        {
+            let mut stmt = self.conn.prepare("SELECT 1 FROM view_terms_term_rank")?;
+            if !(stmt.exists([])?) {
+                self.conn.execute_batch(
+                    "
+                    INSERT INTO view_terms_term_rank
+                    SELECT term, reading, MAX(score)
+                    FROM terms
+                    GROUP BY term, reading
+                    ",
+                )?;
+                // 1.9 sec
+            }
+        }
+
+        let tx = self.conn.transaction()?;
+        {
+            let mut stmt = tx.prepare("SELECT rowid, term FROM terms WHERE term_norm IS NULL")?;
+
+            let rows = stmt.query_map([], |row| {
+                Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
+            })?;
+
+            let mut update_stmt = tx.prepare("UPDATE terms SET term_norm = ?1 WHERE rowid = ?2")?;
+
+            for row in rows {
+                let (rowid, term) = row?;
+                let norm = normalize_term(&term); // your Unicode normalizer
+                update_stmt.execute((&norm, rowid))?;
+            }
+        }
+        tx.commit()?;
+        // 5 sec
+
+        self.conn.execute_batch(
+            "
+            CREATE INDEX IF NOT EXISTS terms_term_norm ON terms (term_norm);
+            ",
+        )?;
+        // 1 sec
+
+        Ok(())
+    }
+
     pub fn import_dictionary_zip_file(
         &mut self,
         zip_file: PathBuf,

src-core/src/service.rs

@@ -43,7 +43,7 @@ impl AppService {
 
     pub fn get_yomitan_writer(&self) -> Result<YomitanWriter, CJDicError> {
         let conn = Connection::open(self.db.dir.join("yomitan.db"))?;
-        let writer = YomitanWriter::new(conn)?;
+        let mut writer = YomitanWriter::new(conn)?;
         writer.create_schema()?;
         Ok(writer)
     }