full text search and UI button

src-core/src/db/yomitan.rs

@@ -5,6 +5,7 @@ use zstd::bulk::Decompressor;
 use crate::{
     CJDicError, Timer,
     db::{Database, yomitan_writer::normalize_term},
+    tokenizer::Tokenizer,
 };
 
 #[derive(Serialize, Debug, Clone)]
@@ -22,11 +23,12 @@ pub struct YomitanRow {
 
 pub struct YomitanDatabase {
     db: Database,
+    tokenizer: Tokenizer,
 }
 
 impl YomitanDatabase {
-    pub(crate) fn new(db: Database) -> Self {
-        Self { db }
+    pub(crate) fn new(db: Database, tokenizer: Tokenizer) -> Self {
+        Self { db, tokenizer }
     }
 
     fn decompressor(&self) -> Result<Decompressor<'_>, CJDicError> {
@@ -65,38 +67,77 @@ impl YomitanDatabase {
         // max decompressed: 90KB  (compression ratio on largest: ~2.5x)
         const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom
 
-        let and_or = if q_term == q_reading { "OR" } else { "AND" };
+        let is_fts = q_term.starts_with('~');
+        let q_term = if is_fts {
+            normalize_term(q_term)
+        } else {
+            let mut new_t = String::new();
+            let mut new_seg = String::new();
 
-        let mut new_t = String::new();
-        let mut new_seg = String::new();
-
-        for c in q_term.chars() {
-            match c {
-                '*' | '?' | '[' | ']' | '\\' => {
-                    new_t.push_str(&normalize_term(&new_seg));
-                    new_seg.clear();
-                    new_t.push(c)
+            for c in q_term.chars() {
+                match c {
+                    '*' | '?' | '[' | ']' | '\\' => {
+                        new_t.push_str(&normalize_term(&new_seg));
+                        new_seg.clear();
+                        new_t.push(c)
+                    }
+                    _ => new_seg.push(c),
                 }
-                _ => new_seg.push(c),
             }
-        }
 
-        new_t.push_str(&normalize_term(&new_seg));
-        let q_term_norm = new_t;
+            new_t.push_str(&normalize_term(&new_seg));
+            new_t
+        };
 
-        let eq1 = if q_term_norm.len() > 0 { "GLOB" } else { "=" };
-        let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+        let and_or = if q_term == q_reading { "OR" } else { "AND" };
 
         let conn = self.db.conn.lock()?;
 
-        let mut stmt = conn.prepare(&format!(
+        conn.execute_batch(
             "
-            WITH s AS (
-                SELECT * FROM search.terms
+            CREATE TEMP TABLE IF NOT EXISTS search_ids (term_id INTEGER);
+            DELETE FROM search_ids;
+            ",
+        )?;
+
+        if is_fts {
+            let q = self.tokenizer.ja_normalize(q_term.to_string()).join(" ");
+            println!("fts: {q_term} => {q}");
+
+            if q.len() > 0 {
+                conn.execute(
+                    &format!(
+                        "
+                    INSERT INTO search_ids (term_id)
+                    SELECT rowid FROM search.terms_ft
+                    WHERE term MATCH ?1
+                    ORDER BY rank
+                    LIMIT ?2 OFFSET ?3
+                "
+                    ),
+                    params![q, limit, offset],
+                )?;
+            }
+        } else {
+            let eq1 = if q_term.len() > 0 { "GLOB" } else { "=" };
+            let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+
+            conn.execute(
+                &format!(
+                    "
+                INSERT INTO search_ids (term_id)
+                SELECT id FROM search.terms
                 WHERE term {eq1} ?1 {and_or} reading {eq2} ?2
                 ORDER BY max_score DESC
                 LIMIT ?3 OFFSET ?4
-            )
+            "
+                ),
+                params![q_term, q_reading, limit, offset],
+            )?;
+        }
+
+        let mut stmt = conn.prepare(&format!(
+            "
             SELECT
                 t.term,
                 t.reading,
@@ -107,8 +148,8 @@ impl YomitanDatabase {
                 t.sequence,
                 COALESCE(tt.tags,  '')  AS term_tags,
                 d.title                 AS dict_title
-            FROM s
-            JOIN yomitan.terms              t  ON t.id  = s.id
+            FROM search_ids                 s
+            JOIN yomitan.terms              t  ON t.id  = s.term_id
             LEFT JOIN yomitan.dictionaries  d  ON d.id  = t.dict_id
             LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
             LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
@@ -116,7 +157,7 @@ impl YomitanDatabase {
             LEFT JOIN glossary.glossaries   g  ON g.id  = t.glossary_id
         "
         ))?;
-        let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| {
+        let rows = stmt.query_map([], |r| {
             Ok((
                 YomitanRow {
                     term: r.get(0)?,