full text search and UI button

src-core/src/db/search.rs

@@ -1,5 +1,3 @@
-use std::collections::HashSet;
-
 use rusqlite::{Connection, Transaction, params};
 use wana_kana::utils::katakana_to_hiragana;
 
@@ -114,8 +112,6 @@ impl<'a> SearchDatabase<'a> {
                 DBSCHEMA[DbChild::Search]
             ))?;
 
-            let exclude_pos =
-                HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
             let mut stmt_write_ft = tx.prepare(&format!(
                 "INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)",
                 DBSCHEMA[DbChild::Search]
@@ -130,28 +126,7 @@ impl<'a> SearchDatabase<'a> {
                     max_score
                 ])?;
 
-                let mut segmented: Vec<String> = vec![];
-                for t in tokenizer.tokenize(term) {
-                    let mut base = t.surface;
-                    let mut pos = None;
-
-                    for (i, dt) in t.details.iter().enumerate() {
-                        match i {
-                            0 => base = dt.to_string(),
-                            6 => pos = Some(dt),
-                            _ => {}
-                        }
-                    }
-
-                    if let Some(p) = pos {
-                        if exclude_pos.contains(p.as_str()) {
-                            continue;
-                        }
-                    }
-                    segmented.push(base);
-                }
-
-                stmt_write_ft.execute(params![id, segmented.join(" ")])?;
+                stmt_write_ft.execute(params![id, tokenizer.ja_normalize(term).join(" ")])?;
 
                 if i % 10_000 == 0 {
                     progress_callback(YomitanProgress {

src-core/src/db/yomitan.rs

@@ -5,6 +5,7 @@ use zstd::bulk::Decompressor;
 use crate::{
     CJDicError, Timer,
     db::{Database, yomitan_writer::normalize_term},
+    tokenizer::Tokenizer,
 };
 
 #[derive(Serialize, Debug, Clone)]
@@ -22,11 +23,12 @@ pub struct YomitanRow {
 
 pub struct YomitanDatabase {
     db: Database,
+    tokenizer: Tokenizer,
 }
 
 impl YomitanDatabase {
-    pub(crate) fn new(db: Database) -> Self {
-        Self { db }
+    pub(crate) fn new(db: Database, tokenizer: Tokenizer) -> Self {
+        Self { db, tokenizer }
     }
 
     fn decompressor(&self) -> Result<Decompressor<'_>, CJDicError> {
@@ -65,38 +67,77 @@ impl YomitanDatabase {
         // max decompressed: 90KB  (compression ratio on largest: ~2.5x)
         const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom
 
-        let and_or = if q_term == q_reading { "OR" } else { "AND" };
+        let is_fts = q_term.starts_with('~');
+        let q_term = if is_fts {
+            normalize_term(q_term)
+        } else {
+            let mut new_t = String::new();
+            let mut new_seg = String::new();
 
-        let mut new_t = String::new();
-        let mut new_seg = String::new();
-
-        for c in q_term.chars() {
-            match c {
-                '*' | '?' | '[' | ']' | '\\' => {
-                    new_t.push_str(&normalize_term(&new_seg));
-                    new_seg.clear();
-                    new_t.push(c)
+            for c in q_term.chars() {
+                match c {
+                    '*' | '?' | '[' | ']' | '\\' => {
+                        new_t.push_str(&normalize_term(&new_seg));
+                        new_seg.clear();
+                        new_t.push(c)
+                    }
+                    _ => new_seg.push(c),
                 }
-                _ => new_seg.push(c),
             }
-        }
 
-        new_t.push_str(&normalize_term(&new_seg));
-        let q_term_norm = new_t;
+            new_t.push_str(&normalize_term(&new_seg));
+            new_t
+        };
 
-        let eq1 = if q_term_norm.len() > 0 { "GLOB" } else { "=" };
-        let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+        let and_or = if q_term == q_reading { "OR" } else { "AND" };
 
         let conn = self.db.conn.lock()?;
 
-        let mut stmt = conn.prepare(&format!(
+        conn.execute_batch(
             "
-            WITH s AS (
-                SELECT * FROM search.terms
+            CREATE TEMP TABLE IF NOT EXISTS search_ids (term_id INTEGER);
+            DELETE FROM search_ids;
+            ",
+        )?;
+
+        if is_fts {
+            let q = self.tokenizer.ja_normalize(q_term.to_string()).join(" ");
+            println!("fts: {q_term} => {q}");
+
+            if q.len() > 0 {
+                conn.execute(
+                    &format!(
+                        "
+                    INSERT INTO search_ids (term_id)
+                    SELECT rowid FROM search.terms_ft
+                    WHERE term MATCH ?1
+                    ORDER BY rank
+                    LIMIT ?2 OFFSET ?3
+                "
+                    ),
+                    params![q, limit, offset],
+                )?;
+            }
+        } else {
+            let eq1 = if q_term.len() > 0 { "GLOB" } else { "=" };
+            let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+
+            conn.execute(
+                &format!(
+                    "
+                INSERT INTO search_ids (term_id)
+                SELECT id FROM search.terms
                 WHERE term {eq1} ?1 {and_or} reading {eq2} ?2
                 ORDER BY max_score DESC
                 LIMIT ?3 OFFSET ?4
-            )
+            "
+                ),
+                params![q_term, q_reading, limit, offset],
+            )?;
+        }
+
+        let mut stmt = conn.prepare(&format!(
+            "
             SELECT
                 t.term,
                 t.reading,
@@ -107,8 +148,8 @@ impl YomitanDatabase {
                 t.sequence,
                 COALESCE(tt.tags,  '')  AS term_tags,
                 d.title                 AS dict_title
-            FROM s
-            JOIN yomitan.terms              t  ON t.id  = s.id
+            FROM search_ids                 s
+            JOIN yomitan.terms              t  ON t.id  = s.term_id
             LEFT JOIN yomitan.dictionaries  d  ON d.id  = t.dict_id
             LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
             LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
@@ -116,7 +157,7 @@ impl YomitanDatabase {
             LEFT JOIN glossary.glossaries   g  ON g.id  = t.glossary_id
         "
         ))?;
-        let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| {
+        let rows = stmt.query_map([], |r| {
             Ok((
                 YomitanRow {
                     term: r.get(0)?,