full text search and UI button

src-core/src/db.rs

@@ -18,13 +18,14 @@ pub use yomitan::YomitanRow;
 mod yomitan_writer;
 pub use yomitan_writer::{YomitanProgress, YomitanWriter, YomitanZipImportResult, ZipSource};
 
-use crate::CJDicError;
+use crate::{CJDicError, tokenizer::Tokenizer};
 
 #[derive(Clone)]
 pub(crate) struct Database {
     pub(crate) conn: Arc<Mutex<Connection>>,
     pub(crate) dir: PathBuf,
     is_db_attached: EnumMap<DbChild, Arc<AtomicBool>>,
+    tokenizer: Tokenizer,
 }
 
 #[derive(Debug, Enum, Clone, Copy)]
@@ -52,7 +53,7 @@ pub(crate) static DBSCHEMA: LazyLock<EnumMap<DbChild, &str>> = LazyLock::new(||
 });
 
 impl Database {
-    pub fn new(db_dir: impl AsRef<Path>) -> Result<Self, CJDicError> {
+    pub fn new(db_dir: impl AsRef<Path>, tokenizer: Tokenizer) -> Result<Self, CJDicError> {
         let dir = db_dir.as_ref().to_path_buf();
         let conn = Connection::open(dir.join(USER_DBFILE))?;
 
@@ -73,6 +74,7 @@ impl Database {
                 DbChild::YomitanGlossary => Arc::new(AtomicBool::new(false)),
                 DbChild::Search => Arc::new(AtomicBool::new(false)),
             },
+            tokenizer,
         })
     }
 
@@ -80,7 +82,10 @@ impl Database {
         self.ensure_db_attached(DbChild::Yomitan)?;
         self.ensure_db_attached(DbChild::YomitanGlossary)?;
         self.ensure_db_attached(DbChild::Search)?;
-        Ok(yomitan::YomitanDatabase::new(self.clone()))
+        Ok(yomitan::YomitanDatabase::new(
+            self.clone(),
+            self.tokenizer.clone(),
+        ))
     }
 
     fn ensure_db_attached(&self, db_child: DbChild) -> Result<(), CJDicError> {

src-core/src/db/search.rs

@@ -1,5 +1,3 @@
-use std::collections::HashSet;
-
 use rusqlite::{Connection, Transaction, params};
 use wana_kana::utils::katakana_to_hiragana;
 
@@ -114,8 +112,6 @@ impl<'a> SearchDatabase<'a> {
                 DBSCHEMA[DbChild::Search]
             ))?;
 
-            let exclude_pos =
-                HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
             let mut stmt_write_ft = tx.prepare(&format!(
                 "INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)",
                 DBSCHEMA[DbChild::Search]
@@ -130,28 +126,7 @@ impl<'a> SearchDatabase<'a> {
                     max_score
                 ])?;
 
-                let mut segmented: Vec<String> = vec![];
+                stmt_write_ft.execute(params![id, tokenizer.ja_normalize(term).join(" ")])?;
-                for t in tokenizer.tokenize(term) {
-                    let mut base = t.surface;
-                    let mut pos = None;
-
-                    for (i, dt) in t.details.iter().enumerate() {
-                        match i {
-                            0 => base = dt.to_string(),
-                            6 => pos = Some(dt),
-                            _ => {}
-                        }
-                    }
-
-                    if let Some(p) = pos {
-                        if exclude_pos.contains(p.as_str()) {
-                            continue;
-                        }
-                    }
-                    segmented.push(base);
-                }
-
-                stmt_write_ft.execute(params![id, segmented.join(" ")])?;
 
                 if i % 10_000 == 0 {
                     progress_callback(YomitanProgress {

src-core/src/db/yomitan.rs

@@ -5,6 +5,7 @@ use zstd::bulk::Decompressor;
 use crate::{
     CJDicError, Timer,
     db::{Database, yomitan_writer::normalize_term},
+    tokenizer::Tokenizer,
 };
 
 #[derive(Serialize, Debug, Clone)]
@@ -22,11 +23,12 @@ pub struct YomitanRow {
 
 pub struct YomitanDatabase {
     db: Database,
+    tokenizer: Tokenizer,
 }
 
 impl YomitanDatabase {
-    pub(crate) fn new(db: Database) -> Self {
+    pub(crate) fn new(db: Database, tokenizer: Tokenizer) -> Self {
-        Self { db }
+        Self { db, tokenizer }
     }
 
     fn decompressor(&self) -> Result<Decompressor<'_>, CJDicError> {
@@ -65,38 +67,77 @@ impl YomitanDatabase {
         // max decompressed: 90KB  (compression ratio on largest: ~2.5x)
         const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom
 
-        let and_or = if q_term == q_reading { "OR" } else { "AND" };
+        let is_fts = q_term.starts_with('~');
+        let q_term = if is_fts {
+            normalize_term(q_term)
+        } else {
+            let mut new_t = String::new();
+            let mut new_seg = String::new();
 
-        let mut new_t = String::new();
+            for c in q_term.chars() {
-        let mut new_seg = String::new();
+                match c {
-
+                    '*' | '?' | '[' | ']' | '\\' => {
-        for c in q_term.chars() {
+                        new_t.push_str(&normalize_term(&new_seg));
-            match c {
+                        new_seg.clear();
-                '*' | '?' | '[' | ']' | '\\' => {
+                        new_t.push(c)
-                    new_t.push_str(&normalize_term(&new_seg));
+                    }
-                    new_seg.clear();
+                    _ => new_seg.push(c),
-                    new_t.push(c)
                 }
-                _ => new_seg.push(c),
             }
-        }
 
-        new_t.push_str(&normalize_term(&new_seg));
+            new_t.push_str(&normalize_term(&new_seg));
-        let q_term_norm = new_t;
+            new_t
+        };
 
-        let eq1 = if q_term_norm.len() > 0 { "GLOB" } else { "=" };
+        let and_or = if q_term == q_reading { "OR" } else { "AND" };
-        let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
 
         let conn = self.db.conn.lock()?;
 
-        let mut stmt = conn.prepare(&format!(
+        conn.execute_batch(
             "
-            WITH s AS (
+            CREATE TEMP TABLE IF NOT EXISTS search_ids (term_id INTEGER);
-                SELECT * FROM search.terms
+            DELETE FROM search_ids;
+            ",
+        )?;
+
+        if is_fts {
+            let q = self.tokenizer.ja_normalize(q_term.to_string()).join(" ");
+            println!("fts: {q_term} => {q}");
+
+            if q.len() > 0 {
+                conn.execute(
+                    &format!(
+                        "
+                    INSERT INTO search_ids (term_id)
+                    SELECT rowid FROM search.terms_ft
+                    WHERE term MATCH ?1
+                    ORDER BY rank
+                    LIMIT ?2 OFFSET ?3
+                "
+                    ),
+                    params![q, limit, offset],
+                )?;
+            }
+        } else {
+            let eq1 = if q_term.len() > 0 { "GLOB" } else { "=" };
+            let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+
+            conn.execute(
+                &format!(
+                    "
+                INSERT INTO search_ids (term_id)
+                SELECT id FROM search.terms
                 WHERE term {eq1} ?1 {and_or} reading {eq2} ?2
                 ORDER BY max_score DESC
                 LIMIT ?3 OFFSET ?4
-            )
+            "
+                ),
+                params![q_term, q_reading, limit, offset],
+            )?;
+        }
+
+        let mut stmt = conn.prepare(&format!(
+            "
             SELECT
                 t.term,
                 t.reading,
@@ -107,8 +148,8 @@ impl YomitanDatabase {
                 t.sequence,
                 COALESCE(tt.tags,  '')  AS term_tags,
                 d.title                 AS dict_title
-            FROM s
+            FROM search_ids                 s
-            JOIN yomitan.terms              t  ON t.id  = s.id
+            JOIN yomitan.terms              t  ON t.id  = s.term_id
             LEFT JOIN yomitan.dictionaries  d  ON d.id  = t.dict_id
             LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
             LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
@@ -116,7 +157,7 @@ impl YomitanDatabase {
             LEFT JOIN glossary.glossaries   g  ON g.id  = t.glossary_id
         "
         ))?;
-        let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| {
+        let rows = stmt.query_map([], |r| {
             Ok((
                 YomitanRow {
                     term: r.get(0)?,

src-core/src/lib.rs

@@ -9,7 +9,7 @@ mod util;
 
 pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource};
 pub use error::CJDicError;
-pub use service::{AppService, SqlParam};
+pub use service::{AppService, SqlParam, YomitanDictEntry};
 pub use tokenizer::TokenizeSegment;
 pub use util::Timer;
 

src-core/src/service.rs

@@ -16,6 +16,14 @@ use crate::{
     tokenizer::{TokenizeSegment, Tokenizer},
 };
 
+#[derive(Serialize, Debug)]
+pub struct YomitanDictEntry {
+    pub title: String,
+    pub bundle_name: String,
+    pub revision: String,
+    pub lang: String,
+}
+
 #[derive(Serialize, Debug, Clone)]
 pub struct LoadYomitanZipDirResult {
     pub new_dicts: Vec<String>,
@@ -42,8 +50,9 @@ impl AppService {
         db_dir: impl AsRef<Path>,
         vibrato_dic: impl AsRef<Path>,
     ) -> Result<Self, CJDicError> {
-        let db = Database::new(db_dir)?;
         let tokenizer = Tokenizer::new(vibrato_dic)?;
+        let db = Database::new(db_dir, tokenizer.clone())?;
+
         Ok(Self { db, tokenizer })
     }
 
@@ -125,6 +134,35 @@ impl AppService {
         Ok(writer)
     }
 
+    pub fn list_yomitan_dict(&self) -> Result<Vec<YomitanDictEntry>, CJDicError> {
+        let db_path = self.db.dir.join(DBFILE[DbChild::Yomitan]);
+        let mut out = vec![];
+
+        if db_path.exists() {
+            let conn = Connection::open(&db_path)?;
+            let mut stmt = conn.prepare(
+                "
+            SELECT title, bundle_name, revision, lang
+            FROM dictionaries
+            WHERE bundle_name IS NOT NULL",
+            )?;
+            let rows = stmt.query_map([], |r| {
+                Ok(YomitanDictEntry {
+                    title: r.get(0)?,
+                    bundle_name: r.get(1)?,
+                    revision: r.get(2)?,
+                    lang: r.get(3)?,
+                })
+            })?;
+
+            for r in rows {
+                out.push(r?);
+            }
+        }
+
+        Ok(out)
+    }
+
     pub fn load_yomitan_zip_dir<Z, LoadCallback, ImportCallback>(
         &self,
         zip_dir: Vec<Z>,

src-core/src/tokenizer.rs

@@ -1,4 +1,4 @@
-use std::{fs::File, io::BufReader, path::Path};
+use std::{collections::HashSet, fs::File, io::BufReader, path::Path};
 
 use serde::Serialize;
 
@@ -25,6 +25,7 @@ pub struct TokenizeSegment {
 #[derive(Clone)]
 pub struct Tokenizer {
     ja_tokenizer: vibrato_rkyv::Tokenizer,
+    ja_exclude_pos: HashSet<String>,
 }
 
 impl Tokenizer {
@@ -33,6 +34,9 @@ impl Tokenizer {
         let dictionary = vibrato_rkyv::Dictionary::read(reader)?;
         Ok(Self {
             ja_tokenizer: vibrato_rkyv::Tokenizer::new(dictionary),
+            ja_exclude_pos: HashSet::from(
+                ["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()),
+            ),
         })
     }
 
@@ -56,4 +60,34 @@ impl Tokenizer {
 
         output
     }
+
+    pub fn ja_normalize(&self, text: String) -> Vec<String> {
+        let mut segmented: Vec<String> = vec![];
+        for t in self.ja_tokenize(text) {
+            let mut surface = t.surface;
+            let mut pos = None;
+
+            for (i, dt) in t.details.iter().enumerate() {
+                match i {
+                    0 => pos = Some(dt),
+                    6 => {
+                        let base = dt.to_string();
+                        if base != "*" {
+                            surface = base
+                        }
+                    }
+                    _ => {}
+                }
+            }
+
+            if let Some(p) = pos {
+                if self.ja_exclude_pos.contains(p.as_str()) {
+                    continue;
+                }
+            }
+            segmented.push(surface);
+        }
+
+        segmented
+    }
 }

src-react/src/components/LoadingDialog.tsx

@@ -7,6 +7,13 @@ import { listen } from "@tauri-apps/api/event";
 import { supabase } from "../lib/supabaseClient";
 
 interface YomitanDictEntry {
+  title: string;
+  bundle_name: string;
+  revision: string;
+  lang: string;
+}
+
+interface YomitanDownloadDictEntry {
   url: string;
   filepath: string;
 }
@@ -66,13 +73,17 @@ function LoadingDialog() {
       }
     });
     async function downloadAssets(lang = "ja") {
-      const dicts: YomitanDictEntry[] = [];
+      const existing_dicts =
+        await invoke<YomitanDictEntry[]>("list_yomitan_dict");
+
+      const dicts: YomitanDownloadDictEntry[] = [];
 
       if (lang === "ja") {
-        {
+        const filename = "Pixiv.zip";
-          const filename = "Pixiv.zip";
+        const filepath = `yomitan/${lang}/${filename}`;
-          const filepath = `yomitan/${lang}/${filename}`;
+        const existing = existing_dicts.find((d) => d.bundle_name === filename);
-
+        if (!existing) {
+          // Maybe check revision update
           dicts.push({
             filepath,
             url:
@@ -84,6 +95,11 @@ function LoadingDialog() {
                 return "";
               })) || "",
           });
+        } else {
+          dicts.push({
+            filepath,
+            url: "",
+          });
         }
 
         {

src-react/src/pages/SearchPage.tsx

@@ -115,6 +115,9 @@ function SearchPage() {
     if (isAutoKana) {
       norm_q = norm_q.replace(/n$/i, "ん").replace(/[a-z]$/i, "");
     }
+
+    norm_q = norm_q.replace(/＊/g, "*").replace(/？/g, "?").replace(/^〜/, "~");
+
     if (!norm_q) return;
 
     const ender =
@@ -391,6 +394,9 @@ function SearchPage() {
                         {"）"}
                       </>
                     ) : null}
+                    <button onClick={() => onTermClicked("~" + term)}>
+                      🔗
+                    </button>
                     <span>【{dict_title}】</span>
                     <span>{JSONdumpClean(it)}</span>
                   </Card.Header>

src-tauri/src/command.rs

@@ -5,7 +5,8 @@ use std::{
 };
 
 use cjdic2_core::{
-    AppService, CJDicError, SqlParam, TokenizeSegment, YomitanRow, YomitanZipImportResult,
+    AppService, CJDicError, SqlParam, TokenizeSegment, YomitanDictEntry, YomitanRow,
+    YomitanZipImportResult,
 };
 use serde::{Deserialize, Serialize};
 use tauri::{AppHandle, Emitter, Manager};
@@ -13,8 +14,15 @@ use tauri_plugin_http::reqwest::Client;
 use tokio::io::AsyncWriteExt;
 use zip::ZipArchive;
 
+#[tauri::command]
+pub async fn list_yomitan_dict(
+    state: tauri::State<'_, AppService>,
+) -> Result<Vec<YomitanDictEntry>, CJDicError> {
+    state.list_yomitan_dict()
+}
+
 #[derive(Deserialize)]
-pub struct YomitanDictEntry {
+pub struct YomitanDownloadDictEntry {
     url: String,
     filepath: String,
 }
@@ -22,7 +30,7 @@ pub struct YomitanDictEntry {
 #[tauri::command]
 pub async fn init_yomitan(
     app: AppHandle,
-    dicts: Vec<YomitanDictEntry>,
+    dicts: Vec<YomitanDownloadDictEntry>,
     lang: String,
     state: tauri::State<'_, AppService>,
 ) -> Result<(), CJDicError> {

src-tauri/src/lib.rs

@@ -47,6 +47,7 @@ pub fn run() {
             Ok(())
         })
         .invoke_handler(tauri::generate_handler![
+            list_yomitan_dict,
             init_yomitan,
             import_yomitan_dict,
             remove_yomitan_dict,