From f0cafa450d628f56afa89cc432acf31bdd80ee24 Mon Sep 17 00:00:00 2001 From: polv Date: Mon, 16 Mar 2026 00:05:25 +0700 Subject: [PATCH] full text search and UI button --- src-core/src/db.rs | 11 ++- src-core/src/db/search.rs | 27 +------ src-core/src/db/yomitan.rs | 91 ++++++++++++++++------ src-core/src/lib.rs | 2 +- src-core/src/service.rs | 40 +++++++++- src-core/src/tokenizer.rs | 36 ++++++++- src-react/src/components/LoadingDialog.tsx | 26 +++++-- src-react/src/pages/SearchPage.tsx | 6 ++ src-tauri/src/command.rs | 14 +++- src-tauri/src/lib.rs | 1 + 10 files changed, 189 insertions(+), 65 deletions(-) diff --git a/src-core/src/db.rs b/src-core/src/db.rs index 31385c1..e5a2c18 100644 --- a/src-core/src/db.rs +++ b/src-core/src/db.rs @@ -18,13 +18,14 @@ pub use yomitan::YomitanRow; mod yomitan_writer; pub use yomitan_writer::{YomitanProgress, YomitanWriter, YomitanZipImportResult, ZipSource}; -use crate::CJDicError; +use crate::{CJDicError, tokenizer::Tokenizer}; #[derive(Clone)] pub(crate) struct Database { pub(crate) conn: Arc>, pub(crate) dir: PathBuf, is_db_attached: EnumMap>, + tokenizer: Tokenizer, } #[derive(Debug, Enum, Clone, Copy)] @@ -52,7 +53,7 @@ pub(crate) static DBSCHEMA: LazyLock> = LazyLock::new(|| }); impl Database { - pub fn new(db_dir: impl AsRef) -> Result { + pub fn new(db_dir: impl AsRef, tokenizer: Tokenizer) -> Result { let dir = db_dir.as_ref().to_path_buf(); let conn = Connection::open(dir.join(USER_DBFILE))?; @@ -73,6 +74,7 @@ impl Database { DbChild::YomitanGlossary => Arc::new(AtomicBool::new(false)), DbChild::Search => Arc::new(AtomicBool::new(false)), }, + tokenizer, }) } @@ -80,7 +82,10 @@ impl Database { self.ensure_db_attached(DbChild::Yomitan)?; self.ensure_db_attached(DbChild::YomitanGlossary)?; self.ensure_db_attached(DbChild::Search)?; - Ok(yomitan::YomitanDatabase::new(self.clone())) + Ok(yomitan::YomitanDatabase::new( + self.clone(), + self.tokenizer.clone(), + )) } fn ensure_db_attached(&self, db_child: DbChild) -> Result<(), CJDicError> { diff --git a/src-core/src/db/search.rs b/src-core/src/db/search.rs index ecc40e1..2347325 100644 --- a/src-core/src/db/search.rs +++ b/src-core/src/db/search.rs @@ -1,5 +1,3 @@ -use std::collections::HashSet; - use rusqlite::{Connection, Transaction, params}; use wana_kana::utils::katakana_to_hiragana; @@ -114,8 +112,6 @@ impl<'a> SearchDatabase<'a> { DBSCHEMA[DbChild::Search] ))?; - let exclude_pos = - HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string())); let mut stmt_write_ft = tx.prepare(&format!( "INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)", DBSCHEMA[DbChild::Search] @@ -130,28 +126,7 @@ impl<'a> SearchDatabase<'a> { max_score ])?; - let mut segmented: Vec = vec![]; - for t in tokenizer.tokenize(term) { - let mut base = t.surface; - let mut pos = None; - - for (i, dt) in t.details.iter().enumerate() { - match i { - 0 => base = dt.to_string(), - 6 => pos = Some(dt), - _ => {} - } - } - - if let Some(p) = pos { - if exclude_pos.contains(p.as_str()) { - continue; - } - } - segmented.push(base); - } - - stmt_write_ft.execute(params![id, segmented.join(" ")])?; + stmt_write_ft.execute(params![id, tokenizer.ja_normalize(term).join(" ")])?; if i % 10_000 == 0 { progress_callback(YomitanProgress { diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs index fa6b4d0..002e7da 100644 --- a/src-core/src/db/yomitan.rs +++ b/src-core/src/db/yomitan.rs @@ -5,6 +5,7 @@ use zstd::bulk::Decompressor; use crate::{ CJDicError, Timer, db::{Database, yomitan_writer::normalize_term}, + tokenizer::Tokenizer, }; #[derive(Serialize, Debug, Clone)] @@ -22,11 +23,12 @@ pub struct YomitanRow { pub struct YomitanDatabase { db: Database, + tokenizer: Tokenizer, } impl YomitanDatabase { - pub(crate) fn new(db: Database) -> Self { - Self { db } + pub(crate) fn new(db: Database, tokenizer: Tokenizer) -> Self { + Self { db, tokenizer } } fn decompressor(&self) -> Result, CJDicError> { @@ -65,38 +67,77 @@ impl YomitanDatabase { // max decompressed: 90KB (compression ratio on largest: ~2.5x) const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom - let and_or = if q_term == q_reading { "OR" } else { "AND" }; + let is_fts = q_term.starts_with('~'); + let q_term = if is_fts { + normalize_term(q_term) + } else { + let mut new_t = String::new(); + let mut new_seg = String::new(); - let mut new_t = String::new(); - let mut new_seg = String::new(); - - for c in q_term.chars() { - match c { - '*' | '?' | '[' | ']' | '\\' => { - new_t.push_str(&normalize_term(&new_seg)); - new_seg.clear(); - new_t.push(c) + for c in q_term.chars() { + match c { + '*' | '?' | '[' | ']' | '\\' => { + new_t.push_str(&normalize_term(&new_seg)); + new_seg.clear(); + new_t.push(c) + } + _ => new_seg.push(c), } - _ => new_seg.push(c), } - } - new_t.push_str(&normalize_term(&new_seg)); - let q_term_norm = new_t; + new_t.push_str(&normalize_term(&new_seg)); + new_t + }; - let eq1 = if q_term_norm.len() > 0 { "GLOB" } else { "=" }; - let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" }; + let and_or = if q_term == q_reading { "OR" } else { "AND" }; let conn = self.db.conn.lock()?; - let mut stmt = conn.prepare(&format!( + conn.execute_batch( " - WITH s AS ( - SELECT * FROM search.terms + CREATE TEMP TABLE IF NOT EXISTS search_ids (term_id INTEGER); + DELETE FROM search_ids; + ", + )?; + + if is_fts { + let q = self.tokenizer.ja_normalize(q_term.to_string()).join(" "); + println!("fts: {q_term} => {q}"); + + if q.len() > 0 { + conn.execute( + &format!( + " + INSERT INTO search_ids (term_id) + SELECT rowid FROM search.terms_ft + WHERE term MATCH ?1 + ORDER BY rank + LIMIT ?2 OFFSET ?3 + " + ), + params![q, limit, offset], + )?; + } + } else { + let eq1 = if q_term.len() > 0 { "GLOB" } else { "=" }; + let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" }; + + conn.execute( + &format!( + " + INSERT INTO search_ids (term_id) + SELECT id FROM search.terms WHERE term {eq1} ?1 {and_or} reading {eq2} ?2 ORDER BY max_score DESC LIMIT ?3 OFFSET ?4 - ) + " + ), + params![q_term, q_reading, limit, offset], + )?; + } + + let mut stmt = conn.prepare(&format!( + " SELECT t.term, t.reading, @@ -107,8 +148,8 @@ impl YomitanDatabase { t.sequence, COALESCE(tt.tags, '') AS term_tags, d.title AS dict_title - FROM s - JOIN yomitan.terms t ON t.id = s.id + FROM search_ids s + JOIN yomitan.terms t ON t.id = s.term_id LEFT JOIN yomitan.dictionaries d ON d.id = t.dict_id LEFT JOIN yomitan.def_tag_sets dt ON dt.id = t.def_tags_id LEFT JOIN yomitan.rule_sets r ON r.id = t.rules_id @@ -116,7 +157,7 @@ impl YomitanDatabase { LEFT JOIN glossary.glossaries g ON g.id = t.glossary_id " ))?; - let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| { + let rows = stmt.query_map([], |r| { Ok(( YomitanRow { term: r.get(0)?, diff --git a/src-core/src/lib.rs b/src-core/src/lib.rs index ff200c6..7578633 100644 --- a/src-core/src/lib.rs +++ b/src-core/src/lib.rs @@ -9,7 +9,7 @@ mod util; pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource}; pub use error::CJDicError; -pub use service::{AppService, SqlParam}; +pub use service::{AppService, SqlParam, YomitanDictEntry}; pub use tokenizer::TokenizeSegment; pub use util::Timer; diff --git a/src-core/src/service.rs b/src-core/src/service.rs index b86d5d4..7a7de9f 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -16,6 +16,14 @@ use crate::{ tokenizer::{TokenizeSegment, Tokenizer}, }; +#[derive(Serialize, Debug)] +pub struct YomitanDictEntry { + pub title: String, + pub bundle_name: String, + pub revision: String, + pub lang: String, +} + #[derive(Serialize, Debug, Clone)] pub struct LoadYomitanZipDirResult { pub new_dicts: Vec, @@ -42,8 +50,9 @@ impl AppService { db_dir: impl AsRef, vibrato_dic: impl AsRef, ) -> Result { - let db = Database::new(db_dir)?; let tokenizer = Tokenizer::new(vibrato_dic)?; + let db = Database::new(db_dir, tokenizer.clone())?; + Ok(Self { db, tokenizer }) } @@ -125,6 +134,35 @@ impl AppService { Ok(writer) } + pub fn list_yomitan_dict(&self) -> Result, CJDicError> { + let db_path = self.db.dir.join(DBFILE[DbChild::Yomitan]); + let mut out = vec![]; + + if db_path.exists() { + let conn = Connection::open(&db_path)?; + let mut stmt = conn.prepare( + " + SELECT title, bundle_name, revision, lang + FROM dictionaries + WHERE bundle_name IS NOT NULL", + )?; + let rows = stmt.query_map([], |r| { + Ok(YomitanDictEntry { + title: r.get(0)?, + bundle_name: r.get(1)?, + revision: r.get(2)?, + lang: r.get(3)?, + }) + })?; + + for r in rows { + out.push(r?); + } + } + + Ok(out) + } + pub fn load_yomitan_zip_dir( &self, zip_dir: Vec, diff --git a/src-core/src/tokenizer.rs b/src-core/src/tokenizer.rs index 64e760c..4abc007 100644 --- a/src-core/src/tokenizer.rs +++ b/src-core/src/tokenizer.rs @@ -1,4 +1,4 @@ -use std::{fs::File, io::BufReader, path::Path}; +use std::{collections::HashSet, fs::File, io::BufReader, path::Path}; use serde::Serialize; @@ -25,6 +25,7 @@ pub struct TokenizeSegment { #[derive(Clone)] pub struct Tokenizer { ja_tokenizer: vibrato_rkyv::Tokenizer, + ja_exclude_pos: HashSet, } impl Tokenizer { @@ -33,6 +34,9 @@ impl Tokenizer { let dictionary = vibrato_rkyv::Dictionary::read(reader)?; Ok(Self { ja_tokenizer: vibrato_rkyv::Tokenizer::new(dictionary), + ja_exclude_pos: HashSet::from( + ["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()), + ), }) } @@ -56,4 +60,34 @@ impl Tokenizer { output } + + pub fn ja_normalize(&self, text: String) -> Vec { + let mut segmented: Vec = vec![]; + for t in self.ja_tokenize(text) { + let mut surface = t.surface; + let mut pos = None; + + for (i, dt) in t.details.iter().enumerate() { + match i { + 0 => pos = Some(dt), + 6 => { + let base = dt.to_string(); + if base != "*" { + surface = base + } + } + _ => {} + } + } + + if let Some(p) = pos { + if self.ja_exclude_pos.contains(p.as_str()) { + continue; + } + } + segmented.push(surface); + } + + segmented + } } diff --git a/src-react/src/components/LoadingDialog.tsx b/src-react/src/components/LoadingDialog.tsx index 6559ae6..bb78513 100644 --- a/src-react/src/components/LoadingDialog.tsx +++ b/src-react/src/components/LoadingDialog.tsx @@ -7,6 +7,13 @@ import { listen } from "@tauri-apps/api/event"; import { supabase } from "../lib/supabaseClient"; interface YomitanDictEntry { + title: string; + bundle_name: string; + revision: string; + lang: string; +} + +interface YomitanDownloadDictEntry { url: string; filepath: string; } @@ -66,13 +73,17 @@ function LoadingDialog() { } }); async function downloadAssets(lang = "ja") { - const dicts: YomitanDictEntry[] = []; + const existing_dicts = + await invoke("list_yomitan_dict"); + + const dicts: YomitanDownloadDictEntry[] = []; if (lang === "ja") { - { - const filename = "Pixiv.zip"; - const filepath = `yomitan/${lang}/${filename}`; - + const filename = "Pixiv.zip"; + const filepath = `yomitan/${lang}/${filename}`; + const existing = existing_dicts.find((d) => d.bundle_name === filename); + if (!existing) { + // Maybe check revision update dicts.push({ filepath, url: @@ -84,6 +95,11 @@ function LoadingDialog() { return ""; })) || "", }); + } else { + dicts.push({ + filepath, + url: "", + }); } { diff --git a/src-react/src/pages/SearchPage.tsx b/src-react/src/pages/SearchPage.tsx index 4c40e5a..c401043 100644 --- a/src-react/src/pages/SearchPage.tsx +++ b/src-react/src/pages/SearchPage.tsx @@ -115,6 +115,9 @@ function SearchPage() { if (isAutoKana) { norm_q = norm_q.replace(/n$/i, "ん").replace(/[a-z]$/i, ""); } + + norm_q = norm_q.replace(/＊/g, "*").replace(/？/g, "?").replace(/^〜/, "~"); + if (!norm_q) return; const ender = @@ -391,6 +394,9 @@ function SearchPage() { {"）"} ) : null} + 【{dict_title}】 {JSONdumpClean(it)} diff --git a/src-tauri/src/command.rs b/src-tauri/src/command.rs index 4498964..18bf7d7 100644 --- a/src-tauri/src/command.rs +++ b/src-tauri/src/command.rs @@ -5,7 +5,8 @@ use std::{ }; use cjdic2_core::{ - AppService, CJDicError, SqlParam, TokenizeSegment, YomitanRow, YomitanZipImportResult, + AppService, CJDicError, SqlParam, TokenizeSegment, YomitanDictEntry, YomitanRow, + YomitanZipImportResult, }; use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Emitter, Manager}; @@ -13,8 +14,15 @@ use tauri_plugin_http::reqwest::Client; use tokio::io::AsyncWriteExt; use zip::ZipArchive; +#[tauri::command] +pub async fn list_yomitan_dict( + state: tauri::State<'_, AppService>, +) -> Result, CJDicError> { + state.list_yomitan_dict() +} + #[derive(Deserialize)] -pub struct YomitanDictEntry { +pub struct YomitanDownloadDictEntry { url: String, filepath: String, } @@ -22,7 +30,7 @@ pub struct YomitanDictEntry { #[tauri::command] pub async fn init_yomitan( app: AppHandle, - dicts: Vec, + dicts: Vec, lang: String, state: tauri::State<'_, AppService>, ) -> Result<(), CJDicError> { diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index dd84e5c..f0ebb7d 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -47,6 +47,7 @@ pub fn run() { Ok(()) }) .invoke_handler(tauri::generate_handler![ + list_yomitan_dict, init_yomitan, import_yomitan_dict, remove_yomitan_dict,