diff --git a/src-core/examples/ja_tokenize.rs b/src-core/examples/ja_tokenize.rs index 301112f..2716631 100644 --- a/src-core/examples/ja_tokenize.rs +++ b/src-core/examples/ja_tokenize.rs @@ -15,7 +15,7 @@ fn main() -> Result<(), anyhow::Error> { println!( "{:#?}", - service.ja_tokenize(arg_it.next().unwrap_or( + service.tokenize(arg_it.next().unwrap_or( "武器･防具≪エルガー武器商会≫の裏の鍛冶場を調べ「錆びた鍵」を入手します ".to_string() ))? ); diff --git a/src-core/src/db.rs b/src-core/src/db.rs index 2122ced..31385c1 100644 --- a/src-core/src/db.rs +++ b/src-core/src/db.rs @@ -40,14 +40,14 @@ pub(crate) static DBFILE: LazyLock> = LazyLock::new(|| { enum_map! { DbChild::Yomitan => "yomitan.db", DbChild::YomitanGlossary => "yomitan-glossary.db", - DbChild::Search => "reading.db" + DbChild::Search => "search.db" } }); pub(crate) static DBSCHEMA: LazyLock> = LazyLock::new(|| { enum_map! { DbChild::Yomitan => "yomitan", DbChild::YomitanGlossary => "glossary", - DbChild::Search => "reading" + DbChild::Search => "search" } }); diff --git a/src-core/src/db/search.rs b/src-core/src/db/search.rs index dddc3fb..2b2fefd 100644 --- a/src-core/src/db/search.rs +++ b/src-core/src/db/search.rs @@ -1,4 +1,10 @@ -#[allow(unused)] +use rusqlite::Connection; + +use crate::{ + CJDicError, + db::{DBSCHEMA, DbChild}, +}; + pub(super) fn normalize_reading(s: &str) -> String { s.chars() .map(|c| { @@ -15,3 +21,95 @@ pub(super) fn normalize_reading(s: &str) -> String { }) .collect() } + +pub struct SearchDatabase; +// { +// db: Database, +//} + +// pub struct SearchEntry { +// id: i64, +// term: String, +// reading: String, +// } + +impl SearchDatabase { + // pub(crate) fn new(db: Database) -> Self { + // Self { db } + // } + + pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> { + conn.execute_batch( + &format!(" + CREATE TABLE IF NOT EXISTS {0}.terms ( + id INTEGER PRIMARY KEY, -- yomitan.terms.id OR yomitan dups OR -user.terms.id + term TEXT NOT NULL, -- normalized + reading TEXT NOT NULL -- normalized + ); + + CREATE INDEX IF NOT EXISTS {0}.terms_term ON terms (term); + CREATE INDEX IF NOT EXISTS {0}.terms_reading ON terms (reading); + + CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 ( + term, + content='', + tokenize=unicode61 + ); + ", DBSCHEMA[DbChild::Search]), + )?; + + Ok(()) + } + + // pub fn insert_many( + // &self, + // entries: Vec, + // tokenizer: Tokenizer, + // ) -> Result<(), CJDicError> { + // let mut conn = self.db.conn.lock()?; + // let tx = conn.transaction()?; + // { + // let mut stmt = + // tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?; + // for it in entries.iter() { + // stmt.execute(params![ + // it.id, + // normalize_term(&it.term), + // normalize_reading(&it.reading) + // ])?; + // } + + // let exclude_pos = + // HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string())); + + // let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?; + // for it in entries { + // let mut segmented: Vec = vec![]; + // for t in tokenizer.tokenize(it.term) { + // let mut base = t.surface; + // let mut pos = None; + + // for (i, dt) in t.details.iter().enumerate() { + // match i { + // 0 => base = dt.to_string(), + // 6 => pos = Some(dt), + // _ => {} + // } + // } + + // if let Some(p) = pos { + // if exclude_pos.contains(p.as_str()) { + // continue; + // } + // } + // segmented.push(base); + // } + + // stmt.execute(params![it.id, segmented.join(" ")])?; + // } + // } + // tx.commit()?; + + // Ok(()) + // } +} diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs index 049d643..dc9fd43 100644 --- a/src-core/src/db/yomitan.rs +++ b/src-core/src/db/yomitan.rs @@ -111,10 +111,11 @@ impl YomitanDatabase { rules_id, term_tags_id, sequence - FROM yomitan.terms t + FROM search.terms s + JOIN yomitan.terms t ON t.id = s.id JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading JOIN yomitan.dictionaries d ON d.id = t.dict_id - WHERE t.term_norm {eq1} ?1 {and_or} t.reading {eq2} ?2 + WHERE s.term {eq1} ?1 {and_or} s.reading {eq2} ?2 ORDER BY tr.max_score DESC, d.sort_order DESC LIMIT ?3 OFFSET ?4 " diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs index eacfa88..94e7d86 100644 --- a/src-core/src/db/yomitan_writer.rs +++ b/src-core/src/db/yomitan_writer.rs @@ -13,7 +13,10 @@ use zip::ZipArchive; use crate::{ CJDicError, Timer, - db::{DBFILE, DBSCHEMA, DbChild}, + db::{ + DBFILE, DBSCHEMA, DbChild, + search::{SearchDatabase, normalize_reading}, + }, }; fn blake3_hex(s: &str) -> String { @@ -177,7 +180,7 @@ impl YomitanWriter { self.attach_db(DbChild::YomitanGlossary)?; self.attach_db(DbChild::Search)?; - let schema_version = "2026-03-07"; + let schema_version = "2026-03-08"; let current_schema_version = { let mut stmt = self @@ -188,7 +191,8 @@ impl YomitanWriter { }; if let Some(v) = current_schema_version { - if v < "2".to_string() { + let v = v.as_str(); + if v < "2" { let total = self .conn .query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))? @@ -266,7 +270,60 @@ impl YomitanWriter { )?; } - if v < schema_version.to_string() { + if v < "2026-03-08" { + let total = self + .conn + .query_row("SELECT MAX(id) FROM terms", [], |r| r.get::<_, i64>(0))? + as usize; + + let message = &format!("Generating {}", DBFILE[DbChild::Search]); + progress_callback(YomitanProgress { + message: message.to_string(), + current: 0, + total, + steps: 0, + }); + + let _timer = Timer::new(message.to_string()); + + let tx = self.conn.transaction()?; + { + let mut stmt = tx.prepare("SELECT id, term, reading FROM terms")?; + + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })?; + + let mut stmt_write = tx.prepare( + "INSERT INTO search.terms (id, term, reading) VALUES (?1, ?2, ?3)", + )?; + + for (i, row) in rows.enumerate() { + let (id, term, reading) = row?; + stmt_write.execute(params![ + id, + normalize_term(&term), + normalize_reading(&reading) + ])?; + + if i % 10_000 == 0 { + progress_callback(YomitanProgress { + message: message.to_string(), + current: i, + total, + steps: i, + }); + } + } + } + tx.commit()?; + } + + if v < schema_version { self.conn .prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")? .execute(["schema_version", schema_version])?; @@ -395,6 +452,7 @@ impl YomitanWriter { match db_child { DbChild::YomitanGlossary => self.create_schema_glossary()?, + DbChild::Search => SearchDatabase::create_schema(&self.conn)?, _ => (), } diff --git a/src-core/src/error.rs b/src-core/src/error.rs index 3f4926f..b429f9a 100644 --- a/src-core/src/error.rs +++ b/src-core/src/error.rs @@ -6,7 +6,7 @@ use std::{ use rusqlite::Connection; use serde::Serialize; use thiserror::Error; -use vibrato::errors::VibratoError; +use vibrato::{Tokenizer, errors::VibratoError}; use zip::result::ZipError; #[derive(Debug, Error, Serialize)] @@ -39,6 +39,9 @@ pub enum CJDicError { #[error("ConnectionMutexGuardError: {0}")] ConnectionMutexGuardError(String), + #[error("TokenizerMutexGuardError: {0}")] + TokenizerMutexGuardError(String), + #[error("Not found")] NotFound, @@ -101,3 +104,10 @@ impl From>> for CJDicError { CJDicError::ConnectionMutexGuardError(e.to_string()) } } + +impl From>> for CJDicError { + fn from(e: PoisonError>) -> Self { + eprintln!("{e:#}"); + CJDicError::TokenizerMutexGuardError(e.to_string()) + } +} diff --git a/src-core/src/lib.rs b/src-core/src/lib.rs index fcd366d..ff200c6 100644 --- a/src-core/src/lib.rs +++ b/src-core/src/lib.rs @@ -4,11 +4,13 @@ extern crate enum_map; mod db; mod error; mod service; +mod tokenizer; mod util; pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource}; pub use error::CJDicError; -pub use service::{AppService, SqlParam, TokenizeSegment}; +pub use service::{AppService, SqlParam}; +pub use tokenizer::TokenizeSegment; pub use util::Timer; pub fn add(left: u64, right: u64) -> u64 { diff --git a/src-core/src/service.rs b/src-core/src/service.rs index 754fe80..f5adc1c 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -1,12 +1,10 @@ use std::{ collections::{HashMap, HashSet}, - fs::File, path::Path, }; use rusqlite::{Connection, params_from_iter}; use serde::{Deserialize, Serialize}; -use vibrato::{Dictionary, Tokenizer}; use crate::{ Timer, ZipSource, @@ -15,6 +13,7 @@ use crate::{ YomitanZipImportResult, }, error::CJDicError, + tokenizer::{TokenizeSegment, Tokenizer}, }; #[derive(Serialize, Debug, Clone)] @@ -33,55 +32,23 @@ pub enum SqlParam { Bool(bool), } -#[derive(Debug, Serialize)] -pub struct TokenizeSegment { - surface: String, - /// 0 品詞 Part-of-speech - /// 1 品詞細分類1 Part-of-speech subcategory 1 - /// 2 品詞細分類2 Part-of-speech subcategory 2 - /// 3 品詞細分類3 Part-of-speech subcategory 3 - /// 4 活用形 Conjugation form - /// 5 活用型 Conjugation type - /// 6 原形 Base form - /// 7 読み Reading - /// 8 発音 Pronunciation - /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html - details: Vec, -} - pub struct AppService { db: Database, - ja_tokenizer: Tokenizer, + tokenizer: Tokenizer, } impl AppService { - pub fn new>(db_dir: P, vibrato_dict: P) -> Result { + pub fn new( + db_dir: impl AsRef, + vibrato_dict: impl AsRef, + ) -> Result { let db = Database::new(db_dir)?; - - let ja_tokenizer = { - let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?; - let dictionary = Dictionary::read(reader)?; - Tokenizer::new(dictionary) - }; - - Ok(Self { db, ja_tokenizer }) + let tokenizer = Tokenizer::new(vibrato_dict)?; + Ok(Self { db, tokenizer }) } - pub fn ja_tokenize(&self, text: String) -> Result, CJDicError> { - let mut worker = self.ja_tokenizer.new_worker(); - worker.reset_sentence(text); - worker.tokenize(); - - let mut output: Vec = vec![]; - - for token in worker.token_iter() { - let surface = token.surface().to_string(); - let details = token.feature().split(",").map(|s| s.to_string()).collect(); - - output.push(TokenizeSegment { surface, details }); - } - - Ok(output) + pub fn tokenize(&self, text: String) -> Result, CJDicError> { + Ok(self.tokenizer.tokenize(text)) } pub fn execute_sql( diff --git a/src-core/src/tokenizer.rs b/src-core/src/tokenizer.rs new file mode 100644 index 0000000..59fdc2e --- /dev/null +++ b/src-core/src/tokenizer.rs @@ -0,0 +1,56 @@ +use std::{fs::File, path::Path}; + +use serde::Serialize; + +use crate::CJDicError; + +#[derive(Debug, Serialize)] +pub struct TokenizeSegment { + pub surface: String, + /// 0 品詞 Part-of-speech + /// 1 品詞細分類1 Part-of-speech subcategory 1 + /// 2 品詞細分類2 Part-of-speech subcategory 2 + /// 3 品詞細分類3 Part-of-speech subcategory 3 + /// 4 活用形 Conjugation form + /// 5 活用型 Conjugation type + /// 6 原形 Base form + /// 7 読み Reading + /// 8 発音 Pronunciation + /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html + pub details: Vec, +} + +pub struct Tokenizer { + ja_tokenizer: vibrato::Tokenizer, +} + +impl Tokenizer { + pub fn new(vibrato_dict: impl AsRef) -> Result { + let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?; + let dictionary = vibrato::Dictionary::read(reader)?; + Ok(Self { + ja_tokenizer: vibrato::Tokenizer::new(dictionary), + }) + } + + pub fn tokenize(&self, text: String) -> Vec { + self.ja_tokenize(text) + } + + fn ja_tokenize(&self, text: String) -> Vec { + let mut worker = self.ja_tokenizer.new_worker(); + worker.reset_sentence(text); + worker.tokenize(); + + let mut output: Vec = vec![]; + + for token in worker.token_iter() { + let surface = token.surface().to_string(); + let details = token.feature().split(",").map(|s| s.to_string()).collect(); + + output.push(TokenizeSegment { surface, details }); + } + + output + } +} diff --git a/src-tauri/src/command.rs b/src-tauri/src/command.rs index 9e1cd19..4498964 100644 --- a/src-tauri/src/command.rs +++ b/src-tauri/src/command.rs @@ -251,5 +251,5 @@ pub async fn tokenize( text: String, state: tauri::State<'_, AppService>, ) -> Result, CJDicError> { - state.ja_tokenize(text) + state.tokenize(text) }