normalize reading

src-core/examples/ja_tokenize.rs

@@ -15,7 +15,7 @@ fn main() -> Result<(), anyhow::Error> {
 
     println!(
         "{:#?}",
-        service.ja_tokenize(arg_it.next().unwrap_or(
+        service.tokenize(arg_it.next().unwrap_or(
             "武器･防具≪エルガー武器商会≫の裏の鍛冶場を調べ「錆びた鍵」を入手します ".to_string()
         ))?
     );

src-core/src/db.rs

@@ -40,14 +40,14 @@ pub(crate) static DBFILE: LazyLock<EnumMap<DbChild, &str>> = LazyLock::new(|| {
     enum_map! {
         DbChild::Yomitan => "yomitan.db",
         DbChild::YomitanGlossary => "yomitan-glossary.db",
-        DbChild::Search => "reading.db"
+        DbChild::Search => "search.db"
     }
 });
 pub(crate) static DBSCHEMA: LazyLock<EnumMap<DbChild, &str>> = LazyLock::new(|| {
     enum_map! {
         DbChild::Yomitan => "yomitan",
         DbChild::YomitanGlossary => "glossary",
-        DbChild::Search => "reading"
+        DbChild::Search => "search"
     }
 });
 

src-core/src/db/search.rs

@@ -1,4 +1,10 @@
-#[allow(unused)]
+use rusqlite::Connection;
+
+use crate::{
+    CJDicError,
+    db::{DBSCHEMA, DbChild},
+};
+
 pub(super) fn normalize_reading(s: &str) -> String {
     s.chars()
         .map(|c| {
@@ -15,3 +21,95 @@ pub(super) fn normalize_reading(s: &str) -> String {
         })
         .collect()
 }
+
+pub struct SearchDatabase;
+//  {
+//     db: Database,
+//}
+
+// pub struct SearchEntry {
+//     id: i64,
+//     term: String,
+//     reading: String,
+// }
+
+impl SearchDatabase {
+    // pub(crate) fn new(db: Database) -> Self {
+    //     Self { db }
+    // }
+
+    pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> {
+        conn.execute_batch(
+            &format!("
+            CREATE TABLE IF NOT EXISTS {0}.terms (
+                id          INTEGER PRIMARY KEY,    -- yomitan.terms.id OR yomitan dups OR -user.terms.id
+                term        TEXT NOT NULL,          -- normalized
+                reading     TEXT NOT NULL           -- normalized
+            );
+
+            CREATE INDEX IF NOT EXISTS {0}.terms_term       ON terms (term);
+            CREATE INDEX IF NOT EXISTS {0}.terms_reading    ON terms (reading);
+
+            CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 (
+                term,
+                content='',
+                tokenize=unicode61
+            );
+        ", DBSCHEMA[DbChild::Search]),
+        )?;
+
+        Ok(())
+    }
+
+    // pub fn insert_many(
+    //     &self,
+    //     entries: Vec<SearchEntry>,
+    //     tokenizer: Tokenizer,
+    // ) -> Result<(), CJDicError> {
+    //     let mut conn = self.db.conn.lock()?;
+    //     let tx = conn.transaction()?;
+    //     {
+    //         let mut stmt =
+    //             tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?;
+    //         for it in entries.iter() {
+    //             stmt.execute(params![
+    //                 it.id,
+    //                 normalize_term(&it.term),
+    //                 normalize_reading(&it.reading)
+    //             ])?;
+    //         }
+
+    //         let exclude_pos =
+    //             HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
+
+    //         let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?;
+    //         for it in entries {
+    //             let mut segmented: Vec<String> = vec![];
+    //             for t in tokenizer.tokenize(it.term) {
+    //                 let mut base = t.surface;
+    //                 let mut pos = None;
+
+    //                 for (i, dt) in t.details.iter().enumerate() {
+    //                     match i {
+    //                         0 => base = dt.to_string(),
+    //                         6 => pos = Some(dt),
+    //                         _ => {}
+    //                     }
+    //                 }
+
+    //                 if let Some(p) = pos {
+    //                     if exclude_pos.contains(p.as_str()) {
+    //                         continue;
+    //                     }
+    //                 }
+    //                 segmented.push(base);
+    //             }
+
+    //             stmt.execute(params![it.id, segmented.join(" ")])?;
+    //         }
+    //     }
+    //     tx.commit()?;
+
+    //     Ok(())
+    // }
+}

src-core/src/db/yomitan.rs

@@ -111,10 +111,11 @@ impl YomitanDatabase {
                 rules_id,
                 term_tags_id,
                 sequence
-            FROM yomitan.terms                t
+            FROM search.terms                 s
+            JOIN yomitan.terms                t  ON t.id = s.id
             JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
             JOIN yomitan.dictionaries         d  ON d.id  = t.dict_id
-            WHERE t.term_norm {eq1} ?1 {and_or} t.reading {eq2} ?2
+            WHERE s.term {eq1} ?1 {and_or} s.reading {eq2} ?2
             ORDER BY tr.max_score DESC, d.sort_order DESC
             LIMIT ?3 OFFSET ?4
         "

src-core/src/db/yomitan_writer.rs

@@ -13,7 +13,10 @@ use zip::ZipArchive;
 
 use crate::{
     CJDicError, Timer,
-    db::{DBFILE, DBSCHEMA, DbChild},
+    db::{
+        DBFILE, DBSCHEMA, DbChild,
+        search::{SearchDatabase, normalize_reading},
+    },
 };
 
 fn blake3_hex(s: &str) -> String {
@@ -177,7 +180,7 @@ impl YomitanWriter {
         self.attach_db(DbChild::YomitanGlossary)?;
         self.attach_db(DbChild::Search)?;
 
-        let schema_version = "2026-03-07";
+        let schema_version = "2026-03-08";
 
         let current_schema_version = {
             let mut stmt = self
@@ -188,7 +191,8 @@ impl YomitanWriter {
         };
 
         if let Some(v) = current_schema_version {
-            if v < "2".to_string() {
+            let v = v.as_str();
+            if v < "2" {
                 let total = self
                     .conn
                     .query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))?
@@ -266,7 +270,60 @@ impl YomitanWriter {
                 )?;
             }
 
-            if v < schema_version.to_string() {
+            if v < "2026-03-08" {
+                let total = self
+                    .conn
+                    .query_row("SELECT MAX(id) FROM terms", [], |r| r.get::<_, i64>(0))?
+                    as usize;
+
+                let message = &format!("Generating {}", DBFILE[DbChild::Search]);
+                progress_callback(YomitanProgress {
+                    message: message.to_string(),
+                    current: 0,
+                    total,
+                    steps: 0,
+                });
+
+                let _timer = Timer::new(message.to_string());
+
+                let tx = self.conn.transaction()?;
+                {
+                    let mut stmt = tx.prepare("SELECT id, term, reading FROM terms")?;
+
+                    let rows = stmt.query_map([], |r| {
+                        Ok((
+                            r.get::<_, i64>(0)?,
+                            r.get::<_, String>(1)?,
+                            r.get::<_, String>(2)?,
+                        ))
+                    })?;
+
+                    let mut stmt_write = tx.prepare(
+                        "INSERT INTO search.terms (id, term, reading) VALUES (?1, ?2, ?3)",
+                    )?;
+
+                    for (i, row) in rows.enumerate() {
+                        let (id, term, reading) = row?;
+                        stmt_write.execute(params![
+                            id,
+                            normalize_term(&term),
+                            normalize_reading(&reading)
+                        ])?;
+
+                        if i % 10_000 == 0 {
+                            progress_callback(YomitanProgress {
+                                message: message.to_string(),
+                                current: i,
+                                total,
+                                steps: i,
+                            });
+                        }
+                    }
+                }
+                tx.commit()?;
+            }
+
+            if v < schema_version {
                 self.conn
                     .prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")?
                     .execute(["schema_version", schema_version])?;
@@ -395,6 +452,7 @@ impl YomitanWriter {
 
         match db_child {
             DbChild::YomitanGlossary => self.create_schema_glossary()?,
+            DbChild::Search => SearchDatabase::create_schema(&self.conn)?,
             _ => (),
         }
 

src-core/src/error.rs

@@ -6,7 +6,7 @@ use std::{
 use rusqlite::Connection;
 use serde::Serialize;
 use thiserror::Error;
-use vibrato::errors::VibratoError;
+use vibrato::{Tokenizer, errors::VibratoError};
 use zip::result::ZipError;
 
 #[derive(Debug, Error, Serialize)]
@@ -39,6 +39,9 @@ pub enum CJDicError {
     #[error("ConnectionMutexGuardError: {0}")]
     ConnectionMutexGuardError(String),
 
+    #[error("TokenizerMutexGuardError: {0}")]
+    TokenizerMutexGuardError(String),
+
     #[error("Not found")]
     NotFound,
 
@@ -101,3 +104,10 @@ impl From<PoisonError<MutexGuard<'_, Connection>>> for CJDicError {
         CJDicError::ConnectionMutexGuardError(e.to_string())
     }
 }
+
+impl From<PoisonError<MutexGuard<'_, Tokenizer>>> for CJDicError {
+    fn from(e: PoisonError<MutexGuard<'_, Tokenizer>>) -> Self {
+        eprintln!("{e:#}");
+        CJDicError::TokenizerMutexGuardError(e.to_string())
+    }
+}

src-core/src/lib.rs

@@ -4,11 +4,13 @@ extern crate enum_map;
 mod db;
 mod error;
 mod service;
+mod tokenizer;
 mod util;
 
 pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource};
 pub use error::CJDicError;
-pub use service::{AppService, SqlParam, TokenizeSegment};
+pub use service::{AppService, SqlParam};
+pub use tokenizer::TokenizeSegment;
 pub use util::Timer;
 
 pub fn add(left: u64, right: u64) -> u64 {

src-core/src/service.rs

@@ -1,12 +1,10 @@
 use std::{
     collections::{HashMap, HashSet},
-    fs::File,
     path::Path,
 };
 
 use rusqlite::{Connection, params_from_iter};
 use serde::{Deserialize, Serialize};
-use vibrato::{Dictionary, Tokenizer};
 
 use crate::{
     Timer, ZipSource,
@@ -15,6 +13,7 @@ use crate::{
         YomitanZipImportResult,
     },
     error::CJDicError,
+    tokenizer::{TokenizeSegment, Tokenizer},
 };
 
 #[derive(Serialize, Debug, Clone)]
@@ -33,55 +32,23 @@ pub enum SqlParam {
     Bool(bool),
 }
 
-#[derive(Debug, Serialize)]
-pub struct TokenizeSegment {
-    surface: String,
-    /// 0	品詞	Part-of-speech
-    /// 1	品詞細分類1	Part-of-speech subcategory 1
-    /// 2	品詞細分類2	Part-of-speech subcategory 2
-    /// 3	品詞細分類3	Part-of-speech subcategory 3
-    /// 4	活用形	Conjugation form
-    /// 5	活用型	Conjugation type
-    /// 6	原形	Base form
-    /// 7	読み	Reading
-    /// 8	発音	Pronunciation
-    /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
-    details: Vec<String>,
-}
-
 pub struct AppService {
     db: Database,
-    ja_tokenizer: Tokenizer,
+    tokenizer: Tokenizer,
 }
 
 impl AppService {
-    pub fn new<P: AsRef<Path>>(db_dir: P, vibrato_dict: P) -> Result<Self, CJDicError> {
+    pub fn new(
+        db_dir: impl AsRef<Path>,
+        vibrato_dict: impl AsRef<Path>,
+    ) -> Result<Self, CJDicError> {
         let db = Database::new(db_dir)?;
-
+        let tokenizer = Tokenizer::new(vibrato_dict)?;
-        let ja_tokenizer = {
+        Ok(Self { db, tokenizer })
-            let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
-            let dictionary = Dictionary::read(reader)?;
-            Tokenizer::new(dictionary)
-        };
-
-        Ok(Self { db, ja_tokenizer })
     }
 
-    pub fn ja_tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
+    pub fn tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
-        let mut worker = self.ja_tokenizer.new_worker();
+        Ok(self.tokenizer.tokenize(text))
-        worker.reset_sentence(text);
-        worker.tokenize();
-
-        let mut output: Vec<TokenizeSegment> = vec![];
-
-        for token in worker.token_iter() {
-            let surface = token.surface().to_string();
-            let details = token.feature().split(",").map(|s| s.to_string()).collect();
-
-            output.push(TokenizeSegment { surface, details });
-        }
-
-        Ok(output)
     }
 
     pub fn execute_sql(

src-core/src/tokenizer.rs

@@ -0,0 +1,56 @@
+use std::{fs::File, path::Path};
+
+use serde::Serialize;
+
+use crate::CJDicError;
+
+#[derive(Debug, Serialize)]
+pub struct TokenizeSegment {
+    pub surface: String,
+    /// 0	品詞	Part-of-speech
+    /// 1	品詞細分類1	Part-of-speech subcategory 1
+    /// 2	品詞細分類2	Part-of-speech subcategory 2
+    /// 3	品詞細分類3	Part-of-speech subcategory 3
+    /// 4	活用形	Conjugation form
+    /// 5	活用型	Conjugation type
+    /// 6	原形	Base form
+    /// 7	読み	Reading
+    /// 8	発音	Pronunciation
+    /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
+    pub details: Vec<String>,
+}
+
+pub struct Tokenizer {
+    ja_tokenizer: vibrato::Tokenizer,
+}
+
+impl Tokenizer {
+    pub fn new(vibrato_dict: impl AsRef<Path>) -> Result<Self, CJDicError> {
+        let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
+        let dictionary = vibrato::Dictionary::read(reader)?;
+        Ok(Self {
+            ja_tokenizer: vibrato::Tokenizer::new(dictionary),
+        })
+    }
+
+    pub fn tokenize(&self, text: String) -> Vec<TokenizeSegment> {
+        self.ja_tokenize(text)
+    }
+
+    fn ja_tokenize(&self, text: String) -> Vec<TokenizeSegment> {
+        let mut worker = self.ja_tokenizer.new_worker();
+        worker.reset_sentence(text);
+        worker.tokenize();
+
+        let mut output: Vec<TokenizeSegment> = vec![];
+
+        for token in worker.token_iter() {
+            let surface = token.surface().to_string();
+            let details = token.feature().split(",").map(|s| s.to_string()).collect();
+
+            output.push(TokenizeSegment { surface, details });
+        }
+
+        output
+    }
+}

src-tauri/src/command.rs

@@ -251,5 +251,5 @@ pub async fn tokenize(
     text: String,
     state: tauri::State<'_, AppService>,
 ) -> Result<Vec<TokenizeSegment>, CJDicError> {
-    state.ja_tokenize(text)
+    state.tokenize(text)
 }