normalize reading

src-core/src/db/search.rs

@@ -1,4 +1,10 @@
-#[allow(unused)]
+use rusqlite::Connection;
+
+use crate::{
+    CJDicError,
+    db::{DBSCHEMA, DbChild},
+};
+
 pub(super) fn normalize_reading(s: &str) -> String {
     s.chars()
         .map(|c| {
@@ -15,3 +21,95 @@ pub(super) fn normalize_reading(s: &str) -> String {
         })
         .collect()
 }
+
+pub struct SearchDatabase;
+//  {
+//     db: Database,
+//}
+
+// pub struct SearchEntry {
+//     id: i64,
+//     term: String,
+//     reading: String,
+// }
+
+impl SearchDatabase {
+    // pub(crate) fn new(db: Database) -> Self {
+    //     Self { db }
+    // }
+
+    pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> {
+        conn.execute_batch(
+            &format!("
+            CREATE TABLE IF NOT EXISTS {0}.terms (
+                id          INTEGER PRIMARY KEY,    -- yomitan.terms.id OR yomitan dups OR -user.terms.id
+                term        TEXT NOT NULL,          -- normalized
+                reading     TEXT NOT NULL           -- normalized
+            );
+
+            CREATE INDEX IF NOT EXISTS {0}.terms_term       ON terms (term);
+            CREATE INDEX IF NOT EXISTS {0}.terms_reading    ON terms (reading);
+
+            CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 (
+                term,
+                content='',
+                tokenize=unicode61
+            );
+        ", DBSCHEMA[DbChild::Search]),
+        )?;
+
+        Ok(())
+    }
+
+    // pub fn insert_many(
+    //     &self,
+    //     entries: Vec<SearchEntry>,
+    //     tokenizer: Tokenizer,
+    // ) -> Result<(), CJDicError> {
+    //     let mut conn = self.db.conn.lock()?;
+    //     let tx = conn.transaction()?;
+    //     {
+    //         let mut stmt =
+    //             tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?;
+    //         for it in entries.iter() {
+    //             stmt.execute(params![
+    //                 it.id,
+    //                 normalize_term(&it.term),
+    //                 normalize_reading(&it.reading)
+    //             ])?;
+    //         }
+
+    //         let exclude_pos =
+    //             HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
+
+    //         let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?;
+    //         for it in entries {
+    //             let mut segmented: Vec<String> = vec![];
+    //             for t in tokenizer.tokenize(it.term) {
+    //                 let mut base = t.surface;
+    //                 let mut pos = None;
+
+    //                 for (i, dt) in t.details.iter().enumerate() {
+    //                     match i {
+    //                         0 => base = dt.to_string(),
+    //                         6 => pos = Some(dt),
+    //                         _ => {}
+    //                     }
+    //                 }
+
+    //                 if let Some(p) = pos {
+    //                     if exclude_pos.contains(p.as_str()) {
+    //                         continue;
+    //                     }
+    //                 }
+    //                 segmented.push(base);
+    //             }
+
+    //             stmt.execute(params![it.id, segmented.join(" ")])?;
+    //         }
+    //     }
+    //     tx.commit()?;
+
+    //     Ok(())
+    // }
+}

src-core/src/db/yomitan.rs

@@ -111,10 +111,11 @@ impl YomitanDatabase {
                 rules_id,
                 term_tags_id,
                 sequence
-            FROM yomitan.terms                t
+            FROM search.terms                 s
+            JOIN yomitan.terms                t  ON t.id = s.id
             JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
             JOIN yomitan.dictionaries         d  ON d.id  = t.dict_id
-            WHERE t.term_norm {eq1} ?1 {and_or} t.reading {eq2} ?2
+            WHERE s.term {eq1} ?1 {and_or} s.reading {eq2} ?2
             ORDER BY tr.max_score DESC, d.sort_order DESC
             LIMIT ?3 OFFSET ?4
         "

src-core/src/db/yomitan_writer.rs

@@ -13,7 +13,10 @@ use zip::ZipArchive;
 
 use crate::{
     CJDicError, Timer,
-    db::{DBFILE, DBSCHEMA, DbChild},
+    db::{
+        DBFILE, DBSCHEMA, DbChild,
+        search::{SearchDatabase, normalize_reading},
+    },
 };
 
 fn blake3_hex(s: &str) -> String {
@@ -177,7 +180,7 @@ impl YomitanWriter {
         self.attach_db(DbChild::YomitanGlossary)?;
         self.attach_db(DbChild::Search)?;
 
-        let schema_version = "2026-03-07";
+        let schema_version = "2026-03-08";
 
         let current_schema_version = {
             let mut stmt = self
@@ -188,7 +191,8 @@ impl YomitanWriter {
         };
 
         if let Some(v) = current_schema_version {
-            if v < "2".to_string() {
+            let v = v.as_str();
+            if v < "2" {
                 let total = self
                     .conn
                     .query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))?
@@ -266,7 +270,60 @@ impl YomitanWriter {
                 )?;
             }
 
-            if v < schema_version.to_string() {
+            if v < "2026-03-08" {
+                let total = self
+                    .conn
+                    .query_row("SELECT MAX(id) FROM terms", [], |r| r.get::<_, i64>(0))?
+                    as usize;
+
+                let message = &format!("Generating {}", DBFILE[DbChild::Search]);
+                progress_callback(YomitanProgress {
+                    message: message.to_string(),
+                    current: 0,
+                    total,
+                    steps: 0,
+                });
+
+                let _timer = Timer::new(message.to_string());
+
+                let tx = self.conn.transaction()?;
+                {
+                    let mut stmt = tx.prepare("SELECT id, term, reading FROM terms")?;
+
+                    let rows = stmt.query_map([], |r| {
+                        Ok((
+                            r.get::<_, i64>(0)?,
+                            r.get::<_, String>(1)?,
+                            r.get::<_, String>(2)?,
+                        ))
+                    })?;
+
+                    let mut stmt_write = tx.prepare(
+                        "INSERT INTO search.terms (id, term, reading) VALUES (?1, ?2, ?3)",
+                    )?;
+
+                    for (i, row) in rows.enumerate() {
+                        let (id, term, reading) = row?;
+                        stmt_write.execute(params![
+                            id,
+                            normalize_term(&term),
+                            normalize_reading(&reading)
+                        ])?;
+
+                        if i % 10_000 == 0 {
+                            progress_callback(YomitanProgress {
+                                message: message.to_string(),
+                                current: i,
+                                total,
+                                steps: i,
+                            });
+                        }
+                    }
+                }
+                tx.commit()?;
+            }
+
+            if v < schema_version {
                 self.conn
                     .prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")?
                     .execute(["schema_version", schema_version])?;
@@ -395,6 +452,7 @@ impl YomitanWriter {
 
         match db_child {
             DbChild::YomitanGlossary => self.create_schema_glossary()?,
+            DbChild::Search => SearchDatabase::create_schema(&self.conn)?,
             _ => (),
         }