move glossary (def text) to a separate db file

Cargo.lock

@@ -639,7 +639,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "blake3",
- "log",
  "rusqlite",
  "serde",
  "serde_json",

src-core/Cargo.toml

@@ -14,6 +14,5 @@ thiserror = "2.0.18"
 zip = "8.1.0"
 blake3 = "1.8.3"
 anyhow = "1.0.102"
-log = "0.4.29"
 vibrato = "0.5.2"
 zstd = "0.13.3"

src-core/examples/add_yomitan_dict_folder.rs

@@ -13,7 +13,9 @@ use crate::common::get_vibrato_dict_dir;
 
 fn main() -> Result<(), anyhow::Error> {
     let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
-    let mut writer = service.get_yomitan_writer()?;
+    let mut writer = service.get_yomitan_writer(|p| {
+        println!("{:?}", p);
+    })?;
 
     let zip_dir = Path::new("src-tauri/resources/yomitan");
     println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root

src-core/examples/benchmark_json_zstd.rs

@@ -0,0 +1,57 @@
+use anyhow::Ok;
+
+mod common;
+use common::get_db_dir;
+use rusqlite::{Connection, params};
+
+fn benchmark_dict_sizes(samples: &[String]) {
+    // 112640; // 110KB default gives the best 5.8x compression
+    for size in [16384, 65536, 112640, 524288] {
+        let total: usize = samples.iter().map(|s| s.len()).sum();
+        let min_total = size * 100;
+
+        println!(
+            "samples={} total={}KB min_required={}KB",
+            samples.len(),
+            total / 1024,
+            min_total / 1024
+        );
+
+        let dict = zstd::dict::from_samples(samples, size).unwrap();
+        let compressed_sizes: usize = samples
+            .iter()
+            .map(|s| {
+                zstd::bulk::Compressor::with_dictionary(3, &dict)
+                    .unwrap()
+                    .compress(s.as_bytes())
+                    .unwrap()
+                    .len()
+            })
+            .sum();
+        let original_sizes: usize = samples.iter().map(|s| s.len()).sum();
+        println!(
+            "dict={}KB ratio={:.1}x",
+            size / 1024,
+            original_sizes as f64 / compressed_sizes as f64
+        );
+    }
+}
+
+fn main() -> Result<(), anyhow::Error> {
+    let conn = Connection::open(get_db_dir().join("yomitan.db"))?;
+
+    let max_id: i64 = conn.query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get(0))?;
+    let step = max_id / 1000;
+
+    let samples: Vec<String> = conn
+        .prepare("SELECT content FROM glossaries WHERE id % ?1 = 0 LIMIT 1000")?
+        .query_map(params![step], |r| r.get::<_, String>(0))?
+        .filter_map(|r| r.ok())
+        .collect();
+
+    println!("{} {}", step, samples.len());
+
+    benchmark_dict_sizes(&samples);
+
+    Ok(())
+}

src-core/examples/benchmark_zstd_decomp.rs

@@ -0,0 +1,79 @@
+use anyhow::{Ok, Result};
+
+mod common;
+use common::get_db_dir;
+use rusqlite::Connection;
+
+#[allow(unused)]
+fn benchmark_decompressor(conn: &Connection) -> Result<()> {
+    let zstd_dict: Vec<u8> =
+        conn.query_row("SELECT value FROM meta WHERE key = 'zstd_dict'", [], |r| {
+            r.get(0)
+        })?;
+
+    let samples: Vec<Vec<u8>> = conn
+        .prepare("SELECT b FROM glossaries WHERE id % 2111 = 0 LIMIT 100")?
+        .query_map([], |r| r.get::<_, Vec<u8>>(0))?
+        .filter_map(|r| r.ok())
+        .collect();
+
+    // benchmark: new decompressor per call
+    let start = std::time::Instant::now();
+    for sample in &samples {
+        let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&zstd_dict)?;
+        let _ = decompressor.decompress(sample, 1024 * 1024)?;
+    }
+
+    // new decompressor per call: 75.002µs avg
+    let per_call = start.elapsed() / samples.len() as u32;
+    println!("new decompressor per call: {:?} avg", per_call);
+
+    // benchmark: reuse decompressor
+    let start = std::time::Instant::now();
+    let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&zstd_dict)?;
+    for sample in &samples {
+        let _ = decompressor.decompress(sample, 1024 * 1024)?;
+    }
+    // reused decompressor:       44.231µs avg
+    let per_call = start.elapsed() / samples.len() as u32;
+    println!("reused decompressor:       {:?} avg", per_call);
+
+    Ok(())
+}
+
+// measure your actual maximum
+fn find_max_content_size(conn: &Connection) -> Result<()> {
+    // load dictionary once
+    let dict = conn.query_row("SELECT value FROM meta WHERE key = 'zstd_dict'", [], |r| {
+        r.get::<_, Vec<u8>>(0)
+    })?;
+
+    let (avg, max): (f64, i64) = conn.query_row(
+        "SELECT avg(length(b)), max(length(b)) FROM glossaries",
+        [],
+        |r| rusqlite::Result::Ok((r.get(0)?, r.get(1)?)),
+    )?;
+    println!("avg compressed: {:.0} bytes", avg);
+    println!("max compressed: {} bytes", max);
+
+    // decompress the largest one to find actual max decompressed size
+    let largest: Vec<u8> = conn.query_row(
+        "SELECT b FROM glossaries
+        WHERE length(b) = (SELECT max(length(b)) FROM glossaries)",
+        [],
+        |r| r.get(0),
+    )?;
+
+    // use a generous cap just to measure
+    let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&dict)?;
+    let decompressed = decompressor.decompress(&largest, 100 * 1024 * 1024)?;
+    println!("max decompressed: {} bytes", decompressed.len());
+
+    Ok(())
+}
+
+fn main() -> Result<(), anyhow::Error> {
+    let conn = Connection::open(get_db_dir().join("yomitan-glossary.db"))?;
+    find_max_content_size(&conn)?;
+    Ok(())
+}

src-core/examples/common/mod.rs

@@ -4,6 +4,7 @@ pub fn get_db_dir() -> PathBuf {
     PathBuf::from(r"C:\Users\HP\AppData\Roaming\cc.polv.cjdic2")
 }
 
+#[allow(unused)]
 pub fn get_vibrato_dict_dir() -> PathBuf {
     PathBuf::from("src-tauri/resources/naist-jdic-mecab/system.dic.zst")
 }

src-core/examples/remove_one_yomitan_dict.rs

@@ -11,7 +11,9 @@ fn main() -> Result<(), anyhow::Error> {
 
     {
         let _timer = Timer::new("remove one".to_string());
-        let mut writer = service.get_yomitan_writer()?;
+        let mut writer = service.get_yomitan_writer(|p| {
+            println!("{:?}", p);
+        })?;
         AppService::remove_yomitan_dictionary(&mut writer, "JMnedict.zip", "ja")?;
     }
 

src-core/examples/yomitan_update_schema.rs

@@ -11,7 +11,9 @@ fn main() -> Result<(), anyhow::Error> {
 
     {
         let _timer = Timer::new("update schema".to_string());
-        service.get_yomitan_writer()?;
+        service.get_yomitan_writer(|p| {
+            println!("{:?}", p);
+        })?;
     }
 
     Ok(())

src-core/src/db.rs

@@ -14,19 +14,24 @@ mod yomitan;
 pub use yomitan::YomitanRow;
 
 mod yomitan_writer;
-pub use yomitan_writer::{
+pub use yomitan_writer::{YomitanProgress, YomitanWriter, YomitanZipImportResult, ZipSource};
-    YomitanWriter, YomitanZipImportProgress, YomitanZipImportResult, ZipSource,
-};
 
 #[derive(Clone)]
 pub(crate) struct Database {
     pub(crate) conn: Arc<Mutex<Connection>>,
     pub(crate) dir: PathBuf,
     yomitan_attached: Arc<AtomicBool>,
+    yomitan_glossary_attached: Arc<AtomicBool>,
+}
+
+enum DbChild {
+    Yomitan,
+    YomitanGlossary,
 }
 
 const USER_DBFILE: &str = "user.db";
 pub(crate) const YOMITAN_DBFILE: &str = "yomitan.db";
+pub(crate) const YOMITAN_GLOSSARY_DBFILE: &str = "yomitan-glossary.db";
 
 impl Database {
     pub fn new(db_dir: impl AsRef<Path>) -> Result<Self> {
@@ -46,31 +51,55 @@ impl Database {
             conn: Arc::new(Mutex::new(conn)),
             dir,
             yomitan_attached: Arc::new(AtomicBool::new(false)),
+            yomitan_glossary_attached: Arc::new(AtomicBool::new(false)),
         })
     }
 
     pub fn yomitan(&self) -> Result<yomitan::YomitanDatabase> {
-        self.ensure_yomitan_attached()?;
+        self.ensure_db_attached(DbChild::Yomitan)?;
+        self.ensure_db_attached(DbChild::YomitanGlossary)?;
         Ok(yomitan::YomitanDatabase::new(self.clone()))
     }
 
-    fn ensure_yomitan_attached(&self) -> Result<()> {
+    fn ensure_db_attached(&self, db_child: DbChild) -> Result<()> {
+        match db_child {
+            DbChild::Yomitan => {
                 if self.yomitan_attached.load(Ordering::Acquire) {
                     return Ok(());
                 }
+            }
+            DbChild::YomitanGlossary => {
+                if self.yomitan_glossary_attached.load(Ordering::Acquire) {
+                    return Ok(());
+                }
+            }
+        }
 
         let conn = self.conn.lock().unwrap();
-        if !self.yomitan_attached.load(Ordering::Relaxed) {
+
+        let is_attached = match db_child {
+            DbChild::Yomitan => self.yomitan_attached.load(Ordering::Relaxed),
+            DbChild::YomitanGlossary => self.yomitan_glossary_attached.load(Ordering::Relaxed),
+        };
+
+        if !is_attached {
             let dir = self.dir.clone();
             let path = if self.dir.is_absolute() {
                 dir
             } else {
                 current_dir()?.join(dir)
             };
-            let path = path
+
-                .join(YOMITAN_DBFILE)
+            let db_file = match db_child {
-                .to_string_lossy()
+                DbChild::Yomitan => YOMITAN_DBFILE,
-                .replace(r"\", r"/");
+                DbChild::YomitanGlossary => YOMITAN_GLOSSARY_DBFILE,
+            };
+            let schema_name = match db_child {
+                DbChild::Yomitan => "yomitan",
+                DbChild::YomitanGlossary => "glossary",
+            };
+
+            let path = path.join(db_file).to_string_lossy().replace(r"\", r"/");
 
             #[cfg(windows)]
             let path = if !path.starts_with("/") {
@@ -81,15 +110,20 @@ impl Database {
 
             let uri = format!("file:{}?mode=ro&immutable=1", path);
 
-            conn.execute("ATTACH DATABASE ?1 AS yomitan", [uri])?;
+            conn.execute(&format!("ATTACH DATABASE ?1 AS {}", schema_name), [uri])?;
-
+            conn.execute_batch(&format!(
-            conn.execute_batch(
                 r"
-                PRAGMA yomitan.mmap_size = 3000000000; -- 3_000_000_000
+                PRAGMA {}.mmap_size = 3000000000; -- 3_000_000_000
                 ",
-            )?;
+                schema_name
+            ))?;
 
-            self.yomitan_attached.store(true, Ordering::Release);
+            match db_child {
+                DbChild::Yomitan => self.yomitan_attached.store(true, Ordering::Release),
+                DbChild::YomitanGlossary => self
+                    .yomitan_glossary_attached
+                    .store(true, Ordering::Release),
+            };
         }
 
         Ok(())

src-core/src/db/yomitan.rs

@@ -1,5 +1,6 @@
 use rusqlite::{Result, params};
 use serde::Serialize;
+use zstd::bulk::Decompressor;
 
 use crate::{
     CJDicError, Timer,
@@ -14,6 +15,7 @@ pub struct YomitanRow {
     rules: String,
     score: i64,
     glossary_json: String,
+    glossary_b: Vec<u8>,
     sequence: Option<i64>,
     term_tags: String,
     dict_title: String,
@@ -28,6 +30,19 @@ impl YomitanDatabase {
         Self { db }
     }
 
+    fn decompressor(&self) -> Result<Decompressor<'_>, CJDicError> {
+        let conn = self.db.conn.lock().unwrap();
+
+        let row = conn.query_row(
+            "SELECT value FROM glossary.meta WHERE key = 'zstd_dict'",
+            [],
+            |r| r.get::<_, Vec<u8>>(0),
+        )?;
+
+        let decompressor = zstd::bulk::Decompressor::with_dictionary(&row)?;
+        Ok(decompressor)
+    }
+
     pub fn search_yomitan(
         &self,
         q_term: &str,
@@ -37,6 +52,11 @@ impl YomitanDatabase {
     ) -> Result<Vec<YomitanRow>, CJDicError> {
         let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading));
 
+        // must be run before self.db.conn.lock()
+        let mut decompressor = self.decompressor()?;
+        // max decompressed: 90KB  (compression ratio on largest: ~2.5x)
+        const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom
+
         let and_or = if q_term == q_reading { "OR" } else { "AND" };
 
         let mut new_t = String::new();
@@ -60,7 +80,6 @@ impl YomitanDatabase {
         let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
 
         let conn = self.db.conn.lock().unwrap();
-
         conn.execute_batch(
             "
             CREATE TEMP TABLE IF NOT EXISTS yomitan_lookup_keys (
@@ -104,7 +123,7 @@ impl YomitanDatabase {
         // still the most time bottleneck step, several seconds
         conn.execute(&sql_string, params![q_term_norm, q_reading, limit, offset])?;
 
-        // let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading));
+        let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading));
         let mut stmt = conn.prepare(
             "
             SELECT
@@ -113,12 +132,12 @@ impl YomitanDatabase {
                 COALESCE(dt.tags,  '')  AS def_tags,
                 COALESCE(r.rules,  '')  AS rules,
                 t.score,
-                g.content               AS glossary_json,
+                COALESCE(g.b, '[]')     AS glossary,
                 t.sequence,
                 COALESCE(tt.tags,  '')  AS term_tags,
                 t.title                 AS dict_title
             FROM yomitan_lookup_keys t
-            JOIN yomitan.glossaries         g  ON g.id  = t.glossary_id
+            LEFT JOIN glossary.glossaries    g  ON g.id  = t.glossary_id
             LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
             LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
             LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id
@@ -131,7 +150,8 @@ impl YomitanDatabase {
                 def_tags: r.get(2)?,
                 rules: r.get(3)?,
                 score: r.get(4)?,
-                glossary_json: r.get(5)?,
+                glossary_json: String::new(),
+                glossary_b: r.get(5)?,
                 sequence: r.get(6)?,
                 term_tags: r.get(7)?,
                 dict_title: r.get(8)?,
@@ -139,8 +159,13 @@ impl YomitanDatabase {
         })?;
 
         let mut out = Vec::new();
-        for r in rows {
+        for row in rows {
-            out.push(r?);
+            let mut r = row?;
+            r.glossary_json =
+                String::from_utf8(decompressor.decompress(&r.glossary_b, MAX_DECOMPRESSED_SIZE)?)?;
+            r.glossary_b = vec![];
+
+            out.push(r);
         }
         Ok(out)
     }

src-core/src/db/yomitan_writer.rs

@@ -1,16 +1,20 @@
 use blake3::hash;
-use rusqlite::{Connection, Result, Transaction, params};
+use rusqlite::{Connection, OptionalExtension, Result, Transaction, params};
 use serde::Serialize;
 use serde_json::Value;
 use std::{
     borrow::Cow,
     collections::HashMap,
+    env::current_dir,
     io::{Cursor, Read},
     path::{Path, PathBuf},
 };
 use zip::ZipArchive;
 
-use crate::{CJDicError, Timer};
+use crate::{
+    CJDicError, Timer,
+    db::{YOMITAN_DBFILE, YOMITAN_GLOSSARY_DBFILE},
+};
 
 fn blake3_hex(s: &str) -> String {
     format!("{}", hash(s.as_bytes()))
@@ -39,11 +43,11 @@ impl ZipSource for PathBuf {
 }
 
 #[derive(Serialize, Debug, Clone)]
-pub struct YomitanZipImportProgress {
+pub struct YomitanProgress {
     pub message: String,
-    pub current: u32,
+    pub current: usize,
-    pub total: u32,
+    pub total: usize,
-    pub steps: u64,
+    pub steps: usize,
 }
 
 #[derive(Serialize, Debug)]
@@ -54,11 +58,15 @@ pub struct YomitanZipImportResult {
 }
 
 pub struct YomitanWriter {
+    dir: PathBuf,
     conn: Connection,
 }
 
 impl YomitanWriter {
-    pub fn new(conn: Connection) -> Result<Self> {
+    pub fn new(db_dir: impl AsRef<Path>) -> Result<Self> {
+        let dir = db_dir.as_ref().to_path_buf();
+        let conn = Connection::open(dir.join(YOMITAN_DBFILE))?;
+
         conn.execute_batch(
             "
             PRAGMA journal_mode = WAL;
@@ -69,15 +77,13 @@ impl YomitanWriter {
             ",
         )?;
 
-        Ok(Self { conn })
+        Ok(Self { dir, conn })
     }
 
-    pub fn cleanup(&self) -> Result<()> {
+    pub fn create_schema(
-        self.conn.execute_batch("VACUUM")?;
+        &mut self,
-        Ok(())
+        progress_callback: impl Fn(YomitanProgress),
-    }
+    ) -> Result<(), CJDicError> {
-
-    pub fn create_schema(&mut self) -> Result<()> {
         self.conn.execute_batch(
         "
             CREATE TABLE IF NOT EXISTS schema_meta (
@@ -97,9 +103,9 @@ impl YomitanWriter {
                 lang         TEXT    NOT NULL,      -- custom
                 UNIQUE (title, revision)
             );
-            CREATE TABLE IF NOT EXISTS glossaries (
+            CREATE TABLE IF NOT EXISTS glossaries (     -- not used. moved to a new file
                 id      INTEGER PRIMARY KEY,
-                hash    TEXT NOT NULL UNIQUE,
+                hash    TEXT NOT NULL UNIQUE,           -- maybe used in the future for excerpts
                 content TEXT NOT NULL
             );
             CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
@@ -168,14 +174,89 @@ impl YomitanWriter {
         )?;
 
         self.create_materialized_views()?;
+        self.attach_glossary()?;
 
+        let schema_version = "2";
+
+        let current_schema_version = {
             let mut stmt = self
                 .conn
-            .prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?;
+                .prepare("SELECT value FROM schema_meta WHERE key = 'schema_version'")?;
-        if stmt.exists([])? == false {
+
+            stmt.query_one([], |r| r.get::<_, String>(0)).optional()?
+        };
+
+        if let Some(v) = current_schema_version {
+            if v < "2".to_string() {
+                let total = self
+                    .conn
+                    .query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))?
+                    as usize;
+
+                let message = &format!("migrate to {}", YOMITAN_GLOSSARY_DBFILE);
+                progress_callback(YomitanProgress {
+                    message: message.to_string(),
+                    current: 0,
+                    total,
+                    steps: 0,
+                });
+
+                let _timer = Timer::new(message.to_string());
+
+                let tx = self.conn.transaction()?;
+                {
+                    let mut stmt = tx.prepare("SELECT id, hash, content FROM glossaries")?;
+
+                    let rows = stmt.query_map([], |r| {
+                        Ok((
+                            r.get::<_, i64>(0)?,
+                            r.get::<_, String>(1)?,
+                            r.get::<_, String>(2)?,
+                        ))
+                    })?;
+
+                    let mut stmt_write = tx.prepare(
+                        "INSERT INTO glossary.glossaries (id, hash, b) VALUES (?1, ?2, ?3)",
+                    )?;
+
+                    for (i, row) in rows.enumerate() {
+                        let (id, hash, glossary_json) = row?;
+                        stmt_write.execute(params![
+                            id,
+                            hash,
+                            zstd::encode_all(glossary_json.as_bytes(), 3)?
+                        ])?;
+
+                        if i % 1000 == 0 {
+                            progress_callback(YomitanProgress {
+                                message: message.to_string(),
+                                current: i,
+                                total,
+                                steps: 0,
+                            });
+                        }
+                    }
+                }
+                tx.commit()?;
+
+                self.hash_glossary()?;
+
+                let _timer = Timer::new(format!("cleanup for {}", YOMITAN_GLOSSARY_DBFILE));
+                self.conn.execute_batch(
+                    "
+                    UPDATE glossaries SET content = '[]';
+                    VACUUM;
+                ",
+                )?;
+            }
+
             self.conn
-                .prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")?
+                .prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")?
-                .execute([])?;
+                .execute(["schema_version", schema_version])?;
+        } else {
+            self.conn
+                .prepare("INSERT INTO schema_meta (key, value) VALUES (?1, ?2)")?
+                .execute(["schema_version", schema_version])?;
             self.conn
                 .prepare(
                     "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))",
@@ -263,6 +344,86 @@ impl YomitanWriter {
         Ok(())
     }
 
+    fn attach_glossary(&mut self) -> Result<(), CJDicError> {
+        let dir = self.dir.clone();
+        let path = if self.dir.is_absolute() {
+            dir
+        } else {
+            current_dir()?.join(dir)
+        };
+
+        let db_file = YOMITAN_GLOSSARY_DBFILE;
+        let schema_name = "glossary";
+
+        let path = path.join(db_file).to_string_lossy().replace(r"\", r"/");
+
+        #[cfg(windows)]
+        let path = if !path.starts_with("/") {
+            format!("/{}", path)
+        } else {
+            path.to_string()
+        };
+
+        let uri = format!("file:{}", path);
+
+        self.conn
+            .execute(&format!("ATTACH DATABASE ?1 AS {}", schema_name), [uri])?;
+        self.conn.execute_batch(&format!(
+            r"
+                PRAGMA {}.mmap_size = 3000000000; -- 3_000_000_000
+                ",
+            schema_name
+        ))?;
+
+        self.create_schema_glossary()?;
+
+        Ok(())
+    }
+
+    fn create_schema_glossary(&self) -> Result<()> {
+        self.conn.execute_batch(&format!(
+            "
+            CREATE TABLE IF NOT EXISTS glossary.meta (
+                key     TEXT PRIMARY KEY,
+                value   BLOB NOT NULL
+            );
+
+            CREATE TABLE IF NOT EXISTS glossary.glossaries (
+                id      INTEGER PRIMARY KEY,
+                hash    TEXT NOT NULL UNIQUE,
+                b       BLOB NOT NULL
+            );
+        "
+        ))?;
+
+        Ok(())
+    }
+
+    fn hash_glossary(&self) -> Result<(), CJDicError> {
+        let max_id: i64 =
+            self.conn
+                .query_row("SELECT MAX(id) FROM glossary.glossaries", [], |r| r.get(0))?;
+
+        let step = max_id / 1000;
+
+        let samples: Vec<Vec<u8>> = self
+            .conn
+            .prepare("SELECT b FROM glossary.glossaries WHERE id % ?1 = 0 LIMIT 1000")?
+            .query_map(params![step], |r| r.get::<_, Vec<u8>>(0))?
+            .filter_map(|r| r.ok())
+            .collect();
+
+        let dict_size = 112640; // 110KB default
+        let dictionary = zstd::dict::from_samples(&samples, dict_size)?;
+
+        self.conn.execute(
+            "INSERT OR REPLACE INTO glossary.meta (key, value) VALUES (?1, ?2)",
+            params!["zstd_dict", dictionary],
+        )?;
+
+        Ok(())
+    }
+
     pub fn import_dictionary_zip_file<Z, Callback>(
         &mut self,
         zip_file: &Z,
@@ -271,7 +432,7 @@ impl YomitanWriter {
     ) -> anyhow::Result<YomitanZipImportResult, CJDicError>
     where
         Z: ZipSource,
-        Callback: Fn(YomitanZipImportProgress),
+        Callback: Fn(YomitanProgress),
     {
         let bundle_name = zip_file.file_name();
         let _timer = Timer::new(format!("Importing {}", bundle_name));
@@ -298,8 +459,8 @@ impl YomitanWriter {
             }
         };
 
-        let mut steps: u64 = 0;
+        let mut steps: usize = 0;
-        progress_callback(YomitanZipImportProgress {
+        progress_callback(YomitanProgress {
             message: format!("Opening {}", bundle_name),
             current: 0,
             total: 0,
@@ -351,6 +512,9 @@ impl YomitanWriter {
             }
         }
 
+        self.attach_glossary()?;
+
+        {
             let tx = self.conn.transaction()?;
             tx.execute(
                 "INSERT INTO dictionaries
@@ -370,8 +534,9 @@ impl YomitanWriter {
             let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?;
 
             // prepare statements
-        let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)";
+            let insert_glossary =
-        let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1";
+                "INSERT OR IGNORE INTO glossary.glossaries (hash, b) VALUES (?1, ?2)";
+            let select_glossary = "SELECT id FROM glossary.glossaries WHERE hash = ?1";
             let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)";
             let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1";
             let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)";
@@ -379,8 +544,7 @@ impl YomitanWriter {
             let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)";
             let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1";
             let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)";
-        let insert_meta =
+            let insert_meta = "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
-            "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
             let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)";
 
             let mut glossary_cache: HashMap<String, i64> = HashMap::new();
@@ -388,7 +552,7 @@ impl YomitanWriter {
             let mut term_tags_cache: HashMap<String, i64> = HashMap::new();
             let mut rules_cache: HashMap<String, i64> = HashMap::new();
 
-        let mut n_bank: u32 = 0;
+            let mut n_bank: usize = 0;
 
             for filename in archive.file_names() {
                 let starts_with = "term_bank_";
@@ -399,7 +563,7 @@ impl YomitanWriter {
 
                     if let Some(stop_idx) = text[start_idx..].find(".") {
                         let sub = &text[start_idx..(start_idx + stop_idx)];
-                    if let Some(i_bank) = sub.parse::<u32>().ok() {
+                        if let Some(i_bank) = sub.parse::<usize>().ok() {
                             if i_bank > n_bank {
                                 n_bank = i_bank;
                             }
@@ -418,8 +582,8 @@ impl YomitanWriter {
                         f.read_to_string(&mut s)?;
                         let entries: Vec<Value> = serde_json::from_str(&s)?;
 
-                    steps += entries.len() as u64;
+                        steps += entries.len();
-                    progress_callback(YomitanZipImportProgress {
+                        progress_callback(YomitanProgress {
                             message: format!("Reading term banks"),
                             current: bank_i,
                             total: n_bank,
@@ -456,7 +620,10 @@ impl YomitanWriter {
                             let glossary_id = if let Some(&id) = glossary_cache.get(&hash) {
                                 id
                             } else {
-                            tx.execute(insert_glossary, params![hash, glossary_json])?;
+                                tx.execute(
+                                    insert_glossary,
+                                    params![hash, zstd::encode_all(glossary_json.as_bytes(), 3)?],
+                                )?;
                                 let id: i64 =
                                     tx.query_row(select_glossary, params![hash], |r| r.get(0))?;
                                 glossary_cache.insert(hash.clone(), id);
@@ -520,7 +687,7 @@ impl YomitanWriter {
             }
 
             steps += 1;
-        progress_callback(YomitanZipImportProgress {
+            progress_callback(YomitanProgress {
                 message: format!("Reading term meta banks"),
                 current: 0,
                 total: 0,
@@ -546,7 +713,13 @@ impl YomitanWriter {
                                 .map(|s| s.to_string());
                             tx.execute(
                                 insert_meta,
-                            params![dict_id, term, mode, reading, serde_json::to_string(&data)?],
+                                params![
+                                    dict_id,
+                                    term,
+                                    mode,
+                                    reading,
+                                    serde_json::to_string(&data)?
+                                ],
                             )?;
                         }
                         meta_i += 1;
@@ -557,7 +730,7 @@ impl YomitanWriter {
             }
 
             steps += 1;
-        progress_callback(YomitanZipImportProgress {
+            progress_callback(YomitanProgress {
                 message: format!("Reading tag banks"),
                 current: 0,
                 total: 0,
@@ -592,6 +765,7 @@ impl YomitanWriter {
             }
 
             tx.commit()?;
+        }
 
         self.conn.execute_batch(
             "
@@ -600,6 +774,8 @@ impl YomitanWriter {
             ",
         )?; // < 1 ms
 
+        self.hash_glossary()?;
+
         // self.conn.execute_batch("VACUUM;")?; // Not needed, twice time taken than import
 
         Ok(YomitanZipImportResult {

src-core/src/error.rs

@@ -1,3 +1,5 @@
+use std::string::FromUtf8Error;
+
 use serde::Serialize;
 use thiserror::Error;
 use vibrato::errors::VibratoError;
@@ -27,6 +29,9 @@ pub enum CJDicError {
     #[error("Vibrato error: {0}")]
     VibratoError(String),
 
+    #[error("FromUtf8Error: {0}")]
+    FromUtf8Error(String),
+
     #[error("Not found")]
     NotFound,
 
@@ -36,42 +41,49 @@ pub enum CJDicError {
 
 impl From<rusqlite::Error> for CJDicError {
     fn from(e: rusqlite::Error) -> Self {
-        log::error!("{e:#}");
+        eprintln!("{e:#}");
         CJDicError::Database(e.to_string())
     }
 }
 
 impl From<serde_json::Error> for CJDicError {
     fn from(e: serde_json::Error) -> Self {
-        log::error!("{e:#}");
+        eprintln!("{e:#}");
         CJDicError::Serialization(e.to_string())
     }
 }
 
 impl From<anyhow::Error> for CJDicError {
     fn from(e: anyhow::Error) -> Self {
-        log::error!("{e:#}");
+        eprintln!("{e:#}");
         CJDicError::AnyhowError(e.to_string())
     }
 }
 
 impl From<std::io::Error> for CJDicError {
     fn from(e: std::io::Error) -> Self {
-        log::error!("{e:#}");
+        eprintln!("{e:#}");
         CJDicError::IOError(e.to_string())
     }
 }
 
 impl From<ZipError> for CJDicError {
     fn from(e: ZipError) -> Self {
-        log::error!("{e:#}");
+        eprintln!("{e:#}");
         CJDicError::ZipError(e.to_string())
     }
 }
 
 impl From<VibratoError> for CJDicError {
     fn from(e: VibratoError) -> Self {
-        log::error!("{e:#}");
+        eprintln!("{e:#}");
         CJDicError::VibratoError(e.to_string())
     }
 }
+
+impl From<FromUtf8Error> for CJDicError {
+    fn from(e: FromUtf8Error) -> Self {
+        eprintln!("{e:#}");
+        CJDicError::FromUtf8Error(e.to_string())
+    }
+}

src-core/src/lib.rs

@@ -3,7 +3,7 @@ mod error;
 mod service;
 mod util;
 
-pub use db::{YomitanRow, YomitanZipImportProgress, YomitanZipImportResult, ZipSource};
+pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource};
 pub use error::CJDicError;
 pub use service::{AppService, SqlParam, TokenizeSegment};
 pub use util::Timer;

src-core/src/service.rs

@@ -11,7 +11,7 @@ use vibrato::{Dictionary, Tokenizer};
 use crate::{
     Timer, ZipSource,
     db::{
-        Database, YOMITAN_DBFILE, YomitanRow, YomitanWriter, YomitanZipImportProgress,
+        Database, YOMITAN_DBFILE, YomitanProgress, YomitanRow, YomitanWriter,
         YomitanZipImportResult,
     },
     error::CJDicError,
@@ -149,17 +149,12 @@ impl AppService {
             .search_yomitan(q_term, q_reading, limit, offset)?)
     }
 
-    pub fn get_yomitan_writer(&self) -> Result<YomitanWriter, CJDicError> {
+    pub fn get_yomitan_writer(
-        let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?;
+        &self,
-        let mut writer = YomitanWriter::new(conn)?;
+        progress_callback: impl Fn(YomitanProgress),
-        writer.create_schema()?;
+    ) -> Result<YomitanWriter, CJDicError> {
-        Ok(writer)
+        let mut writer = YomitanWriter::new(self.db.dir.clone())?;
-    }
+        writer.create_schema(&progress_callback)?;
-
-    pub fn cleanup_yomitan_writer(&self) -> Result<YomitanWriter, CJDicError> {
-        let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?;
-        let writer = YomitanWriter::new(conn)?;
-        writer.cleanup()?;
         Ok(writer)
     }
 
@@ -173,7 +168,7 @@ impl AppService {
     where
         Z: ZipSource,
         LoadCallback: Fn(LoadYomitanZipDirResult),
-        ImportCallback: Fn(YomitanZipImportProgress),
+        ImportCallback: Fn(YomitanProgress),
     {
         let mut dir_zip_map: HashMap<String, &Z> = HashMap::new();
         let mut zip_dir_ord: HashMap<String, usize> = HashMap::new();
@@ -184,17 +179,13 @@ impl AppService {
             zip_dir_ord.insert(filename.to_string(), i);
         }
 
-        {
-            let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?;
-            let mut writer = YomitanWriter::new(conn)?;
-            writer.create_schema()?;
-        }
-
         let mut new_dicts: Vec<String> = vec![];
         let mut to_be_removed_dicts: Vec<String> = vec![];
 
-        {
+        let db_path = self.db.dir.join(YOMITAN_DBFILE);
-            let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?;
+
+        if db_path.exists() {
+            let conn = Connection::open(&db_path)?;
             let mut stmt =
                 conn.prepare("SELECT DISTINCT bundle_name FROM dictionaries WHERE lang = ?1")?;
             let mut rows = stmt.query([lang])?;
@@ -228,7 +219,12 @@ impl AppService {
         });
 
         {
-            let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?;
+            let mut writer = YomitanWriter::new(self.db.dir.clone())?;
+            writer.create_schema(&import_callback)?;
+        }
+
+        {
+            let conn = Connection::open(&db_path)?;
             for z in to_be_removed_dicts.clone().iter() {
                 conn.execute(
                     "DELETE FROM dictionaries WHERE bundle_name = ?1 AND lang = ?2",
@@ -238,8 +234,7 @@ impl AppService {
         }
 
         {
-            let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?;
+            let mut writer = YomitanWriter::new(self.db.dir.clone())?;
-            let mut writer = YomitanWriter::new(conn)?;
 
             for z in new_dicts.clone().iter() {
                 if let Some(zip_file) = dir_zip_map.get(z) {
@@ -262,7 +257,7 @@ impl AppService {
     ) -> Result<YomitanZipImportResult, CJDicError>
     where
         Z: ZipSource,
-        Callback: Fn(YomitanZipImportProgress),
+        Callback: Fn(YomitanProgress),
     {
         Ok(writer.import_dictionary_zip_file(zip_file, lang, progress_callback)?)
     }

src-tauri/src/command.rs

@@ -97,7 +97,9 @@ pub async fn import_yomitan_dict(
     data_dir.push(lang);
     data_dir.push(bundle_name);
 
-    let mut writer = state.get_yomitan_writer()?;
+    let mut writer = state.get_yomitan_writer(|r| {
+        app.emit("yomitan-import-progress", r).unwrap();
+    })?;
     AppService::import_yomitan_zip_file(&mut writer, &data_dir, lang, |r| {
         app.emit("yomitan-import-progress", r).unwrap();
     })
@@ -105,11 +107,14 @@ pub async fn import_yomitan_dict(
 
 #[tauri::command]
 pub async fn remove_yomitan_dict(
+    app: AppHandle,
     bundle_name: &str,
     lang: &str,
     state: tauri::State<'_, AppService>,
 ) -> Result<(), CJDicError> {
-    let mut writer = state.get_yomitan_writer()?;
+    let mut writer = state.get_yomitan_writer(|r| {
+        app.emit("yomitan-import-progress", r).unwrap();
+    })?;
     AppService::remove_yomitan_dictionary(&mut writer, bundle_name, lang)
 }