move glossary (def text) to a separate db file

src-core/examples/add_yomitan_dict_folder.rs

@@ -13,7 +13,9 @@ use crate::common::get_vibrato_dict_dir;
 
 fn main() -> Result<(), anyhow::Error> {
     let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
-    let mut writer = service.get_yomitan_writer()?;
+    let mut writer = service.get_yomitan_writer(|p| {
+        println!("{:?}", p);
+    })?;
 
     let zip_dir = Path::new("src-tauri/resources/yomitan");
     println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root

src-core/examples/benchmark_json_zstd.rs

@@ -0,0 +1,57 @@
+use anyhow::Ok;
+
+mod common;
+use common::get_db_dir;
+use rusqlite::{Connection, params};
+
+fn benchmark_dict_sizes(samples: &[String]) {
+    // 112640; // 110KB default gives the best 5.8x compression
+    for size in [16384, 65536, 112640, 524288] {
+        let total: usize = samples.iter().map(|s| s.len()).sum();
+        let min_total = size * 100;
+
+        println!(
+            "samples={} total={}KB min_required={}KB",
+            samples.len(),
+            total / 1024,
+            min_total / 1024
+        );
+
+        let dict = zstd::dict::from_samples(samples, size).unwrap();
+        let compressed_sizes: usize = samples
+            .iter()
+            .map(|s| {
+                zstd::bulk::Compressor::with_dictionary(3, &dict)
+                    .unwrap()
+                    .compress(s.as_bytes())
+                    .unwrap()
+                    .len()
+            })
+            .sum();
+        let original_sizes: usize = samples.iter().map(|s| s.len()).sum();
+        println!(
+            "dict={}KB ratio={:.1}x",
+            size / 1024,
+            original_sizes as f64 / compressed_sizes as f64
+        );
+    }
+}
+
+fn main() -> Result<(), anyhow::Error> {
+    let conn = Connection::open(get_db_dir().join("yomitan.db"))?;
+
+    let max_id: i64 = conn.query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get(0))?;
+    let step = max_id / 1000;
+
+    let samples: Vec<String> = conn
+        .prepare("SELECT content FROM glossaries WHERE id % ?1 = 0 LIMIT 1000")?
+        .query_map(params![step], |r| r.get::<_, String>(0))?
+        .filter_map(|r| r.ok())
+        .collect();
+
+    println!("{} {}", step, samples.len());
+
+    benchmark_dict_sizes(&samples);
+
+    Ok(())
+}

src-core/examples/benchmark_zstd_decomp.rs

@@ -0,0 +1,79 @@
+use anyhow::{Ok, Result};
+
+mod common;
+use common::get_db_dir;
+use rusqlite::Connection;
+
+#[allow(unused)]
+fn benchmark_decompressor(conn: &Connection) -> Result<()> {
+    let zstd_dict: Vec<u8> =
+        conn.query_row("SELECT value FROM meta WHERE key = 'zstd_dict'", [], |r| {
+            r.get(0)
+        })?;
+
+    let samples: Vec<Vec<u8>> = conn
+        .prepare("SELECT b FROM glossaries WHERE id % 2111 = 0 LIMIT 100")?
+        .query_map([], |r| r.get::<_, Vec<u8>>(0))?
+        .filter_map(|r| r.ok())
+        .collect();
+
+    // benchmark: new decompressor per call
+    let start = std::time::Instant::now();
+    for sample in &samples {
+        let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&zstd_dict)?;
+        let _ = decompressor.decompress(sample, 1024 * 1024)?;
+    }
+
+    // new decompressor per call: 75.002µs avg
+    let per_call = start.elapsed() / samples.len() as u32;
+    println!("new decompressor per call: {:?} avg", per_call);
+
+    // benchmark: reuse decompressor
+    let start = std::time::Instant::now();
+    let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&zstd_dict)?;
+    for sample in &samples {
+        let _ = decompressor.decompress(sample, 1024 * 1024)?;
+    }
+    // reused decompressor:       44.231µs avg
+    let per_call = start.elapsed() / samples.len() as u32;
+    println!("reused decompressor:       {:?} avg", per_call);
+
+    Ok(())
+}
+
+// measure your actual maximum
+fn find_max_content_size(conn: &Connection) -> Result<()> {
+    // load dictionary once
+    let dict = conn.query_row("SELECT value FROM meta WHERE key = 'zstd_dict'", [], |r| {
+        r.get::<_, Vec<u8>>(0)
+    })?;
+
+    let (avg, max): (f64, i64) = conn.query_row(
+        "SELECT avg(length(b)), max(length(b)) FROM glossaries",
+        [],
+        |r| rusqlite::Result::Ok((r.get(0)?, r.get(1)?)),
+    )?;
+    println!("avg compressed: {:.0} bytes", avg);
+    println!("max compressed: {} bytes", max);
+
+    // decompress the largest one to find actual max decompressed size
+    let largest: Vec<u8> = conn.query_row(
+        "SELECT b FROM glossaries
+        WHERE length(b) = (SELECT max(length(b)) FROM glossaries)",
+        [],
+        |r| r.get(0),
+    )?;
+
+    // use a generous cap just to measure
+    let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&dict)?;
+    let decompressed = decompressor.decompress(&largest, 100 * 1024 * 1024)?;
+    println!("max decompressed: {} bytes", decompressed.len());
+
+    Ok(())
+}
+
+fn main() -> Result<(), anyhow::Error> {
+    let conn = Connection::open(get_db_dir().join("yomitan-glossary.db"))?;
+    find_max_content_size(&conn)?;
+    Ok(())
+}

src-core/examples/common/mod.rs

@@ -4,6 +4,7 @@ pub fn get_db_dir() -> PathBuf {
     PathBuf::from(r"C:\Users\HP\AppData\Roaming\cc.polv.cjdic2")
 }
 
+#[allow(unused)]
 pub fn get_vibrato_dict_dir() -> PathBuf {
     PathBuf::from("src-tauri/resources/naist-jdic-mecab/system.dic.zst")
 }

src-core/examples/remove_one_yomitan_dict.rs

@@ -11,7 +11,9 @@ fn main() -> Result<(), anyhow::Error> {
 
     {
         let _timer = Timer::new("remove one".to_string());
-        let mut writer = service.get_yomitan_writer()?;
+        let mut writer = service.get_yomitan_writer(|p| {
+            println!("{:?}", p);
+        })?;
         AppService::remove_yomitan_dictionary(&mut writer, "JMnedict.zip", "ja")?;
     }
 

src-core/examples/yomitan_update_schema.rs

@@ -11,7 +11,9 @@ fn main() -> Result<(), anyhow::Error> {
 
     {
         let _timer = Timer::new("update schema".to_string());
-        service.get_yomitan_writer()?;
+        service.get_yomitan_writer(|p| {
+            println!("{:?}", p);
+        })?;
     }
 
     Ok(())