From afeb43fc2c353185887e96e90abb226deff70520 Mon Sep 17 00:00:00 2001 From: polv Date: Sat, 7 Mar 2026 08:10:48 +0700 Subject: [PATCH] move glossary (def text) to a separate db file --- Cargo.lock | 1 - src-core/Cargo.toml | 1 - src-core/examples/add_yomitan_dict_folder.rs | 4 +- src-core/examples/benchmark_json_zstd.rs | 57 ++ src-core/examples/benchmark_zstd_decomp.rs | 79 +++ src-core/examples/common/mod.rs | 1 + src-core/examples/remove_one_yomitan_dict.rs | 4 +- src-core/examples/yomitan_update_schema.rs | 4 +- src-core/src/db.rs | 70 +- src-core/src/db/yomitan.rs | 39 +- src-core/src/db/yomitan_writer.rs | 646 ++++++++++++------- src-core/src/error.rs | 24 +- src-core/src/lib.rs | 2 +- src-core/src/service.rs | 45 +- src-tauri/src/command.rs | 9 +- 15 files changed, 687 insertions(+), 299 deletions(-) create mode 100644 src-core/examples/benchmark_json_zstd.rs create mode 100644 src-core/examples/benchmark_zstd_decomp.rs diff --git a/Cargo.lock b/Cargo.lock index a34ae01..426060c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -639,7 +639,6 @@ version = "0.1.0" dependencies = [ "anyhow", "blake3", - "log", "rusqlite", "serde", "serde_json", diff --git a/src-core/Cargo.toml b/src-core/Cargo.toml index 2c8b230..de0d015 100644 --- a/src-core/Cargo.toml +++ b/src-core/Cargo.toml @@ -14,6 +14,5 @@ thiserror = "2.0.18" zip = "8.1.0" blake3 = "1.8.3" anyhow = "1.0.102" -log = "0.4.29" vibrato = "0.5.2" zstd = "0.13.3" diff --git a/src-core/examples/add_yomitan_dict_folder.rs b/src-core/examples/add_yomitan_dict_folder.rs index e4f4cb9..256d63f 100644 --- a/src-core/examples/add_yomitan_dict_folder.rs +++ b/src-core/examples/add_yomitan_dict_folder.rs @@ -13,7 +13,9 @@ use crate::common::get_vibrato_dict_dir; fn main() -> Result<(), anyhow::Error> { let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?; - let mut writer = service.get_yomitan_writer()?; + let mut writer = service.get_yomitan_writer(|p| { + println!("{:?}", p); + })?; let zip_dir = Path::new("src-tauri/resources/yomitan"); println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root diff --git a/src-core/examples/benchmark_json_zstd.rs b/src-core/examples/benchmark_json_zstd.rs new file mode 100644 index 0000000..33e70b9 --- /dev/null +++ b/src-core/examples/benchmark_json_zstd.rs @@ -0,0 +1,57 @@ +use anyhow::Ok; + +mod common; +use common::get_db_dir; +use rusqlite::{Connection, params}; + +fn benchmark_dict_sizes(samples: &[String]) { + // 112640; // 110KB default gives the best 5.8x compression + for size in [16384, 65536, 112640, 524288] { + let total: usize = samples.iter().map(|s| s.len()).sum(); + let min_total = size * 100; + + println!( + "samples={} total={}KB min_required={}KB", + samples.len(), + total / 1024, + min_total / 1024 + ); + + let dict = zstd::dict::from_samples(samples, size).unwrap(); + let compressed_sizes: usize = samples + .iter() + .map(|s| { + zstd::bulk::Compressor::with_dictionary(3, &dict) + .unwrap() + .compress(s.as_bytes()) + .unwrap() + .len() + }) + .sum(); + let original_sizes: usize = samples.iter().map(|s| s.len()).sum(); + println!( + "dict={}KB ratio={:.1}x", + size / 1024, + original_sizes as f64 / compressed_sizes as f64 + ); + } +} + +fn main() -> Result<(), anyhow::Error> { + let conn = Connection::open(get_db_dir().join("yomitan.db"))?; + + let max_id: i64 = conn.query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get(0))?; + let step = max_id / 1000; + + let samples: Vec = conn + .prepare("SELECT content FROM glossaries WHERE id % ?1 = 0 LIMIT 1000")? + .query_map(params![step], |r| r.get::<_, String>(0))? + .filter_map(|r| r.ok()) + .collect(); + + println!("{} {}", step, samples.len()); + + benchmark_dict_sizes(&samples); + + Ok(()) +} diff --git a/src-core/examples/benchmark_zstd_decomp.rs b/src-core/examples/benchmark_zstd_decomp.rs new file mode 100644 index 0000000..8a2bdd1 --- /dev/null +++ b/src-core/examples/benchmark_zstd_decomp.rs @@ -0,0 +1,79 @@ +use anyhow::{Ok, Result}; + +mod common; +use common::get_db_dir; +use rusqlite::Connection; + +#[allow(unused)] +fn benchmark_decompressor(conn: &Connection) -> Result<()> { + let zstd_dict: Vec = + conn.query_row("SELECT value FROM meta WHERE key = 'zstd_dict'", [], |r| { + r.get(0) + })?; + + let samples: Vec> = conn + .prepare("SELECT b FROM glossaries WHERE id % 2111 = 0 LIMIT 100")? + .query_map([], |r| r.get::<_, Vec>(0))? + .filter_map(|r| r.ok()) + .collect(); + + // benchmark: new decompressor per call + let start = std::time::Instant::now(); + for sample in &samples { + let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&zstd_dict)?; + let _ = decompressor.decompress(sample, 1024 * 1024)?; + } + + // new decompressor per call: 75.002µs avg + let per_call = start.elapsed() / samples.len() as u32; + println!("new decompressor per call: {:?} avg", per_call); + + // benchmark: reuse decompressor + let start = std::time::Instant::now(); + let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&zstd_dict)?; + for sample in &samples { + let _ = decompressor.decompress(sample, 1024 * 1024)?; + } + // reused decompressor: 44.231µs avg + let per_call = start.elapsed() / samples.len() as u32; + println!("reused decompressor: {:?} avg", per_call); + + Ok(()) +} + +// measure your actual maximum +fn find_max_content_size(conn: &Connection) -> Result<()> { + // load dictionary once + let dict = conn.query_row("SELECT value FROM meta WHERE key = 'zstd_dict'", [], |r| { + r.get::<_, Vec>(0) + })?; + + let (avg, max): (f64, i64) = conn.query_row( + "SELECT avg(length(b)), max(length(b)) FROM glossaries", + [], + |r| rusqlite::Result::Ok((r.get(0)?, r.get(1)?)), + )?; + println!("avg compressed: {:.0} bytes", avg); + println!("max compressed: {} bytes", max); + + // decompress the largest one to find actual max decompressed size + let largest: Vec = conn.query_row( + "SELECT b FROM glossaries + WHERE length(b) = (SELECT max(length(b)) FROM glossaries)", + [], + |r| r.get(0), + )?; + + // use a generous cap just to measure + let mut decompressor = zstd::bulk::Decompressor::with_dictionary(&dict)?; + let decompressed = decompressor.decompress(&largest, 100 * 1024 * 1024)?; + println!("max decompressed: {} bytes", decompressed.len()); + + Ok(()) +} + +fn main() -> Result<(), anyhow::Error> { + let conn = Connection::open(get_db_dir().join("yomitan-glossary.db"))?; + find_max_content_size(&conn)?; + Ok(()) +} diff --git a/src-core/examples/common/mod.rs b/src-core/examples/common/mod.rs index 83b520c..760c719 100644 --- a/src-core/examples/common/mod.rs +++ b/src-core/examples/common/mod.rs @@ -4,6 +4,7 @@ pub fn get_db_dir() -> PathBuf { PathBuf::from(r"C:\Users\HP\AppData\Roaming\cc.polv.cjdic2") } +#[allow(unused)] pub fn get_vibrato_dict_dir() -> PathBuf { PathBuf::from("src-tauri/resources/naist-jdic-mecab/system.dic.zst") } diff --git a/src-core/examples/remove_one_yomitan_dict.rs b/src-core/examples/remove_one_yomitan_dict.rs index 3298a8e..5180113 100644 --- a/src-core/examples/remove_one_yomitan_dict.rs +++ b/src-core/examples/remove_one_yomitan_dict.rs @@ -11,7 +11,9 @@ fn main() -> Result<(), anyhow::Error> { { let _timer = Timer::new("remove one".to_string()); - let mut writer = service.get_yomitan_writer()?; + let mut writer = service.get_yomitan_writer(|p| { + println!("{:?}", p); + })?; AppService::remove_yomitan_dictionary(&mut writer, "JMnedict.zip", "ja")?; } diff --git a/src-core/examples/yomitan_update_schema.rs b/src-core/examples/yomitan_update_schema.rs index 58caad3..72e15b3 100644 --- a/src-core/examples/yomitan_update_schema.rs +++ b/src-core/examples/yomitan_update_schema.rs @@ -11,7 +11,9 @@ fn main() -> Result<(), anyhow::Error> { { let _timer = Timer::new("update schema".to_string()); - service.get_yomitan_writer()?; + service.get_yomitan_writer(|p| { + println!("{:?}", p); + })?; } Ok(()) diff --git a/src-core/src/db.rs b/src-core/src/db.rs index 5f4f7d2..7c57b52 100644 --- a/src-core/src/db.rs +++ b/src-core/src/db.rs @@ -14,19 +14,24 @@ mod yomitan; pub use yomitan::YomitanRow; mod yomitan_writer; -pub use yomitan_writer::{ - YomitanWriter, YomitanZipImportProgress, YomitanZipImportResult, ZipSource, -}; +pub use yomitan_writer::{YomitanProgress, YomitanWriter, YomitanZipImportResult, ZipSource}; #[derive(Clone)] pub(crate) struct Database { pub(crate) conn: Arc>, pub(crate) dir: PathBuf, yomitan_attached: Arc, + yomitan_glossary_attached: Arc, +} + +enum DbChild { + Yomitan, + YomitanGlossary, } const USER_DBFILE: &str = "user.db"; pub(crate) const YOMITAN_DBFILE: &str = "yomitan.db"; +pub(crate) const YOMITAN_GLOSSARY_DBFILE: &str = "yomitan-glossary.db"; impl Database { pub fn new(db_dir: impl AsRef) -> Result { @@ -46,31 +51,55 @@ impl Database { conn: Arc::new(Mutex::new(conn)), dir, yomitan_attached: Arc::new(AtomicBool::new(false)), + yomitan_glossary_attached: Arc::new(AtomicBool::new(false)), }) } pub fn yomitan(&self) -> Result { - self.ensure_yomitan_attached()?; + self.ensure_db_attached(DbChild::Yomitan)?; + self.ensure_db_attached(DbChild::YomitanGlossary)?; Ok(yomitan::YomitanDatabase::new(self.clone())) } - fn ensure_yomitan_attached(&self) -> Result<()> { - if self.yomitan_attached.load(Ordering::Acquire) { - return Ok(()); + fn ensure_db_attached(&self, db_child: DbChild) -> Result<()> { + match db_child { + DbChild::Yomitan => { + if self.yomitan_attached.load(Ordering::Acquire) { + return Ok(()); + } + } + DbChild::YomitanGlossary => { + if self.yomitan_glossary_attached.load(Ordering::Acquire) { + return Ok(()); + } + } } let conn = self.conn.lock().unwrap(); - if !self.yomitan_attached.load(Ordering::Relaxed) { + + let is_attached = match db_child { + DbChild::Yomitan => self.yomitan_attached.load(Ordering::Relaxed), + DbChild::YomitanGlossary => self.yomitan_glossary_attached.load(Ordering::Relaxed), + }; + + if !is_attached { let dir = self.dir.clone(); let path = if self.dir.is_absolute() { dir } else { current_dir()?.join(dir) }; - let path = path - .join(YOMITAN_DBFILE) - .to_string_lossy() - .replace(r"\", r"/"); + + let db_file = match db_child { + DbChild::Yomitan => YOMITAN_DBFILE, + DbChild::YomitanGlossary => YOMITAN_GLOSSARY_DBFILE, + }; + let schema_name = match db_child { + DbChild::Yomitan => "yomitan", + DbChild::YomitanGlossary => "glossary", + }; + + let path = path.join(db_file).to_string_lossy().replace(r"\", r"/"); #[cfg(windows)] let path = if !path.starts_with("/") { @@ -81,15 +110,20 @@ impl Database { let uri = format!("file:{}?mode=ro&immutable=1", path); - conn.execute("ATTACH DATABASE ?1 AS yomitan", [uri])?; - - conn.execute_batch( + conn.execute(&format!("ATTACH DATABASE ?1 AS {}", schema_name), [uri])?; + conn.execute_batch(&format!( r" - PRAGMA yomitan.mmap_size = 3000000000; -- 3_000_000_000 + PRAGMA {}.mmap_size = 3000000000; -- 3_000_000_000 ", - )?; + schema_name + ))?; - self.yomitan_attached.store(true, Ordering::Release); + match db_child { + DbChild::Yomitan => self.yomitan_attached.store(true, Ordering::Release), + DbChild::YomitanGlossary => self + .yomitan_glossary_attached + .store(true, Ordering::Release), + }; } Ok(()) diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs index 3498622..fce1ce9 100644 --- a/src-core/src/db/yomitan.rs +++ b/src-core/src/db/yomitan.rs @@ -1,5 +1,6 @@ use rusqlite::{Result, params}; use serde::Serialize; +use zstd::bulk::Decompressor; use crate::{ CJDicError, Timer, @@ -14,6 +15,7 @@ pub struct YomitanRow { rules: String, score: i64, glossary_json: String, + glossary_b: Vec, sequence: Option, term_tags: String, dict_title: String, @@ -28,6 +30,19 @@ impl YomitanDatabase { Self { db } } + fn decompressor(&self) -> Result, CJDicError> { + let conn = self.db.conn.lock().unwrap(); + + let row = conn.query_row( + "SELECT value FROM glossary.meta WHERE key = 'zstd_dict'", + [], + |r| r.get::<_, Vec>(0), + )?; + + let decompressor = zstd::bulk::Decompressor::with_dictionary(&row)?; + Ok(decompressor) + } + pub fn search_yomitan( &self, q_term: &str, @@ -37,6 +52,11 @@ impl YomitanDatabase { ) -> Result, CJDicError> { let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading)); + // must be run before self.db.conn.lock() + let mut decompressor = self.decompressor()?; + // max decompressed: 90KB (compression ratio on largest: ~2.5x) + const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom + let and_or = if q_term == q_reading { "OR" } else { "AND" }; let mut new_t = String::new(); @@ -60,7 +80,6 @@ impl YomitanDatabase { let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" }; let conn = self.db.conn.lock().unwrap(); - conn.execute_batch( " CREATE TEMP TABLE IF NOT EXISTS yomitan_lookup_keys ( @@ -104,7 +123,7 @@ impl YomitanDatabase { // still the most time bottleneck step, several seconds conn.execute(&sql_string, params![q_term_norm, q_reading, limit, offset])?; - // let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading)); + let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading)); let mut stmt = conn.prepare( " SELECT @@ -113,12 +132,12 @@ impl YomitanDatabase { COALESCE(dt.tags, '') AS def_tags, COALESCE(r.rules, '') AS rules, t.score, - g.content AS glossary_json, + COALESCE(g.b, '[]') AS glossary, t.sequence, COALESCE(tt.tags, '') AS term_tags, t.title AS dict_title FROM yomitan_lookup_keys t - JOIN yomitan.glossaries g ON g.id = t.glossary_id + LEFT JOIN glossary.glossaries g ON g.id = t.glossary_id LEFT JOIN yomitan.def_tag_sets dt ON dt.id = t.def_tags_id LEFT JOIN yomitan.rule_sets r ON r.id = t.rules_id LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id @@ -131,7 +150,8 @@ impl YomitanDatabase { def_tags: r.get(2)?, rules: r.get(3)?, score: r.get(4)?, - glossary_json: r.get(5)?, + glossary_json: String::new(), + glossary_b: r.get(5)?, sequence: r.get(6)?, term_tags: r.get(7)?, dict_title: r.get(8)?, @@ -139,8 +159,13 @@ impl YomitanDatabase { })?; let mut out = Vec::new(); - for r in rows { - out.push(r?); + for row in rows { + let mut r = row?; + r.glossary_json = + String::from_utf8(decompressor.decompress(&r.glossary_b, MAX_DECOMPRESSED_SIZE)?)?; + r.glossary_b = vec![]; + + out.push(r); } Ok(out) } diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs index 2e390fa..3209821 100644 --- a/src-core/src/db/yomitan_writer.rs +++ b/src-core/src/db/yomitan_writer.rs @@ -1,16 +1,20 @@ use blake3::hash; -use rusqlite::{Connection, Result, Transaction, params}; +use rusqlite::{Connection, OptionalExtension, Result, Transaction, params}; use serde::Serialize; use serde_json::Value; use std::{ borrow::Cow, collections::HashMap, + env::current_dir, io::{Cursor, Read}, path::{Path, PathBuf}, }; use zip::ZipArchive; -use crate::{CJDicError, Timer}; +use crate::{ + CJDicError, Timer, + db::{YOMITAN_DBFILE, YOMITAN_GLOSSARY_DBFILE}, +}; fn blake3_hex(s: &str) -> String { format!("{}", hash(s.as_bytes())) @@ -39,11 +43,11 @@ impl ZipSource for PathBuf { } #[derive(Serialize, Debug, Clone)] -pub struct YomitanZipImportProgress { +pub struct YomitanProgress { pub message: String, - pub current: u32, - pub total: u32, - pub steps: u64, + pub current: usize, + pub total: usize, + pub steps: usize, } #[derive(Serialize, Debug)] @@ -54,11 +58,15 @@ pub struct YomitanZipImportResult { } pub struct YomitanWriter { + dir: PathBuf, conn: Connection, } impl YomitanWriter { - pub fn new(conn: Connection) -> Result { + pub fn new(db_dir: impl AsRef) -> Result { + let dir = db_dir.as_ref().to_path_buf(); + let conn = Connection::open(dir.join(YOMITAN_DBFILE))?; + conn.execute_batch( " PRAGMA journal_mode = WAL; @@ -69,15 +77,13 @@ impl YomitanWriter { ", )?; - Ok(Self { conn }) + Ok(Self { dir, conn }) } - pub fn cleanup(&self) -> Result<()> { - self.conn.execute_batch("VACUUM")?; - Ok(()) - } - - pub fn create_schema(&mut self) -> Result<()> { + pub fn create_schema( + &mut self, + progress_callback: impl Fn(YomitanProgress), + ) -> Result<(), CJDicError> { self.conn.execute_batch( " CREATE TABLE IF NOT EXISTS schema_meta ( @@ -97,9 +103,9 @@ impl YomitanWriter { lang TEXT NOT NULL, -- custom UNIQUE (title, revision) ); - CREATE TABLE IF NOT EXISTS glossaries ( + CREATE TABLE IF NOT EXISTS glossaries ( -- not used. moved to a new file id INTEGER PRIMARY KEY, - hash TEXT NOT NULL UNIQUE, + hash TEXT NOT NULL UNIQUE, -- maybe used in the future for excerpts content TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); @@ -168,14 +174,89 @@ impl YomitanWriter { )?; self.create_materialized_views()?; + self.attach_glossary()?; + + let schema_version = "2"; + + let current_schema_version = { + let mut stmt = self + .conn + .prepare("SELECT value FROM schema_meta WHERE key = 'schema_version'")?; + + stmt.query_one([], |r| r.get::<_, String>(0)).optional()? + }; + + if let Some(v) = current_schema_version { + if v < "2".to_string() { + let total = self + .conn + .query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))? + as usize; + + let message = &format!("migrate to {}", YOMITAN_GLOSSARY_DBFILE); + progress_callback(YomitanProgress { + message: message.to_string(), + current: 0, + total, + steps: 0, + }); + + let _timer = Timer::new(message.to_string()); + + let tx = self.conn.transaction()?; + { + let mut stmt = tx.prepare("SELECT id, hash, content FROM glossaries")?; + + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })?; + + let mut stmt_write = tx.prepare( + "INSERT INTO glossary.glossaries (id, hash, b) VALUES (?1, ?2, ?3)", + )?; + + for (i, row) in rows.enumerate() { + let (id, hash, glossary_json) = row?; + stmt_write.execute(params![ + id, + hash, + zstd::encode_all(glossary_json.as_bytes(), 3)? + ])?; + + if i % 1000 == 0 { + progress_callback(YomitanProgress { + message: message.to_string(), + current: i, + total, + steps: 0, + }); + } + } + } + tx.commit()?; + + self.hash_glossary()?; + + let _timer = Timer::new(format!("cleanup for {}", YOMITAN_GLOSSARY_DBFILE)); + self.conn.execute_batch( + " + UPDATE glossaries SET content = '[]'; + VACUUM; + ", + )?; + } - let mut stmt = self - .conn - .prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?; - if stmt.exists([])? == false { self.conn - .prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")? - .execute([])?; + .prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")? + .execute(["schema_version", schema_version])?; + } else { + self.conn + .prepare("INSERT INTO schema_meta (key, value) VALUES (?1, ?2)")? + .execute(["schema_version", schema_version])?; self.conn .prepare( "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))", @@ -263,6 +344,86 @@ impl YomitanWriter { Ok(()) } + fn attach_glossary(&mut self) -> Result<(), CJDicError> { + let dir = self.dir.clone(); + let path = if self.dir.is_absolute() { + dir + } else { + current_dir()?.join(dir) + }; + + let db_file = YOMITAN_GLOSSARY_DBFILE; + let schema_name = "glossary"; + + let path = path.join(db_file).to_string_lossy().replace(r"\", r"/"); + + #[cfg(windows)] + let path = if !path.starts_with("/") { + format!("/{}", path) + } else { + path.to_string() + }; + + let uri = format!("file:{}", path); + + self.conn + .execute(&format!("ATTACH DATABASE ?1 AS {}", schema_name), [uri])?; + self.conn.execute_batch(&format!( + r" + PRAGMA {}.mmap_size = 3000000000; -- 3_000_000_000 + ", + schema_name + ))?; + + self.create_schema_glossary()?; + + Ok(()) + } + + fn create_schema_glossary(&self) -> Result<()> { + self.conn.execute_batch(&format!( + " + CREATE TABLE IF NOT EXISTS glossary.meta ( + key TEXT PRIMARY KEY, + value BLOB NOT NULL + ); + + CREATE TABLE IF NOT EXISTS glossary.glossaries ( + id INTEGER PRIMARY KEY, + hash TEXT NOT NULL UNIQUE, + b BLOB NOT NULL + ); + " + ))?; + + Ok(()) + } + + fn hash_glossary(&self) -> Result<(), CJDicError> { + let max_id: i64 = + self.conn + .query_row("SELECT MAX(id) FROM glossary.glossaries", [], |r| r.get(0))?; + + let step = max_id / 1000; + + let samples: Vec> = self + .conn + .prepare("SELECT b FROM glossary.glossaries WHERE id % ?1 = 0 LIMIT 1000")? + .query_map(params![step], |r| r.get::<_, Vec>(0))? + .filter_map(|r| r.ok()) + .collect(); + + let dict_size = 112640; // 110KB default + let dictionary = zstd::dict::from_samples(&samples, dict_size)?; + + self.conn.execute( + "INSERT OR REPLACE INTO glossary.meta (key, value) VALUES (?1, ?2)", + params!["zstd_dict", dictionary], + )?; + + Ok(()) + } + pub fn import_dictionary_zip_file( &mut self, zip_file: &Z, @@ -271,7 +432,7 @@ impl YomitanWriter { ) -> anyhow::Result where Z: ZipSource, - Callback: Fn(YomitanZipImportProgress), + Callback: Fn(YomitanProgress), { let bundle_name = zip_file.file_name(); let _timer = Timer::new(format!("Importing {}", bundle_name)); @@ -298,8 +459,8 @@ impl YomitanWriter { } }; - let mut steps: u64 = 0; - progress_callback(YomitanZipImportProgress { + let mut steps: usize = 0; + progress_callback(YomitanProgress { message: format!("Opening {}", bundle_name), current: 0, total: 0, @@ -351,247 +512,260 @@ impl YomitanWriter { } } - let tx = self.conn.transaction()?; - tx.execute( - "INSERT INTO dictionaries - (title, revision, author, url, description, lang, bundle_name) VALUES - ( ?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - title, - revision, - index_file.get("author").and_then(Value::as_str), - index_file.get("url").and_then(Value::as_str), - index_file.get("description").and_then(Value::as_str), - lang, - bundle_name, - ], - )?; + self.attach_glossary()?; - let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?; + { + let tx = self.conn.transaction()?; + tx.execute( + "INSERT INTO dictionaries + (title, revision, author, url, description, lang, bundle_name) VALUES + ( ?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + title, + revision, + index_file.get("author").and_then(Value::as_str), + index_file.get("url").and_then(Value::as_str), + index_file.get("description").and_then(Value::as_str), + lang, + bundle_name, + ], + )?; - // prepare statements - let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)"; - let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1"; - let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)"; - let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1"; - let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)"; - let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1"; - let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)"; - let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1"; - let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)"; - let insert_meta = - "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)"; - let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)"; + let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?; - let mut glossary_cache: HashMap = HashMap::new(); - let mut def_cache: HashMap = HashMap::new(); - let mut term_tags_cache: HashMap = HashMap::new(); - let mut rules_cache: HashMap = HashMap::new(); + // prepare statements + let insert_glossary = + "INSERT OR IGNORE INTO glossary.glossaries (hash, b) VALUES (?1, ?2)"; + let select_glossary = "SELECT id FROM glossary.glossaries WHERE hash = ?1"; + let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)"; + let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1"; + let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)"; + let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1"; + let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)"; + let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1"; + let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)"; + let insert_meta = "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)"; + let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)"; - let mut n_bank: u32 = 0; + let mut glossary_cache: HashMap = HashMap::new(); + let mut def_cache: HashMap = HashMap::new(); + let mut term_tags_cache: HashMap = HashMap::new(); + let mut rules_cache: HashMap = HashMap::new(); - for filename in archive.file_names() { - let starts_with = "term_bank_"; + let mut n_bank: usize = 0; - if filename.starts_with(starts_with) { - let text = filename.to_string(); - let start_idx = starts_with.len(); + for filename in archive.file_names() { + let starts_with = "term_bank_"; - if let Some(stop_idx) = text[start_idx..].find(".") { - let sub = &text[start_idx..(start_idx + stop_idx)]; - if let Some(i_bank) = sub.parse::().ok() { - if i_bank > n_bank { - n_bank = i_bank; + if filename.starts_with(starts_with) { + let text = filename.to_string(); + let start_idx = starts_with.len(); + + if let Some(stop_idx) = text[start_idx..].find(".") { + let sub = &text[start_idx..(start_idx + stop_idx)]; + if let Some(i_bank) = sub.parse::().ok() { + if i_bank > n_bank { + n_bank = i_bank; + } } } } } - } - // term banks - let mut bank_i = 1; - loop { - let name = format!("term_bank_{}.json", bank_i); - match archive.by_name(&name) { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - let entries: Vec = serde_json::from_str(&s)?; + // term banks + let mut bank_i = 1; + loop { + let name = format!("term_bank_{}.json", bank_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; - steps += entries.len() as u64; - progress_callback(YomitanZipImportProgress { - message: format!("Reading term banks"), - current: bank_i, - total: n_bank, - steps, - }); + steps += entries.len(); + progress_callback(YomitanProgress { + message: format!("Reading term banks"), + current: bank_i, + total: n_bank, + steps, + }); - for e in entries { - let term = e.get(0).and_then(Value::as_str).unwrap_or(""); - let reading = e.get(1).and_then(Value::as_str).unwrap_or(""); - let def_tags = e - .get(2) - .and_then(Value::as_str) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - let rules = e - .get(3) - .and_then(Value::as_str) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - let score = e.get(4).and_then(Value::as_i64).unwrap_or(0); - let glossary_val = e.get(5).cloned().unwrap_or(Value::Null); - let sequence = e.get(6).and_then(Value::as_i64); - let term_tags = e - .get(7) - .and_then(Value::as_str) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); + for e in entries { + let term = e.get(0).and_then(Value::as_str).unwrap_or(""); + let reading = e.get(1).and_then(Value::as_str).unwrap_or(""); + let def_tags = e + .get(2) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let rules = e + .get(3) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let score = e.get(4).and_then(Value::as_i64).unwrap_or(0); + let glossary_val = e.get(5).cloned().unwrap_or(Value::Null); + let sequence = e.get(6).and_then(Value::as_i64); + let term_tags = e + .get(7) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); - let glossary_json = serde_json::to_string(&glossary_val)?; - let hash = blake3_hex(&glossary_json); - let glossary_id = if let Some(&id) = glossary_cache.get(&hash) { - id - } else { - tx.execute(insert_glossary, params![hash, glossary_json])?; - let id: i64 = - tx.query_row(select_glossary, params![hash], |r| r.get(0))?; - glossary_cache.insert(hash.clone(), id); - id - }; + let glossary_json = serde_json::to_string(&glossary_val)?; + let hash = blake3_hex(&glossary_json); + let glossary_id = if let Some(&id) = glossary_cache.get(&hash) { + id + } else { + tx.execute( + insert_glossary, + params![hash, zstd::encode_all(glossary_json.as_bytes(), 3)?], + )?; + let id: i64 = + tx.query_row(select_glossary, params![hash], |r| r.get(0))?; + glossary_cache.insert(hash.clone(), id); + id + }; - let def_id = if let Some(s) = def_tags.as_deref() { - intern(&tx, insert_def, select_def, &mut def_cache, s)? - } else { - 0 - }; - let rules_id = if let Some(s) = rules.as_deref() { - intern(&tx, insert_rules, select_rules, &mut rules_cache, s)? - } else { - 0 - }; - let term_tags_id = if let Some(s) = term_tags.as_deref() { - intern( - &tx, - insert_term_tags, - select_term_tags, - &mut term_tags_cache, - s, - )? - } else { - 0 - }; + let def_id = if let Some(s) = def_tags.as_deref() { + intern(&tx, insert_def, select_def, &mut def_cache, s)? + } else { + 0 + }; + let rules_id = if let Some(s) = rules.as_deref() { + intern(&tx, insert_rules, select_rules, &mut rules_cache, s)? + } else { + 0 + }; + let term_tags_id = if let Some(s) = term_tags.as_deref() { + intern( + &tx, + insert_term_tags, + select_term_tags, + &mut term_tags_cache, + s, + )? + } else { + 0 + }; - tx.execute( - insert_term, - params![ - dict_id, - term, - reading, - if def_id != 0 { - Some(def_id) - } else { - Option::::None - }, - if rules_id != 0 { - Some(rules_id) - } else { - Option::::None - }, - score, - glossary_id, - sequence, - if term_tags_id != 0 { - Some(term_tags_id) - } else { - Option::::None - }, - ], - )?; + tx.execute( + insert_term, + params![ + dict_id, + term, + reading, + if def_id != 0 { + Some(def_id) + } else { + Option::::None + }, + if rules_id != 0 { + Some(rules_id) + } else { + Option::::None + }, + score, + glossary_id, + sequence, + if term_tags_id != 0 { + Some(term_tags_id) + } else { + Option::::None + }, + ], + )?; + } + bank_i += 1; + continue; } - bank_i += 1; - continue; + Err(_) => break, } - Err(_) => break, } - } - steps += 1; - progress_callback(YomitanZipImportProgress { - message: format!("Reading term meta banks"), - current: 0, - total: 0, - steps, - }); + steps += 1; + progress_callback(YomitanProgress { + message: format!("Reading term meta banks"), + current: 0, + total: 0, + steps, + }); - // term_meta banks - let mut meta_i = 1; - loop { - let name = format!("term_meta_bank_{}.json", meta_i); - match archive.by_name(&name) { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - let entries: Vec = serde_json::from_str(&s)?; - for e in entries { - let term = e.get(0).and_then(Value::as_str).unwrap_or(""); - let mode = e.get(1).and_then(Value::as_str).unwrap_or(""); - let data = e.get(2).cloned().unwrap_or(Value::Null); - let reading = data - .get("reading") - .and_then(Value::as_str) - .map(|s| s.to_string()); - tx.execute( - insert_meta, - params![dict_id, term, mode, reading, serde_json::to_string(&data)?], - )?; + // term_meta banks + let mut meta_i = 1; + loop { + let name = format!("term_meta_bank_{}.json", meta_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let term = e.get(0).and_then(Value::as_str).unwrap_or(""); + let mode = e.get(1).and_then(Value::as_str).unwrap_or(""); + let data = e.get(2).cloned().unwrap_or(Value::Null); + let reading = data + .get("reading") + .and_then(Value::as_str) + .map(|s| s.to_string()); + tx.execute( + insert_meta, + params![ + dict_id, + term, + mode, + reading, + serde_json::to_string(&data)? + ], + )?; + } + meta_i += 1; + continue; } - meta_i += 1; - continue; + Err(_) => break, } - Err(_) => break, } - } - steps += 1; - progress_callback(YomitanZipImportProgress { - message: format!("Reading tag banks"), - current: 0, - total: 0, - steps, - }); + steps += 1; + progress_callback(YomitanProgress { + message: format!("Reading tag banks"), + current: 0, + total: 0, + steps, + }); - // tag banks - let mut tag_i = 1; - loop { - let name = format!("tag_bank_{}.json", tag_i); - match archive.by_name(&name) { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - let entries: Vec = serde_json::from_str(&s)?; - for e in entries { - let name = e.get(0).and_then(Value::as_str).unwrap_or(""); - let category = e.get(1).and_then(Value::as_str); - let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0); - let notes = e.get(3).and_then(Value::as_str); - let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0); - tx.execute( - insert_tag, - params![dict_id, name, category, sort_order, notes, tag_score], - )?; + // tag banks + let mut tag_i = 1; + loop { + let name = format!("tag_bank_{}.json", tag_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let name = e.get(0).and_then(Value::as_str).unwrap_or(""); + let category = e.get(1).and_then(Value::as_str); + let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0); + let notes = e.get(3).and_then(Value::as_str); + let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0); + tx.execute( + insert_tag, + params![dict_id, name, category, sort_order, notes, tag_score], + )?; + } + tag_i += 1; + continue; } - tag_i += 1; - continue; + Err(_) => break, } - Err(_) => break, } - } - tx.commit()?; + tx.commit()?; + } self.conn.execute_batch( " @@ -600,6 +774,8 @@ impl YomitanWriter { ", )?; // < 1 ms + self.hash_glossary()?; + // self.conn.execute_batch("VACUUM;")?; // Not needed, twice time taken than import Ok(YomitanZipImportResult { diff --git a/src-core/src/error.rs b/src-core/src/error.rs index 921e6f1..3d949a2 100644 --- a/src-core/src/error.rs +++ b/src-core/src/error.rs @@ -1,3 +1,5 @@ +use std::string::FromUtf8Error; + use serde::Serialize; use thiserror::Error; use vibrato::errors::VibratoError; @@ -27,6 +29,9 @@ pub enum CJDicError { #[error("Vibrato error: {0}")] VibratoError(String), + #[error("FromUtf8Error: {0}")] + FromUtf8Error(String), + #[error("Not found")] NotFound, @@ -36,42 +41,49 @@ pub enum CJDicError { impl From for CJDicError { fn from(e: rusqlite::Error) -> Self { - log::error!("{e:#}"); + eprintln!("{e:#}"); CJDicError::Database(e.to_string()) } } impl From for CJDicError { fn from(e: serde_json::Error) -> Self { - log::error!("{e:#}"); + eprintln!("{e:#}"); CJDicError::Serialization(e.to_string()) } } impl From for CJDicError { fn from(e: anyhow::Error) -> Self { - log::error!("{e:#}"); + eprintln!("{e:#}"); CJDicError::AnyhowError(e.to_string()) } } impl From for CJDicError { fn from(e: std::io::Error) -> Self { - log::error!("{e:#}"); + eprintln!("{e:#}"); CJDicError::IOError(e.to_string()) } } impl From for CJDicError { fn from(e: ZipError) -> Self { - log::error!("{e:#}"); + eprintln!("{e:#}"); CJDicError::ZipError(e.to_string()) } } impl From for CJDicError { fn from(e: VibratoError) -> Self { - log::error!("{e:#}"); + eprintln!("{e:#}"); CJDicError::VibratoError(e.to_string()) } } + +impl From for CJDicError { + fn from(e: FromUtf8Error) -> Self { + eprintln!("{e:#}"); + CJDicError::FromUtf8Error(e.to_string()) + } +} diff --git a/src-core/src/lib.rs b/src-core/src/lib.rs index 08d088b..5714329 100644 --- a/src-core/src/lib.rs +++ b/src-core/src/lib.rs @@ -3,7 +3,7 @@ mod error; mod service; mod util; -pub use db::{YomitanRow, YomitanZipImportProgress, YomitanZipImportResult, ZipSource}; +pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource}; pub use error::CJDicError; pub use service::{AppService, SqlParam, TokenizeSegment}; pub use util::Timer; diff --git a/src-core/src/service.rs b/src-core/src/service.rs index f46f6df..7b6ead4 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -11,7 +11,7 @@ use vibrato::{Dictionary, Tokenizer}; use crate::{ Timer, ZipSource, db::{ - Database, YOMITAN_DBFILE, YomitanRow, YomitanWriter, YomitanZipImportProgress, + Database, YOMITAN_DBFILE, YomitanProgress, YomitanRow, YomitanWriter, YomitanZipImportResult, }, error::CJDicError, @@ -149,17 +149,12 @@ impl AppService { .search_yomitan(q_term, q_reading, limit, offset)?) } - pub fn get_yomitan_writer(&self) -> Result { - let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?; - let mut writer = YomitanWriter::new(conn)?; - writer.create_schema()?; - Ok(writer) - } - - pub fn cleanup_yomitan_writer(&self) -> Result { - let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?; - let writer = YomitanWriter::new(conn)?; - writer.cleanup()?; + pub fn get_yomitan_writer( + &self, + progress_callback: impl Fn(YomitanProgress), + ) -> Result { + let mut writer = YomitanWriter::new(self.db.dir.clone())?; + writer.create_schema(&progress_callback)?; Ok(writer) } @@ -173,7 +168,7 @@ impl AppService { where Z: ZipSource, LoadCallback: Fn(LoadYomitanZipDirResult), - ImportCallback: Fn(YomitanZipImportProgress), + ImportCallback: Fn(YomitanProgress), { let mut dir_zip_map: HashMap = HashMap::new(); let mut zip_dir_ord: HashMap = HashMap::new(); @@ -184,17 +179,13 @@ impl AppService { zip_dir_ord.insert(filename.to_string(), i); } - { - let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?; - let mut writer = YomitanWriter::new(conn)?; - writer.create_schema()?; - } - let mut new_dicts: Vec = vec![]; let mut to_be_removed_dicts: Vec = vec![]; - { - let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?; + let db_path = self.db.dir.join(YOMITAN_DBFILE); + + if db_path.exists() { + let conn = Connection::open(&db_path)?; let mut stmt = conn.prepare("SELECT DISTINCT bundle_name FROM dictionaries WHERE lang = ?1")?; let mut rows = stmt.query([lang])?; @@ -228,7 +219,12 @@ impl AppService { }); { - let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?; + let mut writer = YomitanWriter::new(self.db.dir.clone())?; + writer.create_schema(&import_callback)?; + } + + { + let conn = Connection::open(&db_path)?; for z in to_be_removed_dicts.clone().iter() { conn.execute( "DELETE FROM dictionaries WHERE bundle_name = ?1 AND lang = ?2", @@ -238,8 +234,7 @@ impl AppService { } { - let conn = Connection::open(self.db.dir.join(YOMITAN_DBFILE))?; - let mut writer = YomitanWriter::new(conn)?; + let mut writer = YomitanWriter::new(self.db.dir.clone())?; for z in new_dicts.clone().iter() { if let Some(zip_file) = dir_zip_map.get(z) { @@ -262,7 +257,7 @@ impl AppService { ) -> Result where Z: ZipSource, - Callback: Fn(YomitanZipImportProgress), + Callback: Fn(YomitanProgress), { Ok(writer.import_dictionary_zip_file(zip_file, lang, progress_callback)?) } diff --git a/src-tauri/src/command.rs b/src-tauri/src/command.rs index 20ee618..445e07e 100644 --- a/src-tauri/src/command.rs +++ b/src-tauri/src/command.rs @@ -97,7 +97,9 @@ pub async fn import_yomitan_dict( data_dir.push(lang); data_dir.push(bundle_name); - let mut writer = state.get_yomitan_writer()?; + let mut writer = state.get_yomitan_writer(|r| { + app.emit("yomitan-import-progress", r).unwrap(); + })?; AppService::import_yomitan_zip_file(&mut writer, &data_dir, lang, |r| { app.emit("yomitan-import-progress", r).unwrap(); }) @@ -105,11 +107,14 @@ pub async fn import_yomitan_dict( #[tauri::command] pub async fn remove_yomitan_dict( + app: AppHandle, bundle_name: &str, lang: &str, state: tauri::State<'_, AppService>, ) -> Result<(), CJDicError> { - let mut writer = state.get_yomitan_writer()?; + let mut writer = state.get_yomitan_writer(|r| { + app.emit("yomitan-import-progress", r).unwrap(); + })?; AppService::remove_yomitan_dictionary(&mut writer, bundle_name, lang) }