From 8bd087b869596abd77be0e4959afb6eb63a4a606 Mon Sep 17 00:00:00 2001 From: polv Date: Tue, 24 Feb 2026 13:33:36 +0700 Subject: [PATCH] dict importer progress --- src-core/examples/add_yomitan_dict_folder.rs | 4 +- src-core/src/db.rs | 2 +- src-core/src/db/yomitan_writer.rs | 71 +++++++++++++++++++- src-core/src/service.rs | 12 ++-- 4 files changed, 80 insertions(+), 9 deletions(-) diff --git a/src-core/examples/add_yomitan_dict_folder.rs b/src-core/examples/add_yomitan_dict_folder.rs index a28c8e8..18c36c4 100644 --- a/src-core/examples/add_yomitan_dict_folder.rs +++ b/src-core/examples/add_yomitan_dict_folder.rs @@ -29,7 +29,9 @@ fn main() -> Result<(), anyhow::Error> { println!("zip_file: {:?}", p); println!( "{:?}", - AppService::import_yomitan_zip_file(&mut writer, p, "ja")?, + AppService::import_yomitan_zip_file(&mut writer, p, "ja", |progress| { + println!("{:?}", progress); + })?, ); println!("[{:.2?}]", start.elapsed()); } else { diff --git a/src-core/src/db.rs b/src-core/src/db.rs index a5cf3fd..f69feb1 100644 --- a/src-core/src/db.rs +++ b/src-core/src/db.rs @@ -16,7 +16,7 @@ mod yomitan; pub use yomitan::YomitanRow; mod yomitan_writer; -pub use yomitan_writer::{YomitanWriter, YomitanZipImportResult}; +pub use yomitan_writer::{YomitanWriter, YomitanZipImportProgress, YomitanZipImportResult}; #[derive(Clone)] pub(crate) struct Database { diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs index 3db47ee..e383f64 100644 --- a/src-core/src/db/yomitan_writer.rs +++ b/src-core/src/db/yomitan_writer.rs @@ -19,6 +19,14 @@ pub(super) fn normalize_term(s: &str) -> String { .collect() } +#[derive(Serialize, Debug)] +pub struct YomitanZipImportProgress { + pub message: String, + pub current: u32, + pub total: u32, + pub steps: u64, +} + #[derive(Serialize, Debug)] pub struct YomitanZipImportResult { pub exists: bool, @@ -171,7 +179,7 @@ impl YomitanWriter { WHEN NEW.score > 0 -- This automatically excludes NULL (since NULL > 0 is false) BEGIN - INSERT INTO term_rank (term, reading, max_score) + INSERT INTO view_terms_term_rank (term, reading, max_score) VALUES (NEW.term, NEW.reading, NEW.score) ON CONFLICT(term, reading) DO UPDATE SET max_score = @@ -229,11 +237,15 @@ impl YomitanWriter { Ok(()) } - pub fn import_dictionary_zip_file( + pub fn import_dictionary_zip_file( &mut self, zip_file: PathBuf, lang: &str, - ) -> anyhow::Result { + progress_callback: Callback, + ) -> anyhow::Result + where + Callback: Fn(YomitanZipImportProgress), + { self.create_schema()?; // < 1 ms let start_time = Instant::now(); @@ -260,6 +272,14 @@ impl YomitanWriter { } }; + let mut steps: u64 = 0; + progress_callback(YomitanZipImportProgress { + message: format!("Extracted {}", zip_file.display()), + current: 0, + total: 0, + steps, + }); + fn remove_timestamp(s: &str) -> String { if let Some(pos) = s.rfind('[') { // Trim whitespace only before the '[' and return the substring @@ -339,6 +359,26 @@ impl YomitanWriter { let mut term_tags_cache: HashMap = HashMap::new(); let mut rules_cache: HashMap = HashMap::new(); + let mut n_bank: u32 = 0; + + for filename in archive.file_names() { + let starts_with = "term_bank_"; + + if filename.starts_with(starts_with) { + let text = filename.to_string(); + let start_idx = starts_with.len(); + + if let Some(stop_idx) = text[start_idx..].find(".") { + let sub = &text[start_idx..(start_idx + stop_idx)]; + if let Some(i_bank) = sub.parse::().ok() { + if i_bank > n_bank { + n_bank = i_bank; + } + } + } + } + } + // term banks let mut bank_i = 1; loop { @@ -348,6 +388,15 @@ impl YomitanWriter { let mut s = String::new(); f.read_to_string(&mut s)?; let entries: Vec = serde_json::from_str(&s)?; + + steps += entries.len() as u64; + progress_callback(YomitanZipImportProgress { + message: format!("Reading term bank {} of {}", bank_i, n_bank), + current: bank_i, + total: n_bank, + steps, + }); + for e in entries { let term = e.get(0).and_then(Value::as_str).unwrap_or(""); let reading = e.get(1).and_then(Value::as_str).unwrap_or(""); @@ -441,6 +490,14 @@ impl YomitanWriter { } } + steps += 1; + progress_callback(YomitanZipImportProgress { + message: format!("Reading term meta bank"), + current: 0, + total: 0, + steps, + }); + // term_meta banks let mut meta_i = 1; loop { @@ -470,6 +527,14 @@ impl YomitanWriter { } } + steps += 1; + progress_callback(YomitanZipImportProgress { + message: format!("Reading tag bank"), + current: 0, + total: 0, + steps, + }); + // tag banks let mut tag_i = 1; loop { diff --git a/src-core/src/service.rs b/src-core/src/service.rs index feb0191..1d4b514 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use rusqlite::Connection; use crate::{ - db::{Database, YomitanRow, YomitanWriter, YomitanZipImportResult}, + db::{Database, YomitanRow, YomitanWriter, YomitanZipImportProgress, YomitanZipImportResult}, error::CJDicError, models::Entry, }; @@ -48,12 +48,16 @@ impl AppService { Ok(writer) } - pub fn import_yomitan_zip_file( + pub fn import_yomitan_zip_file( writer: &mut YomitanWriter, zip_file: PathBuf, lang: &str, - ) -> Result { - Ok(writer.import_dictionary_zip_file(zip_file, lang)?) + progress_callback: Callback, + ) -> Result + where + Callback: Fn(YomitanZipImportProgress), + { + Ok(writer.import_dictionary_zip_file(zip_file, lang, progress_callback)?) } pub fn remove_yomitan_dictionary(