From fa5bfb56b080ff4ef6a93699b6c2f4470bebe053 Mon Sep 17 00:00:00 2001 From: polv Date: Mon, 23 Feb 2026 14:56:42 +0700 Subject: [PATCH] main as user.db, then attach readonly db's --- src-core/examples/add-to-yomitan.rs | 12 +- src-core/examples/common/mod.rs | 4 +- src-core/examples/search-yomitan.rs | 8 +- src-core/examples/yomitan-mod-schema.rs | 8 +- src-core/src/api.rs | 5 - src-core/src/db.rs | 74 ++++- src-core/src/db/yomitan.rs | 403 +---------------------- src-core/src/db/yomitan_writer.rs | 404 ++++++++++++++++++++++++ src-core/src/lib.rs | 6 +- src-core/src/service.rs | 31 +- src-tauri/src/lib.rs | 6 +- 11 files changed, 526 insertions(+), 435 deletions(-) create mode 100644 src-core/src/db/yomitan_writer.rs diff --git a/src-core/examples/add-to-yomitan.rs b/src-core/examples/add-to-yomitan.rs index 911db20..aed65f7 100644 --- a/src-core/examples/add-to-yomitan.rs +++ b/src-core/examples/add-to-yomitan.rs @@ -5,13 +5,14 @@ use std::{ }; use anyhow::{Context, Ok}; -use cjdic2_core::db::Database; +use cjdic2_core::AppService; mod common; -use common::get_db_path; +use common::get_db_dir; fn main() -> Result<(), anyhow::Error> { - let db = Database::new(get_db_path())?; + let service = AppService::new(get_db_dir())?; + let mut writer = service.get_yomitan_writer()?; let zip_dir = Path::new("tmp/yomitan"); println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root @@ -26,7 +27,10 @@ fn main() -> Result<(), anyhow::Error> { let start = Instant::now(); println!("zip_file: {:?}", p); - println!("{:?}", db.yomitan().import_bundled_zip_file(p, "ja")?,); + println!( + "{:?}", + AppService::import_yomitan_zip_file(&mut writer, p, "ja")?, + ); println!("[{:.2?}]", start.elapsed(),); } else { println!("not zip_file: {:?}", p); diff --git a/src-core/examples/common/mod.rs b/src-core/examples/common/mod.rs index 509b2b2..3058cea 100644 --- a/src-core/examples/common/mod.rs +++ b/src-core/examples/common/mod.rs @@ -1,5 +1,5 @@ use std::path::PathBuf; -pub fn get_db_path() -> PathBuf { - PathBuf::from("test.db") +pub fn get_db_dir() -> PathBuf { + PathBuf::from("tmp/save-db") } diff --git a/src-core/examples/search-yomitan.rs b/src-core/examples/search-yomitan.rs index 639cac2..5fa9e1a 100644 --- a/src-core/examples/search-yomitan.rs +++ b/src-core/examples/search-yomitan.rs @@ -1,16 +1,16 @@ use std::time::Instant; use anyhow::Ok; -use cjdic2_core::db::Database; +use cjdic2_core::AppService; mod common; -use common::get_db_path; +use common::get_db_dir; fn main() -> Result<(), anyhow::Error> { - let db = Database::new(get_db_path())?; + let service = AppService::new(get_db_dir())?; let start = Instant::now(); - let rs = db.yomitan().search_yomitan("擦る", "する", 10, 0)?; + let rs = service.search_yomitan("擦る", "する", 10, 0)?; println!("{:#?}", rs); println!("n={}", rs.len()); println!("[{:.2?}]", start.elapsed()); diff --git a/src-core/examples/yomitan-mod-schema.rs b/src-core/examples/yomitan-mod-schema.rs index 8fb4e7f..0a04ff7 100644 --- a/src-core/examples/yomitan-mod-schema.rs +++ b/src-core/examples/yomitan-mod-schema.rs @@ -1,16 +1,16 @@ use std::time::Instant; use anyhow::Ok; -use cjdic2_core::db::Database; +use cjdic2_core::AppService; mod common; -use common::get_db_path; +use common::get_db_dir; fn main() -> Result<(), anyhow::Error> { - let db = Database::new(get_db_path())?; + let service = AppService::new(get_db_dir())?; let start = Instant::now(); - db.yomitan().create_schema()?; + service.get_yomitan_writer()?; println!("[{:.2?}]", start.elapsed()); Ok(()) diff --git a/src-core/src/api.rs b/src-core/src/api.rs index 7aeaafb..8b13789 100644 --- a/src-core/src/api.rs +++ b/src-core/src/api.rs @@ -1,6 +1 @@ -use crate::{error::CJDicError, service::AppService}; -pub fn list_entries_json(service: &AppService) -> Result { - let entries = service.list_entries()?; - Ok(serde_json::to_string(&entries)?) -} diff --git a/src-core/src/db.rs b/src-core/src/db.rs index 8913471..e04d929 100644 --- a/src-core/src/db.rs +++ b/src-core/src/db.rs @@ -1,30 +1,86 @@ use std::{ - path::Path, - sync::{Arc, Mutex}, + env::current_dir, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, }; -use rusqlite::{Connection, Result}; +use anyhow::Result; +use rusqlite::Connection; use crate::models::Entry; mod yomitan; -pub use yomitan::{YomitanRow, YomitanZipImportResult}; +pub use yomitan::YomitanRow; + +mod yomitan_writer; +pub use yomitan_writer::{YomitanWriter, YomitanZipImportResult}; #[derive(Clone)] -pub struct Database { +pub(crate) struct Database { pub(crate) conn: Arc>, + pub(crate) dir: PathBuf, + yomitan_attached: Arc, } impl Database { - pub fn new>(path: P) -> Result { - let conn = Connection::open(path)?; + pub fn new(db_dir: impl AsRef) -> Result { + let dir = db_dir.as_ref().to_path_buf(); + let conn = Connection::open(dir.join("user.db"))?; + + conn.execute_batch( + r" + PRAGMA jornal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA foreign_keys = ON; + ", + )?; + Ok(Self { conn: Arc::new(Mutex::new(conn)), + dir, + yomitan_attached: Arc::new(AtomicBool::new(false)), }) } - pub fn yomitan(&self) -> yomitan::YomitanDatabase { - yomitan::YomitanDatabase::new(self.clone()) + pub fn yomitan(&self) -> Result { + self.ensure_yomitan_attached()?; + Ok(yomitan::YomitanDatabase::new(self.clone())) + } + + fn ensure_yomitan_attached(&self) -> Result<()> { + if self.yomitan_attached.load(Ordering::Acquire) { + return Ok(()); + } + + let conn = self.conn.lock().unwrap(); + if !self.yomitan_attached.load(Ordering::Relaxed) { + let dir = self.dir.clone(); + let path = if self.dir.is_absolute() { + dir + } else { + current_dir()?.join(dir) + }; + let mut path = path + .join("yomitan.db") + .to_string_lossy() + .replace(r"\", r"/"); + + #[cfg(windows)] + { + if !path.starts_with("/") { + path = format!("/{}", path); + } + } + let uri = format!("file:{}?mode=ro&immutable=1", path); + + conn.execute("ATTACH DATABASE ?1 AS yomitan", [uri])?; + self.yomitan_attached.store(true, Ordering::Release); + } + + Ok(()) } pub fn init(&self) -> Result<()> { diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs index f4a8db8..d88854c 100644 --- a/src-core/src/db/yomitan.rs +++ b/src-core/src/db/yomitan.rs @@ -1,25 +1,8 @@ -use std::{collections::HashMap, fs::File, io::Read, path::PathBuf}; - -use anyhow::Context; -use blake3::hash; -use rusqlite::{Result, Transaction, params}; +use rusqlite::{Result, params}; use serde::Serialize; -use serde_json::Value; -use zip::ZipArchive; use crate::{CJDicError, db::Database}; -fn blake3_hex(s: &str) -> String { - format!("{}", hash(s.as_bytes())) -} - -#[derive(Serialize, Debug)] -pub struct YomitanZipImportResult { - exists: bool, - load: bool, - error: Option, -} - #[derive(Serialize, Debug)] pub struct YomitanRow { term: String, @@ -42,362 +25,6 @@ impl YomitanDatabase { Self { db } } - pub fn create_schema(&self) -> Result<()> { - let conn = self.db.conn.lock().unwrap(); - - conn.execute_batch( - " - PRAGMA journal_mode = WAL; - PRAGMA foreign_keys = ON; - PRAGMA synchronous = NORMAL; - PRAGMA cache_size = -65536; - PRAGMA temp_store = MEMORY; - - CREATE TABLE IF NOT EXISTS schema_meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS dictionaries ( - id INTEGER PRIMARY KEY, - title TEXT NOT NULL, - revision TEXT NOT NULL, - author TEXT, - url TEXT, - description TEXT, - sort_order INTEGER NOT NULL DEFAULT 0, - installed_at TEXT NOT NULL DEFAULT (datetime('now')), - lang TEXT NOT NULL, - UNIQUE (title, revision) - ); - CREATE TABLE IF NOT EXISTS glossaries ( - id INTEGER PRIMARY KEY, - hash TEXT NOT NULL UNIQUE, - content TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); - CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); - CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE); - CREATE TABLE IF NOT EXISTS terms ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - term TEXT NOT NULL, - reading TEXT NOT NULL, - def_tags_id INTEGER, - rules_id INTEGER, - score INTEGER NOT NULL DEFAULT 0, - glossary_id INTEGER NOT NULL REFERENCES glossaries(id), - sequence INTEGER, - term_tags_id INTEGER - ); - CREATE TABLE IF NOT EXISTS term_meta ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - term TEXT NOT NULL, - mode TEXT NOT NULL, - reading TEXT, - data TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS tags ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - name TEXT NOT NULL, - category TEXT, - sort_order INTEGER NOT NULL DEFAULT 0, - notes TEXT, - score INTEGER NOT NULL DEFAULT 0, - UNIQUE (dict_id, name) - ); - CREATE TABLE IF NOT EXISTS kanji ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - kanji TEXT NOT NULL, - onyomi TEXT, - kunyomi TEXT, - tags TEXT, - meanings TEXT NOT NULL DEFAULT '[]', - stats TEXT NOT NULL DEFAULT '{}' - ); - ", - )?; - - conn.execute_batch( - " - CREATE INDEX IF NOT EXISTS dictionaries_lang ON dictionaries (lang); - - CREATE INDEX IF NOT EXISTS terms_term ON terms (term); - CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading); - ", - )?; - - let mut stmt = conn.prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?; - if stmt.exists([])? == false { - conn.prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")? - .execute([])?; - conn.prepare( - "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))", - )? - .execute([])?; - } - - Ok(()) - } - - pub fn import_bundled_zip_file( - &mut self, - zip_path: PathBuf, - lang: &str, - ) -> anyhow::Result { - self.create_schema()?; - - let f = - File::open(&zip_path).with_context(|| format!("opening zip {}", zip_path.display()))?; - let mut archive = ZipArchive::new(f).with_context(|| "reading zip archive")?; - - let index_file = match archive.by_name("index.json") { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - serde_json::from_str::(&s)? - } - Err(e) => { - // TODO: proper error handling - eprintln!("{}", e); - return Ok(YomitanZipImportResult { - exists: false, - load: false, - error: Some(format!("{}", e)), - }); - } - }; - - let title = index_file - .get("title") - .and_then(Value::as_str) - .unwrap_or("") - .to_string(); - let revision = index_file - .get("revision") - .and_then(Value::as_str) - .unwrap_or("") - .to_string(); - - let mut conn = self.db.conn.lock().unwrap(); - - // Skip if already installed - let exists: bool = conn - .prepare("SELECT 1 FROM dictionaries WHERE title = ?1 AND revision = ?2")? - .exists([title, revision])?; - if exists { - return Ok(YomitanZipImportResult { - exists: true, - load: false, - error: None, - }); - } - - let tx = conn.transaction()?; - tx.execute( - "INSERT INTO dictionaries (title, revision, author, url, description, lang) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![ - index_file.get("title").and_then(Value::as_str), - index_file.get("revision").and_then(Value::as_str), - index_file.get("author").and_then(Value::as_str), - index_file.get("url").and_then(Value::as_str), - index_file.get("description").and_then(Value::as_str), - lang, - ], - )?; - - let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?; - - // prepare statements - let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)"; - let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1"; - let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)"; - let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1"; - let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)"; - let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1"; - let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)"; - let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1"; - let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)"; - let insert_meta = - "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)"; - let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)"; - - let mut glossary_cache: HashMap = HashMap::new(); - let mut def_cache: HashMap = HashMap::new(); - let mut term_tags_cache: HashMap = HashMap::new(); - let mut rules_cache: HashMap = HashMap::new(); - - // term banks - let mut bank_i = 1; - loop { - let name = format!("term_bank_{}.json", bank_i); - match archive.by_name(&name) { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - let entries: Vec = serde_json::from_str(&s)?; - for e in entries { - let term = e.get(0).and_then(Value::as_str).unwrap_or(""); - let reading = e.get(1).and_then(Value::as_str).unwrap_or(""); - let def_tags = e - .get(2) - .and_then(Value::as_str) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - let rules = e - .get(3) - .and_then(Value::as_str) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - let score = e.get(4).and_then(Value::as_i64).unwrap_or(0); - let glossary_val = e.get(5).cloned().unwrap_or(Value::Null); - let sequence = e.get(6).and_then(Value::as_i64); - let term_tags = e - .get(7) - .and_then(Value::as_str) - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - - let glossary_json = serde_json::to_string(&glossary_val)?; - let hash = blake3_hex(&glossary_json); - let glossary_id = if let Some(&id) = glossary_cache.get(&hash) { - id - } else { - tx.execute(insert_glossary, params![hash, glossary_json])?; - let id: i64 = - tx.query_row(select_glossary, params![hash], |r| r.get(0))?; - glossary_cache.insert(hash.clone(), id); - id - }; - - let def_id = if let Some(s) = def_tags.as_deref() { - intern(&tx, insert_def, select_def, &mut def_cache, s)? - } else { - 0 - }; - let rules_id = if let Some(s) = rules.as_deref() { - intern(&tx, insert_rules, select_rules, &mut rules_cache, s)? - } else { - 0 - }; - let term_tags_id = if let Some(s) = term_tags.as_deref() { - intern( - &tx, - insert_term_tags, - select_term_tags, - &mut term_tags_cache, - s, - )? - } else { - 0 - }; - - tx.execute( - insert_term, - params![ - dict_id, - term, - reading, - if def_id != 0 { - Some(def_id) - } else { - Option::::None - }, - if rules_id != 0 { - Some(rules_id) - } else { - Option::::None - }, - score, - glossary_id, - sequence, - if term_tags_id != 0 { - Some(term_tags_id) - } else { - Option::::None - }, - ], - )?; - } - bank_i += 1; - continue; - } - Err(_) => break, - } - } - - // term_meta banks - let mut meta_i = 1; - loop { - let name = format!("term_meta_bank_{}.json", meta_i); - match archive.by_name(&name) { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - let entries: Vec = serde_json::from_str(&s)?; - for e in entries { - let term = e.get(0).and_then(Value::as_str).unwrap_or(""); - let mode = e.get(1).and_then(Value::as_str).unwrap_or(""); - let data = e.get(2).cloned().unwrap_or(Value::Null); - let reading = data - .get("reading") - .and_then(Value::as_str) - .map(|s| s.to_string()); - tx.execute( - insert_meta, - params![dict_id, term, mode, reading, serde_json::to_string(&data)?], - )?; - } - meta_i += 1; - continue; - } - Err(_) => break, - } - } - - // tag banks - let mut tag_i = 1; - loop { - let name = format!("tag_bank_{}.json", tag_i); - match archive.by_name(&name) { - Ok(mut f) => { - let mut s = String::new(); - f.read_to_string(&mut s)?; - let entries: Vec = serde_json::from_str(&s)?; - for e in entries { - let name = e.get(0).and_then(Value::as_str).unwrap_or(""); - let category = e.get(1).and_then(Value::as_str); - let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0); - let notes = e.get(3).and_then(Value::as_str); - let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0); - tx.execute( - insert_tag, - params![dict_id, name, category, sort_order, notes, tag_score], - )?; - } - tag_i += 1; - continue; - } - Err(_) => break, - } - } - - tx.commit()?; - - conn.execute_batch("VACUUM;")?; - - Ok(YomitanZipImportResult { - exists: true, - load: true, - error: None, - }) - } - pub fn search_yomitan( &self, q_term: &str, @@ -420,12 +47,12 @@ impl YomitanDatabase { t.sequence, COALESCE(tt.tags, '') AS term_tags, d.title AS dict_title - FROM terms t - JOIN glossaries g ON g.id = t.glossary_id - JOIN dictionaries d ON d.id = t.dict_id - LEFT JOIN def_tag_sets dt ON dt.id = t.def_tags_id - LEFT JOIN rule_sets r ON r.id = t.rules_id - LEFT JOIN term_tag_sets tt ON tt.id = t.term_tags_id + FROM yomitan.terms t + JOIN yomitan.glossaries g ON g.id = t.glossary_id + JOIN yomitan.dictionaries d ON d.id = t.dict_id + LEFT JOIN yomitan.def_tag_sets dt ON dt.id = t.def_tags_id + LEFT JOIN yomitan.rule_sets r ON r.id = t.rules_id + LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id WHERE t.term GLOB ?1 {} t.reading GLOB ?2 ORDER BY t.score DESC LIMIT ?3 OFFSET ?4 @@ -457,19 +84,3 @@ impl YomitanDatabase { Ok(out) } } - -fn intern( - tx: &Transaction, - insert_sql: &str, - select_sql: &str, - cache: &mut HashMap, - value: &str, -) -> rusqlite::Result { - if let Some(&v) = cache.get(value) { - return Ok(v); - } - tx.execute(insert_sql, [value])?; - let id: i64 = tx.query_row(select_sql, [value], |r| r.get(0))?; - cache.insert(value.to_string(), id); - Ok(id) -} diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs new file mode 100644 index 0000000..4f58687 --- /dev/null +++ b/src-core/src/db/yomitan_writer.rs @@ -0,0 +1,404 @@ +use std::{collections::HashMap, fs::File, io::Read, path::PathBuf}; + +use anyhow::Context; +use blake3::hash; +use rusqlite::{Connection, Result, Transaction, params}; +use serde::Serialize; +use serde_json::Value; +use zip::ZipArchive; + +use crate::CJDicError; + +fn blake3_hex(s: &str) -> String { + format!("{}", hash(s.as_bytes())) +} + +#[derive(Serialize, Debug)] +pub struct YomitanZipImportResult { + pub exists: bool, + pub load: bool, + pub error: Option, +} + +pub struct YomitanWriter { + conn: Connection, +} + +impl YomitanWriter { + pub fn new(conn: Connection) -> Self { + Self { conn } + } + + pub fn create_schema(&self) -> Result<()> { + self.conn.execute_batch( + " + PRAGMA journal_mode = WAL; + PRAGMA foreign_keys = ON; + PRAGMA synchronous = NORMAL; + PRAGMA cache_size = -65536; + PRAGMA temp_store = MEMORY; + + CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS dictionaries ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + revision TEXT NOT NULL, + author TEXT, + url TEXT, + description TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + installed_at TEXT NOT NULL DEFAULT (datetime('now')), + lang TEXT NOT NULL, + UNIQUE (title, revision) + ); + CREATE TABLE IF NOT EXISTS glossaries ( + id INTEGER PRIMARY KEY, + hash TEXT NOT NULL UNIQUE, + content TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); + CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); + CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE); + CREATE TABLE IF NOT EXISTS terms ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + term TEXT NOT NULL, + reading TEXT NOT NULL, + def_tags_id INTEGER, + rules_id INTEGER, + score INTEGER NOT NULL DEFAULT 0, + glossary_id INTEGER NOT NULL REFERENCES glossaries(id), + sequence INTEGER, + term_tags_id INTEGER + ); + CREATE TABLE IF NOT EXISTS term_meta ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + term TEXT NOT NULL, + mode TEXT NOT NULL, + reading TEXT, + data TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + name TEXT NOT NULL, + category TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + notes TEXT, + score INTEGER NOT NULL DEFAULT 0, + UNIQUE (dict_id, name) + ); + CREATE TABLE IF NOT EXISTS kanji ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + kanji TEXT NOT NULL, + onyomi TEXT, + kunyomi TEXT, + tags TEXT, + meanings TEXT NOT NULL DEFAULT '[]', + stats TEXT NOT NULL DEFAULT '{}' + ); + ", + )?; + + self.conn.execute_batch( + " + CREATE INDEX IF NOT EXISTS dictionaries_lang ON dictionaries (lang); + + CREATE INDEX IF NOT EXISTS terms_term ON terms (term); + CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading); + ", + )?; + + let mut stmt = self + .conn + .prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?; + if stmt.exists([])? == false { + self.conn + .prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")? + .execute([])?; + self.conn + .prepare( + "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))", + )? + .execute([])?; + } + + Ok(()) + } + + pub fn import_bundled_zip_file( + &mut self, + zip_file: PathBuf, + lang: &str, + ) -> anyhow::Result { + self.create_schema()?; + + let f = + File::open(&zip_file).with_context(|| format!("opening zip {}", zip_file.display()))?; + let mut archive = ZipArchive::new(f).with_context(|| "reading zip archive")?; + + let index_file = match archive.by_name("index.json") { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + serde_json::from_str::(&s)? + } + Err(e) => { + // TODO: proper error handling + eprintln!("{}", e); + return Ok(YomitanZipImportResult { + exists: false, + load: false, + error: Some(format!("{}", e)), + }); + } + }; + + let title = index_file + .get("title") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let revision = index_file + .get("revision") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + + // Skip if already installed + let exists: bool = self + .conn + .prepare("SELECT 1 FROM dictionaries WHERE title = ?1 AND revision = ?2")? + .exists([title, revision])?; + if exists { + return Ok(YomitanZipImportResult { + exists: true, + load: false, + error: None, + }); + } + + let tx = self.conn.transaction()?; + tx.execute( + "INSERT INTO dictionaries (title, revision, author, url, description, lang) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + index_file.get("title").and_then(Value::as_str), + index_file.get("revision").and_then(Value::as_str), + index_file.get("author").and_then(Value::as_str), + index_file.get("url").and_then(Value::as_str), + index_file.get("description").and_then(Value::as_str), + lang, + ], + )?; + + let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?; + + // prepare statements + let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)"; + let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1"; + let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)"; + let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1"; + let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)"; + let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1"; + let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)"; + let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1"; + let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)"; + let insert_meta = + "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)"; + let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)"; + + let mut glossary_cache: HashMap = HashMap::new(); + let mut def_cache: HashMap = HashMap::new(); + let mut term_tags_cache: HashMap = HashMap::new(); + let mut rules_cache: HashMap = HashMap::new(); + + // term banks + let mut bank_i = 1; + loop { + let name = format!("term_bank_{}.json", bank_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let term = e.get(0).and_then(Value::as_str).unwrap_or(""); + let reading = e.get(1).and_then(Value::as_str).unwrap_or(""); + let def_tags = e + .get(2) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let rules = e + .get(3) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let score = e.get(4).and_then(Value::as_i64).unwrap_or(0); + let glossary_val = e.get(5).cloned().unwrap_or(Value::Null); + let sequence = e.get(6).and_then(Value::as_i64); + let term_tags = e + .get(7) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + let glossary_json = serde_json::to_string(&glossary_val)?; + let hash = blake3_hex(&glossary_json); + let glossary_id = if let Some(&id) = glossary_cache.get(&hash) { + id + } else { + tx.execute(insert_glossary, params![hash, glossary_json])?; + let id: i64 = + tx.query_row(select_glossary, params![hash], |r| r.get(0))?; + glossary_cache.insert(hash.clone(), id); + id + }; + + let def_id = if let Some(s) = def_tags.as_deref() { + intern(&tx, insert_def, select_def, &mut def_cache, s)? + } else { + 0 + }; + let rules_id = if let Some(s) = rules.as_deref() { + intern(&tx, insert_rules, select_rules, &mut rules_cache, s)? + } else { + 0 + }; + let term_tags_id = if let Some(s) = term_tags.as_deref() { + intern( + &tx, + insert_term_tags, + select_term_tags, + &mut term_tags_cache, + s, + )? + } else { + 0 + }; + + tx.execute( + insert_term, + params![ + dict_id, + term, + reading, + if def_id != 0 { + Some(def_id) + } else { + Option::::None + }, + if rules_id != 0 { + Some(rules_id) + } else { + Option::::None + }, + score, + glossary_id, + sequence, + if term_tags_id != 0 { + Some(term_tags_id) + } else { + Option::::None + }, + ], + )?; + } + bank_i += 1; + continue; + } + Err(_) => break, + } + } + + // term_meta banks + let mut meta_i = 1; + loop { + let name = format!("term_meta_bank_{}.json", meta_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let term = e.get(0).and_then(Value::as_str).unwrap_or(""); + let mode = e.get(1).and_then(Value::as_str).unwrap_or(""); + let data = e.get(2).cloned().unwrap_or(Value::Null); + let reading = data + .get("reading") + .and_then(Value::as_str) + .map(|s| s.to_string()); + tx.execute( + insert_meta, + params![dict_id, term, mode, reading, serde_json::to_string(&data)?], + )?; + } + meta_i += 1; + continue; + } + Err(_) => break, + } + } + + // tag banks + let mut tag_i = 1; + loop { + let name = format!("tag_bank_{}.json", tag_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let name = e.get(0).and_then(Value::as_str).unwrap_or(""); + let category = e.get(1).and_then(Value::as_str); + let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0); + let notes = e.get(3).and_then(Value::as_str); + let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0); + tx.execute( + insert_tag, + params![dict_id, name, category, sort_order, notes, tag_score], + )?; + } + tag_i += 1; + continue; + } + Err(_) => break, + } + } + + tx.commit()?; + + self.conn.execute_batch("VACUUM;")?; + + Ok(YomitanZipImportResult { + exists: true, + load: true, + error: None, + }) + } +} + +fn intern( + tx: &Transaction, + insert_sql: &str, + select_sql: &str, + cache: &mut HashMap, + value: &str, +) -> rusqlite::Result { + if let Some(&v) = cache.get(value) { + return Ok(v); + } + tx.execute(insert_sql, [value])?; + let id: i64 = tx.query_row(select_sql, [value], |r| r.get(0))?; + cache.insert(value.to_string(), id); + Ok(id) +} diff --git a/src-core/src/lib.rs b/src-core/src/lib.rs index 3969809..92bba00 100644 --- a/src-core/src/lib.rs +++ b/src-core/src/lib.rs @@ -1,7 +1,7 @@ -pub mod api; +mod api; pub mod db; -pub mod error; -pub mod models; +mod error; +mod models; pub mod service; pub use error::CJDicError; diff --git a/src-core/src/service.rs b/src-core/src/service.rs index f365dfe..6e94caa 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -1,14 +1,20 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; -use crate::{db::Database, db::YomitanRow, error::CJDicError, models::Entry}; +use rusqlite::Connection; + +use crate::{ + db::{Database, YomitanRow, YomitanWriter, YomitanZipImportResult}, + error::CJDicError, + models::Entry, +}; pub struct AppService { db: Database, } impl AppService { - pub fn new>(db_path: P) -> Result { - let db = Database::new(db_path)?; + pub fn new>(db_dir: P) -> Result { + let db = Database::new(db_dir)?; db.init()?; Ok(Self { db }) } @@ -31,7 +37,22 @@ impl AppService { ) -> Result, CJDicError> { Ok(self .db - .yomitan() + .yomitan()? .search_yomitan(q_term, q_reading, limit, offset)?) } + + pub fn get_yomitan_writer(&self) -> Result { + let conn = Connection::open(self.db.dir.join("yomitan.db"))?; + let writer = YomitanWriter::new(conn); + writer.create_schema()?; + Ok(writer) + } + + pub fn import_yomitan_zip_file( + writer: &mut YomitanWriter, + zip_file: PathBuf, + lang: &str, + ) -> Result { + Ok(writer.import_bundled_zip_file(zip_file, lang)?) + } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 190002a..b6e7463 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -14,9 +14,9 @@ pub fn run() { let app_dir = app.path().app_config_dir()?; create_dir_all(&app_dir)?; - // let db_path = app_dir.join("cjdic.db"); - let db_path = Path::new(r"D:\Projects\cjdic2\test.db"); - let service = AppService::new(&db_path)?; + // db_path is app_dir; + let db_dir = Path::new(r"D:\Projects\cjdic2\tmp\save-db"); + let service = AppService::new(&db_dir)?; app.manage(service); Ok(())