From 8330a78c12ff4ccc792fac0c5687beff0ffd97e1 Mon Sep 17 00:00:00 2001 From: polv Date: Sun, 22 Feb 2026 11:46:46 +0700 Subject: [PATCH] add yomitan import db/scripts --- .gitignore | 2 + Cargo.lock | 29 +- src-core/Cargo.toml | 3 +- src-core/examples/add-to-yomitan.rs | 29 ++ src-core/examples/search-yomitan.rs | 11 + src-core/src/db.rs | 3 + src-core/src/db/yomitan.rs | 444 +++++++++++++++++++++++++ src-core/src/error.rs | 10 + src-tauri/resources/.gitignore | 4 + src-tauri/resources/yomitan/.gitignore | 2 + 10 files changed, 535 insertions(+), 2 deletions(-) create mode 100644 src-core/examples/add-to-yomitan.rs create mode 100644 src-core/examples/search-yomitan.rs create mode 100644 src-core/src/db/yomitan.rs create mode 100644 src-tauri/resources/.gitignore create mode 100644 src-tauri/resources/yomitan/.gitignore diff --git a/.gitignore b/.gitignore index 6fac338..a6ad858 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ target # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +test.db* diff --git a/Cargo.lock b/Cargo.lock index 6ba90e4..4435cf5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,18 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -273,6 +285,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -515,11 +541,12 @@ dependencies = [ name = "cjdic2-core" version = "0.1.0" dependencies = [ + "anyhow", + "blake3", "reqwest", "rusqlite", "serde", "serde_json", - "sha1", "thiserror 2.0.18", "walkdir", "zip", diff --git a/src-core/Cargo.toml b/src-core/Cargo.toml index ac7bf4b..d1c73fa 100644 --- a/src-core/Cargo.toml +++ b/src-core/Cargo.toml @@ -12,6 +12,7 @@ serde_json = "1" rusqlite = { version = "0.38.0", features = ["bundled", "unlock_notify"] } thiserror = "2.0.18" zip = "8.1.0" -sha1 = "0.10.6" walkdir = "2.5.0" reqwest = "0.13.2" +blake3 = "1.8.3" +anyhow = "1.0.102" diff --git a/src-core/examples/add-to-yomitan.rs b/src-core/examples/add-to-yomitan.rs new file mode 100644 index 0000000..69b33c2 --- /dev/null +++ b/src-core/examples/add-to-yomitan.rs @@ -0,0 +1,29 @@ +use std::{ + fs::read_dir, + path::{Path, absolute}, +}; + +use anyhow::{Context, Ok}; +use cjdic2_core::db::import_bundled_zip_file; +use rusqlite::Connection; + +fn main() -> Result<(), anyhow::Error> { + let conn = &mut Connection::open("test.db")?; + let zip_dir = Path::new("src-tauri/resources/yomitan"); + println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root + + for entry in + read_dir(&zip_dir).with_context(|| format!("reading zip dir {}", zip_dir.display()))? + { + let e = entry?; + let p = e.path(); + + if p.extension().and_then(|s| s.to_str()) == Some("zip") { + println!("zip_file: {:?}", p); + println!("{:?}", import_bundled_zip_file(conn, p)?); + } else { + println!("not zip_file: {:?}", p); + } + } + Ok(()) +} diff --git a/src-core/examples/search-yomitan.rs b/src-core/examples/search-yomitan.rs new file mode 100644 index 0000000..9afc55e --- /dev/null +++ b/src-core/examples/search-yomitan.rs @@ -0,0 +1,11 @@ +use anyhow::Ok; +use cjdic2_core::db::search_yomitan; +use rusqlite::Connection; + +fn main() -> Result<(), anyhow::Error> { + let conn = &mut Connection::open("test.db")?; + let rs = search_yomitan(conn, "食べる", "食べる", 5, 0)?; + println!("{:#?}", rs); + + Ok(()) +} diff --git a/src-core/src/db.rs b/src-core/src/db.rs index b448a53..6d3b674 100644 --- a/src-core/src/db.rs +++ b/src-core/src/db.rs @@ -4,6 +4,9 @@ use rusqlite::{Connection, Result}; use crate::models::Entry; +mod yomitan; +pub use yomitan::{YomitanRow, import_bundled_zip_file, search_yomitan}; + pub struct Database { conn: Mutex, } diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs new file mode 100644 index 0000000..911b18b --- /dev/null +++ b/src-core/src/db/yomitan.rs @@ -0,0 +1,444 @@ +use std::{collections::HashMap, fs::File, io::Read, path::PathBuf}; + +use anyhow::Context; +use blake3::hash; +use rusqlite::{Connection, Transaction, params}; +use serde::Serialize; +use serde_json::Value; +use zip::ZipArchive; + +use crate::CJDicError; + +fn blake3_hex(s: &str) -> String { + format!("{}", hash(s.as_bytes())) +} + +fn create_schema(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA foreign_keys = ON; + PRAGMA synchronous = NORMAL; + PRAGMA cache_size = -65536; + PRAGMA temp_store = MEMORY; + + CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS dictionaries ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + revision TEXT NOT NULL, + author TEXT, + url TEXT, + description TEXT, + is_bundled INTEGER NOT NULL DEFAULT 1, + sort_order INTEGER NOT NULL DEFAULT 0, + installed_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (title, revision) + ); + CREATE TABLE IF NOT EXISTS glossaries ( + id INTEGER PRIMARY KEY, + hash TEXT NOT NULL UNIQUE, + content TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); + CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); + CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE); + CREATE TABLE IF NOT EXISTS terms ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + term TEXT NOT NULL, + reading TEXT NOT NULL, + def_tags_id INTEGER, + rules_id INTEGER, + score INTEGER NOT NULL DEFAULT 0, + glossary_id INTEGER NOT NULL REFERENCES glossaries(id), + sequence INTEGER, + term_tags_id INTEGER + ); + CREATE TABLE IF NOT EXISTS term_meta ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + term TEXT NOT NULL, + mode TEXT NOT NULL, + reading TEXT, + data TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + name TEXT NOT NULL, + category TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + notes TEXT, + score INTEGER NOT NULL DEFAULT 0, + UNIQUE (dict_id, name) + ); + CREATE TABLE IF NOT EXISTS kanji ( + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + kanji TEXT NOT NULL, + onyomi TEXT, + kunyomi TEXT, + tags TEXT, + meanings TEXT NOT NULL DEFAULT '[]', + stats TEXT NOT NULL DEFAULT '{}' + ); + ", + )?; + + let mut stmt = conn.prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?; + if stmt.exists([])? == false { + conn.prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")? + .execute([])?; + conn.prepare( + "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))", + )? + .execute([])?; + } + + Ok(()) +} + +fn intern( + tx: &Transaction, + insert_sql: &str, + select_sql: &str, + cache: &mut HashMap, + value: &str, +) -> rusqlite::Result { + if let Some(&v) = cache.get(value) { + return Ok(v); + } + tx.execute(insert_sql, [value])?; + let id: i64 = tx.query_row(select_sql, [value], |r| r.get(0))?; + cache.insert(value.to_string(), id); + Ok(id) +} + +#[derive(Serialize, Debug)] +pub struct YomitanZipImportResult { + exists: bool, + done: bool, + error: Option, +} + +pub fn import_bundled_zip_file( + conn: &mut Connection, + zip_path: PathBuf, +) -> anyhow::Result { + create_schema(&conn)?; + + let f = File::open(&zip_path).with_context(|| format!("opening zip {}", zip_path.display()))?; + let mut archive = ZipArchive::new(f).with_context(|| "reading zip archive")?; + + let index_file = match archive.by_name("index.json") { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + serde_json::from_str::(&s)? + } + Err(e) => { + // TODO: proper error handling + eprintln!("{}", e); + return Ok(YomitanZipImportResult { + exists: false, + done: false, + error: Some(format!("{}", e)), + }); + } + }; + + let title = index_file + .get("title") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let revision = index_file + .get("revision") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + + // Skip if already installed + let exists: bool = conn + .prepare("SELECT 1 FROM dictionaries WHERE title = ?1 AND revision = ?2")? + .exists([title, revision])?; + if exists { + return Ok(YomitanZipImportResult { + exists: true, + done: false, + error: None, + }); + } + + let tx = conn.transaction()?; + tx.execute( + "INSERT INTO dictionaries (title, revision, author, url, description, is_bundled) VALUES (?1, ?2, ?3, ?4, ?5, 1)", + [ + index_file.get("title").and_then(Value::as_str), + index_file.get("revision").and_then(Value::as_str), + index_file.get("author").and_then(Value::as_str), + index_file.get("url").and_then(Value::as_str), + index_file.get("description").and_then(Value::as_str), + ], + )?; + + let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?; + + // prepare statements + let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)"; + let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1"; + let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)"; + let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1"; + let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)"; + let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1"; + let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)"; + let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1"; + let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)"; + let insert_meta = + "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)"; + let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)"; + + let mut glossary_cache: HashMap = HashMap::new(); + let mut def_cache: HashMap = HashMap::new(); + let mut term_tags_cache: HashMap = HashMap::new(); + let mut rules_cache: HashMap = HashMap::new(); + + // term banks + let mut bank_i = 1; + loop { + let name = format!("term_bank_{}.json", bank_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let term = e.get(0).and_then(Value::as_str).unwrap_or(""); + let reading = e.get(1).and_then(Value::as_str).unwrap_or(""); + let def_tags = e + .get(2) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let rules = e + .get(3) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let score = e.get(4).and_then(Value::as_i64).unwrap_or(0); + let glossary_val = e.get(5).cloned().unwrap_or(Value::Null); + let sequence = e.get(6).and_then(Value::as_i64); + let term_tags = e + .get(7) + .and_then(Value::as_str) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + let glossary_json = serde_json::to_string(&glossary_val)?; + let hash = blake3_hex(&glossary_json); + let glossary_id = if let Some(&id) = glossary_cache.get(&hash) { + id + } else { + tx.execute(insert_glossary, params![hash, glossary_json])?; + let id: i64 = tx.query_row(select_glossary, params![hash], |r| r.get(0))?; + glossary_cache.insert(hash.clone(), id); + id + }; + + let def_id = if let Some(s) = def_tags.as_deref() { + intern(&tx, insert_def, select_def, &mut def_cache, s)? + } else { + 0 + }; + let rules_id = if let Some(s) = rules.as_deref() { + intern(&tx, insert_rules, select_rules, &mut rules_cache, s)? + } else { + 0 + }; + let term_tags_id = if let Some(s) = term_tags.as_deref() { + intern( + &tx, + insert_term_tags, + select_term_tags, + &mut term_tags_cache, + s, + )? + } else { + 0 + }; + + tx.execute( + insert_term, + params![ + dict_id, + term, + reading, + if def_id != 0 { + Some(def_id) + } else { + Option::::None + }, + if rules_id != 0 { + Some(rules_id) + } else { + Option::::None + }, + score, + glossary_id, + sequence, + if term_tags_id != 0 { + Some(term_tags_id) + } else { + Option::::None + }, + ], + )?; + } + bank_i += 1; + continue; + } + Err(_) => break, + } + } + + // term_meta banks + let mut meta_i = 1; + loop { + let name = format!("term_meta_bank_{}.json", meta_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let term = e.get(0).and_then(Value::as_str).unwrap_or(""); + let mode = e.get(1).and_then(Value::as_str).unwrap_or(""); + let data = e.get(2).cloned().unwrap_or(Value::Null); + let reading = data + .get("reading") + .and_then(Value::as_str) + .map(|s| s.to_string()); + tx.execute( + insert_meta, + params![dict_id, term, mode, reading, serde_json::to_string(&data)?], + )?; + } + meta_i += 1; + continue; + } + Err(_) => break, + } + } + + // tag banks + let mut tag_i = 1; + loop { + let name = format!("tag_bank_{}.json", tag_i); + match archive.by_name(&name) { + Ok(mut f) => { + let mut s = String::new(); + f.read_to_string(&mut s)?; + let entries: Vec = serde_json::from_str(&s)?; + for e in entries { + let name = e.get(0).and_then(Value::as_str).unwrap_or(""); + let category = e.get(1).and_then(Value::as_str); + let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0); + let notes = e.get(3).and_then(Value::as_str); + let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0); + tx.execute( + insert_tag, + params![dict_id, name, category, sort_order, notes, tag_score], + )?; + } + tag_i += 1; + continue; + } + Err(_) => break, + } + } + + tx.commit()?; + + conn.execute_batch("VACUUM;")?; + + Ok(YomitanZipImportResult { + exists: true, + done: true, + error: None, + }) +} + +#[derive(Serialize, Debug)] +pub struct YomitanRow { + term: String, + reading: String, + def_tags: String, + rules: String, + score: i64, + glossary_json: String, + sequence: Option, + term_tags: String, + dict_title: String, +} + +pub fn search_yomitan( + conn: &Connection, + q_term: &str, + q_reading: &str, + limit: u32, + offset: u32, +) -> Result, CJDicError> { + let and_or = if q_term == q_reading { "OR" } else { "AND" }; + + let sql = format!( + r#" + SELECT + t.term, + t.reading, + COALESCE(dt.tags, '') AS def_tags, + COALESCE(r.rules, '') AS rules, + t.score, + g.content AS glossary_json, + t.sequence, + COALESCE(tt.tags, '') AS term_tags, + d.title AS dict_title + FROM terms t + JOIN glossaries g ON g.id = t.glossary_id + JOIN dictionaries d ON d.id = t.dict_id + LEFT JOIN def_tag_sets dt ON dt.id = t.def_tags_id + LEFT JOIN rule_sets r ON r.id = t.rules_id + LEFT JOIN term_tag_sets tt ON tt.id = t.term_tags_id + WHERE t.term LIKE ?1 ESCAPE '\' {} t.reading LIKE ?2 ESCAPE '\' + ORDER BY t.score DESC + LIMIT ?3 OFFSET ?4 + "#, + and_or + ); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(params![q_term, q_reading, limit, offset], |r| { + Ok(YomitanRow { + term: r.get(0)?, + reading: r.get(1)?, + def_tags: r.get(2)?, + rules: r.get(3)?, + score: r.get(4)?, + glossary_json: r.get(5)?, + sequence: r.get(6)?, + term_tags: r.get(7)?, + dict_title: r.get(8)?, + }) + })?; + + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) +} diff --git a/src-core/src/error.rs b/src-core/src/error.rs index c0c99c8..6017c0e 100644 --- a/src-core/src/error.rs +++ b/src-core/src/error.rs @@ -1,4 +1,5 @@ use thiserror::Error; +use zip::result::ZipError; #[derive(Debug, Error)] pub enum CJDicError { @@ -8,6 +9,15 @@ pub enum CJDicError { #[error("Serialization error: {0}")] Serialization(#[from] serde_json::Error), + #[error("Anyhow error: {0}")] + AnyhowError(#[from] anyhow::Error), + + #[error("IO error: {0}")] + IOError(#[from] std::io::Error), + + #[error("Zip error: {0}")] + ZipError(#[from] ZipError), + #[error("Not found")] NotFound, } diff --git a/src-tauri/resources/.gitignore b/src-tauri/resources/.gitignore new file mode 100644 index 0000000..ece471a --- /dev/null +++ b/src-tauri/resources/.gitignore @@ -0,0 +1,4 @@ +*.zip +# *.json.zip from https://github.com/scriptin/jmdict-simplified/releases +# TODO: jmdict.db build script +jmdict.db diff --git a/src-tauri/resources/yomitan/.gitignore b/src-tauri/resources/yomitan/.gitignore new file mode 100644 index 0000000..2e67351 --- /dev/null +++ b/src-tauri/resources/yomitan/.gitignore @@ -0,0 +1,2 @@ +*.zip +# Pixiv_*.zip => Pixiv.zip from https://github.com/MarvNC/pixiv-yomitan/releases