add yomitan import db/scripts

.gitignore

@@ -44,3 +44,5 @@ target
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+test.db*

Cargo.lock

@@ -58,6 +58,18 @@ version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
 [[package]]
 name = "async-broadcast"
 version = "0.7.2"
@@ -273,6 +285,20 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "blake3"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+ "cpufeatures",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -515,11 +541,12 @@ dependencies = [
 name = "cjdic2-core"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
+ "blake3",
  "reqwest",
  "rusqlite",
  "serde",
  "serde_json",
- "sha1",
  "thiserror 2.0.18",
  "walkdir",
  "zip",

src-core/Cargo.toml

@@ -12,6 +12,7 @@ serde_json = "1"
 rusqlite = { version = "0.38.0", features = ["bundled", "unlock_notify"] }
 thiserror = "2.0.18"
 zip = "8.1.0"
-sha1 = "0.10.6"
 walkdir = "2.5.0"
 reqwest = "0.13.2"
+blake3 = "1.8.3"
+anyhow = "1.0.102"

src-core/examples/add-to-yomitan.rs

@@ -0,0 +1,29 @@
+use std::{
+    fs::read_dir,
+    path::{Path, absolute},
+};
+
+use anyhow::{Context, Ok};
+use cjdic2_core::db::import_bundled_zip_file;
+use rusqlite::Connection;
+
+fn main() -> Result<(), anyhow::Error> {
+    let conn = &mut Connection::open("test.db")?;
+    let zip_dir = Path::new("src-tauri/resources/yomitan");
+    println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root
+
+    for entry in
+        read_dir(&zip_dir).with_context(|| format!("reading zip dir {}", zip_dir.display()))?
+    {
+        let e = entry?;
+        let p = e.path();
+
+        if p.extension().and_then(|s| s.to_str()) == Some("zip") {
+            println!("zip_file: {:?}", p);
+            println!("{:?}", import_bundled_zip_file(conn, p)?);
+        } else {
+            println!("not zip_file: {:?}", p);
+        }
+    }
+    Ok(())
+}

src-core/examples/search-yomitan.rs

@@ -0,0 +1,11 @@
+use anyhow::Ok;
+use cjdic2_core::db::search_yomitan;
+use rusqlite::Connection;
+
+fn main() -> Result<(), anyhow::Error> {
+    let conn = &mut Connection::open("test.db")?;
+    let rs = search_yomitan(conn, "食べる", "食べる", 5, 0)?;
+    println!("{:#?}", rs);
+
+    Ok(())
+}

src-core/src/db.rs

@@ -4,6 +4,9 @@ use rusqlite::{Connection, Result};
 
 use crate::models::Entry;
 
+mod yomitan;
+pub use yomitan::{YomitanRow, import_bundled_zip_file, search_yomitan};
+
 pub struct Database {
     conn: Mutex<Connection>,
 }

src-core/src/db/yomitan.rs

@@ -0,0 +1,444 @@
+use std::{collections::HashMap, fs::File, io::Read, path::PathBuf};
+
+use anyhow::Context;
+use blake3::hash;
+use rusqlite::{Connection, Transaction, params};
+use serde::Serialize;
+use serde_json::Value;
+use zip::ZipArchive;
+
+use crate::CJDicError;
+
+fn blake3_hex(s: &str) -> String {
+    format!("{}", hash(s.as_bytes()))
+}
+
+fn create_schema(conn: &Connection) -> rusqlite::Result<()> {
+    conn.execute_batch(
+        "PRAGMA journal_mode = WAL;
+        PRAGMA foreign_keys = ON;
+        PRAGMA synchronous  = NORMAL;
+        PRAGMA cache_size   = -65536;
+        PRAGMA temp_store   = MEMORY;
+
+        CREATE TABLE IF NOT EXISTS schema_meta (
+            key   TEXT PRIMARY KEY,
+            value TEXT NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS dictionaries (
+            id           INTEGER PRIMARY KEY,
+            title        TEXT    NOT NULL,
+            revision     TEXT    NOT NULL,
+            author       TEXT,
+            url          TEXT,
+            description  TEXT,
+            is_bundled   INTEGER NOT NULL DEFAULT 1,
+            sort_order   INTEGER NOT NULL DEFAULT 0,
+            installed_at TEXT    NOT NULL DEFAULT (datetime('now')),
+            UNIQUE (title, revision)
+        );
+        CREATE TABLE IF NOT EXISTS glossaries (
+            id      INTEGER PRIMARY KEY,
+            hash    TEXT NOT NULL UNIQUE,
+            content TEXT NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
+        CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
+        CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE);
+        CREATE TABLE IF NOT EXISTS terms (
+            id INTEGER PRIMARY KEY,
+            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+            term TEXT NOT NULL,
+            reading TEXT NOT NULL,
+            def_tags_id INTEGER,
+            rules_id INTEGER,
+            score INTEGER NOT NULL DEFAULT 0,
+            glossary_id INTEGER NOT NULL REFERENCES glossaries(id),
+            sequence INTEGER,
+            term_tags_id INTEGER
+        );
+        CREATE TABLE IF NOT EXISTS term_meta (
+            id INTEGER PRIMARY KEY,
+            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+            term TEXT NOT NULL,
+            mode TEXT NOT NULL,
+            reading TEXT,
+            data TEXT NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS tags (
+            id INTEGER PRIMARY KEY,
+            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+            name TEXT NOT NULL,
+            category TEXT,
+            sort_order INTEGER NOT NULL DEFAULT 0,
+            notes TEXT,
+            score INTEGER NOT NULL DEFAULT 0,
+            UNIQUE (dict_id, name)
+        );
+        CREATE TABLE IF NOT EXISTS kanji (
+            id INTEGER PRIMARY KEY,
+            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+            kanji TEXT NOT NULL,
+            onyomi TEXT,
+            kunyomi TEXT,
+            tags TEXT,
+            meanings TEXT NOT NULL DEFAULT '[]',
+            stats TEXT NOT NULL DEFAULT '{}'
+        );
+        ",
+    )?;
+
+    let mut stmt = conn.prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?;
+    if stmt.exists([])? == false {
+        conn.prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")?
+            .execute([])?;
+        conn.prepare(
+            "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))",
+        )?
+        .execute([])?;
+    }
+
+    Ok(())
+}
+
+fn intern(
+    tx: &Transaction,
+    insert_sql: &str,
+    select_sql: &str,
+    cache: &mut HashMap<String, i64>,
+    value: &str,
+) -> rusqlite::Result<i64> {
+    if let Some(&v) = cache.get(value) {
+        return Ok(v);
+    }
+    tx.execute(insert_sql, [value])?;
+    let id: i64 = tx.query_row(select_sql, [value], |r| r.get(0))?;
+    cache.insert(value.to_string(), id);
+    Ok(id)
+}
+
+#[derive(Serialize, Debug)]
+pub struct YomitanZipImportResult {
+    exists: bool,
+    done: bool,
+    error: Option<String>,
+}
+
+pub fn import_bundled_zip_file(
+    conn: &mut Connection,
+    zip_path: PathBuf,
+) -> anyhow::Result<YomitanZipImportResult, CJDicError> {
+    create_schema(&conn)?;
+
+    let f = File::open(&zip_path).with_context(|| format!("opening zip {}", zip_path.display()))?;
+    let mut archive = ZipArchive::new(f).with_context(|| "reading zip archive")?;
+
+    let index_file = match archive.by_name("index.json") {
+        Ok(mut f) => {
+            let mut s = String::new();
+            f.read_to_string(&mut s)?;
+            serde_json::from_str::<Value>(&s)?
+        }
+        Err(e) => {
+            // TODO: proper error handling
+            eprintln!("{}", e);
+            return Ok(YomitanZipImportResult {
+                exists: false,
+                done: false,
+                error: Some(format!("{}", e)),
+            });
+        }
+    };
+
+    let title = index_file
+        .get("title")
+        .and_then(Value::as_str)
+        .unwrap_or("")
+        .to_string();
+    let revision = index_file
+        .get("revision")
+        .and_then(Value::as_str)
+        .unwrap_or("")
+        .to_string();
+
+    // Skip if already installed
+    let exists: bool = conn
+        .prepare("SELECT 1 FROM dictionaries WHERE title = ?1 AND revision = ?2")?
+        .exists([title, revision])?;
+    if exists {
+        return Ok(YomitanZipImportResult {
+            exists: true,
+            done: false,
+            error: None,
+        });
+    }
+
+    let tx = conn.transaction()?;
+    tx.execute(
+            "INSERT INTO dictionaries (title, revision, author, url, description, is_bundled) VALUES (?1, ?2, ?3, ?4, ?5, 1)",
+            [
+                index_file.get("title").and_then(Value::as_str),
+                index_file.get("revision").and_then(Value::as_str),
+                index_file.get("author").and_then(Value::as_str),
+                index_file.get("url").and_then(Value::as_str),
+                index_file.get("description").and_then(Value::as_str),
+            ],
+        )?;
+
+    let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?;
+
+    // prepare statements
+    let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)";
+    let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1";
+    let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)";
+    let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1";
+    let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)";
+    let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1";
+    let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)";
+    let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1";
+    let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)";
+    let insert_meta =
+        "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
+    let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)";
+
+    let mut glossary_cache: HashMap<String, i64> = HashMap::new();
+    let mut def_cache: HashMap<String, i64> = HashMap::new();
+    let mut term_tags_cache: HashMap<String, i64> = HashMap::new();
+    let mut rules_cache: HashMap<String, i64> = HashMap::new();
+
+    // term banks
+    let mut bank_i = 1;
+    loop {
+        let name = format!("term_bank_{}.json", bank_i);
+        match archive.by_name(&name) {
+            Ok(mut f) => {
+                let mut s = String::new();
+                f.read_to_string(&mut s)?;
+                let entries: Vec<Value> = serde_json::from_str(&s)?;
+                for e in entries {
+                    let term = e.get(0).and_then(Value::as_str).unwrap_or("");
+                    let reading = e.get(1).and_then(Value::as_str).unwrap_or("");
+                    let def_tags = e
+                        .get(2)
+                        .and_then(Value::as_str)
+                        .map(|s| s.trim())
+                        .filter(|s| !s.is_empty())
+                        .map(|s| s.to_string());
+                    let rules = e
+                        .get(3)
+                        .and_then(Value::as_str)
+                        .map(|s| s.trim())
+                        .filter(|s| !s.is_empty())
+                        .map(|s| s.to_string());
+                    let score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
+                    let glossary_val = e.get(5).cloned().unwrap_or(Value::Null);
+                    let sequence = e.get(6).and_then(Value::as_i64);
+                    let term_tags = e
+                        .get(7)
+                        .and_then(Value::as_str)
+                        .map(|s| s.trim())
+                        .filter(|s| !s.is_empty())
+                        .map(|s| s.to_string());
+
+                    let glossary_json = serde_json::to_string(&glossary_val)?;
+                    let hash = blake3_hex(&glossary_json);
+                    let glossary_id = if let Some(&id) = glossary_cache.get(&hash) {
+                        id
+                    } else {
+                        tx.execute(insert_glossary, params![hash, glossary_json])?;
+                        let id: i64 = tx.query_row(select_glossary, params![hash], |r| r.get(0))?;
+                        glossary_cache.insert(hash.clone(), id);
+                        id
+                    };
+
+                    let def_id = if let Some(s) = def_tags.as_deref() {
+                        intern(&tx, insert_def, select_def, &mut def_cache, s)?
+                    } else {
+                        0
+                    };
+                    let rules_id = if let Some(s) = rules.as_deref() {
+                        intern(&tx, insert_rules, select_rules, &mut rules_cache, s)?
+                    } else {
+                        0
+                    };
+                    let term_tags_id = if let Some(s) = term_tags.as_deref() {
+                        intern(
+                            &tx,
+                            insert_term_tags,
+                            select_term_tags,
+                            &mut term_tags_cache,
+                            s,
+                        )?
+                    } else {
+                        0
+                    };
+
+                    tx.execute(
+                        insert_term,
+                        params![
+                            dict_id,
+                            term,
+                            reading,
+                            if def_id != 0 {
+                                Some(def_id)
+                            } else {
+                                Option::<i64>::None
+                            },
+                            if rules_id != 0 {
+                                Some(rules_id)
+                            } else {
+                                Option::<i64>::None
+                            },
+                            score,
+                            glossary_id,
+                            sequence,
+                            if term_tags_id != 0 {
+                                Some(term_tags_id)
+                            } else {
+                                Option::<i64>::None
+                            },
+                        ],
+                    )?;
+                }
+                bank_i += 1;
+                continue;
+            }
+            Err(_) => break,
+        }
+    }
+
+    // term_meta banks
+    let mut meta_i = 1;
+    loop {
+        let name = format!("term_meta_bank_{}.json", meta_i);
+        match archive.by_name(&name) {
+            Ok(mut f) => {
+                let mut s = String::new();
+                f.read_to_string(&mut s)?;
+                let entries: Vec<Value> = serde_json::from_str(&s)?;
+                for e in entries {
+                    let term = e.get(0).and_then(Value::as_str).unwrap_or("");
+                    let mode = e.get(1).and_then(Value::as_str).unwrap_or("");
+                    let data = e.get(2).cloned().unwrap_or(Value::Null);
+                    let reading = data
+                        .get("reading")
+                        .and_then(Value::as_str)
+                        .map(|s| s.to_string());
+                    tx.execute(
+                        insert_meta,
+                        params![dict_id, term, mode, reading, serde_json::to_string(&data)?],
+                    )?;
+                }
+                meta_i += 1;
+                continue;
+            }
+            Err(_) => break,
+        }
+    }
+
+    // tag banks
+    let mut tag_i = 1;
+    loop {
+        let name = format!("tag_bank_{}.json", tag_i);
+        match archive.by_name(&name) {
+            Ok(mut f) => {
+                let mut s = String::new();
+                f.read_to_string(&mut s)?;
+                let entries: Vec<Value> = serde_json::from_str(&s)?;
+                for e in entries {
+                    let name = e.get(0).and_then(Value::as_str).unwrap_or("");
+                    let category = e.get(1).and_then(Value::as_str);
+                    let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0);
+                    let notes = e.get(3).and_then(Value::as_str);
+                    let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
+                    tx.execute(
+                        insert_tag,
+                        params![dict_id, name, category, sort_order, notes, tag_score],
+                    )?;
+                }
+                tag_i += 1;
+                continue;
+            }
+            Err(_) => break,
+        }
+    }
+
+    tx.commit()?;
+
+    conn.execute_batch("VACUUM;")?;
+
+    Ok(YomitanZipImportResult {
+        exists: true,
+        done: true,
+        error: None,
+    })
+}
+
+#[derive(Serialize, Debug)]
+pub struct YomitanRow {
+    term: String,
+    reading: String,
+    def_tags: String,
+    rules: String,
+    score: i64,
+    glossary_json: String,
+    sequence: Option<i64>,
+    term_tags: String,
+    dict_title: String,
+}
+
+pub fn search_yomitan(
+    conn: &Connection,
+    q_term: &str,
+    q_reading: &str,
+    limit: u32,
+    offset: u32,
+) -> Result<Vec<YomitanRow>, CJDicError> {
+    let and_or = if q_term == q_reading { "OR" } else { "AND" };
+
+    let sql = format!(
+        r#"
+      SELECT
+        t.term,
+        t.reading,
+        COALESCE(dt.tags,  '')  AS def_tags,
+        COALESCE(r.rules,  '')  AS rules,
+        t.score,
+        g.content               AS glossary_json,
+        t.sequence,
+        COALESCE(tt.tags,  '')  AS term_tags,
+        d.title                 AS dict_title
+      FROM terms t
+      JOIN  glossaries    g  ON g.id  = t.glossary_id
+      JOIN  dictionaries  d  ON d.id  = t.dict_id
+      LEFT JOIN def_tag_sets  dt ON dt.id = t.def_tags_id
+      LEFT JOIN rule_sets      r ON r.id  = t.rules_id
+      LEFT JOIN term_tag_sets tt ON tt.id = t.term_tags_id
+      WHERE t.term LIKE ?1 ESCAPE '\' {} t.reading LIKE ?2 ESCAPE '\'
+      ORDER BY t.score DESC
+      LIMIT ?3 OFFSET ?4
+    "#,
+        and_or
+    );
+
+    let mut stmt = conn.prepare(&sql)?;
+    let rows = stmt.query_map(params![q_term, q_reading, limit, offset], |r| {
+        Ok(YomitanRow {
+            term: r.get(0)?,
+            reading: r.get(1)?,
+            def_tags: r.get(2)?,
+            rules: r.get(3)?,
+            score: r.get(4)?,
+            glossary_json: r.get(5)?,
+            sequence: r.get(6)?,
+            term_tags: r.get(7)?,
+            dict_title: r.get(8)?,
+        })
+    })?;
+
+    let mut out = Vec::new();
+    for r in rows {
+        out.push(r?);
+    }
+    Ok(out)
+}

src-core/src/error.rs

@@ -1,4 +1,5 @@
 use thiserror::Error;
+use zip::result::ZipError;
 
 #[derive(Debug, Error)]
 pub enum CJDicError {
@@ -8,6 +9,15 @@ pub enum CJDicError {
     #[error("Serialization error: {0}")]
     Serialization(#[from] serde_json::Error),
 
+    #[error("Anyhow error: {0}")]
+    AnyhowError(#[from] anyhow::Error),
+
+    #[error("IO error: {0}")]
+    IOError(#[from] std::io::Error),
+
+    #[error("Zip error: {0}")]
+    ZipError(#[from] ZipError),
+
     #[error("Not found")]
     NotFound,
 }

src-tauri/resources/.gitignore

@@ -0,0 +1,4 @@
+*.zip
+# *.json.zip from https://github.com/scriptin/jmdict-simplified/releases
+# TODO: jmdict.db build script
+jmdict.db

src-tauri/resources/yomitan/.gitignore

@@ -0,0 +1,2 @@
+*.zip
+# Pixiv_*.zip => Pixiv.zip from https://github.com/MarvNC/pixiv-yomitan/releases