main as user.db, then attach readonly db's

src-core/examples/add-to-yomitan.rs

@@ -5,13 +5,14 @@ use std::{
 };
 
 use anyhow::{Context, Ok};
-use cjdic2_core::db::Database;
+use cjdic2_core::AppService;
 
 mod common;
-use common::get_db_path;
+use common::get_db_dir;
 
 fn main() -> Result<(), anyhow::Error> {
-    let db = Database::new(get_db_path())?;
+    let service = AppService::new(get_db_dir())?;
+    let mut writer = service.get_yomitan_writer()?;
 
     let zip_dir = Path::new("tmp/yomitan");
     println!("zip_dir: {:?}", absolute(zip_dir)); // Relative to workspace root
@@ -26,7 +27,10 @@ fn main() -> Result<(), anyhow::Error> {
             let start = Instant::now();
 
             println!("zip_file: {:?}", p);
-            println!("{:?}", db.yomitan().import_bundled_zip_file(p, "ja")?,);
+            println!(
+                "{:?}",
+                AppService::import_yomitan_zip_file(&mut writer, p, "ja")?,
+            );
             println!("[{:.2?}]", start.elapsed(),);
         } else {
             println!("not zip_file: {:?}", p);

src-core/examples/common/mod.rs

@@ -1,5 +1,5 @@
 use std::path::PathBuf;
 
-pub fn get_db_path() -> PathBuf {
+pub fn get_db_dir() -> PathBuf {
-    PathBuf::from("test.db")
+    PathBuf::from("tmp/save-db")
 }

src-core/examples/search-yomitan.rs

@@ -1,16 +1,16 @@
 use std::time::Instant;
 
 use anyhow::Ok;
-use cjdic2_core::db::Database;
+use cjdic2_core::AppService;
 
 mod common;
-use common::get_db_path;
+use common::get_db_dir;
 
 fn main() -> Result<(), anyhow::Error> {
-    let db = Database::new(get_db_path())?;
+    let service = AppService::new(get_db_dir())?;
 
     let start = Instant::now();
-    let rs = db.yomitan().search_yomitan("擦る", "する", 10, 0)?;
+    let rs = service.search_yomitan("擦る", "する", 10, 0)?;
     println!("{:#?}", rs);
     println!("n={}", rs.len());
     println!("[{:.2?}]", start.elapsed());

src-core/examples/yomitan-mod-schema.rs

@@ -1,16 +1,16 @@
 use std::time::Instant;
 
 use anyhow::Ok;
-use cjdic2_core::db::Database;
+use cjdic2_core::AppService;
 
 mod common;
-use common::get_db_path;
+use common::get_db_dir;
 
 fn main() -> Result<(), anyhow::Error> {
-    let db = Database::new(get_db_path())?;
+    let service = AppService::new(get_db_dir())?;
 
     let start = Instant::now();
-    db.yomitan().create_schema()?;
+    service.get_yomitan_writer()?;
     println!("[{:.2?}]", start.elapsed());
 
     Ok(())

src-core/src/api.rs

@@ -1,6 +1 @@
-use crate::{error::CJDicError, service::AppService};
 
-pub fn list_entries_json(service: &AppService) -> Result<String, CJDicError> {
-    let entries = service.list_entries()?;
-    Ok(serde_json::to_string(&entries)?)
-}

src-core/src/db.rs

@@ -1,30 +1,86 @@
 use std::{
-    path::Path,
+    env::current_dir,
-    sync::{Arc, Mutex},
+    path::{Path, PathBuf},
+    sync::{
+        Arc, Mutex,
+        atomic::{AtomicBool, Ordering},
+    },
 };
 
-use rusqlite::{Connection, Result};
+use anyhow::Result;
+use rusqlite::Connection;
 
 use crate::models::Entry;
 
 mod yomitan;
-pub use yomitan::{YomitanRow, YomitanZipImportResult};
+pub use yomitan::YomitanRow;
+
+mod yomitan_writer;
+pub use yomitan_writer::{YomitanWriter, YomitanZipImportResult};
 
 #[derive(Clone)]
-pub struct Database {
+pub(crate) struct Database {
     pub(crate) conn: Arc<Mutex<Connection>>,
+    pub(crate) dir: PathBuf,
+    yomitan_attached: Arc<AtomicBool>,
 }
 
 impl Database {
-    pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {
+    pub fn new(db_dir: impl AsRef<Path>) -> Result<Self> {
-        let conn = Connection::open(path)?;
+        let dir = db_dir.as_ref().to_path_buf();
+        let conn = Connection::open(dir.join("user.db"))?;
+
+        conn.execute_batch(
+            r"
+            PRAGMA jornal_mode = WAL;
+            PRAGMA synchronous = NORMAL;
+            PRAGMA foreign_keys = ON;
+            ",
+        )?;
+
         Ok(Self {
             conn: Arc::new(Mutex::new(conn)),
+            dir,
+            yomitan_attached: Arc::new(AtomicBool::new(false)),
         })
     }
 
-    pub fn yomitan(&self) -> yomitan::YomitanDatabase {
+    pub fn yomitan(&self) -> Result<yomitan::YomitanDatabase> {
-        yomitan::YomitanDatabase::new(self.clone())
+        self.ensure_yomitan_attached()?;
+        Ok(yomitan::YomitanDatabase::new(self.clone()))
+    }
+
+    fn ensure_yomitan_attached(&self) -> Result<()> {
+        if self.yomitan_attached.load(Ordering::Acquire) {
+            return Ok(());
+        }
+
+        let conn = self.conn.lock().unwrap();
+        if !self.yomitan_attached.load(Ordering::Relaxed) {
+            let dir = self.dir.clone();
+            let path = if self.dir.is_absolute() {
+                dir
+            } else {
+                current_dir()?.join(dir)
+            };
+            let mut path = path
+                .join("yomitan.db")
+                .to_string_lossy()
+                .replace(r"\", r"/");
+
+            #[cfg(windows)]
+            {
+                if !path.starts_with("/") {
+                    path = format!("/{}", path);
+                }
+            }
+            let uri = format!("file:{}?mode=ro&immutable=1", path);
+
+            conn.execute("ATTACH DATABASE ?1 AS yomitan", [uri])?;
+            self.yomitan_attached.store(true, Ordering::Release);
+        }
+
+        Ok(())
     }
 
     pub fn init(&self) -> Result<()> {

src-core/src/db/yomitan.rs

@@ -1,25 +1,8 @@
-use std::{collections::HashMap, fs::File, io::Read, path::PathBuf};
+use rusqlite::{Result, params};
-
-use anyhow::Context;
-use blake3::hash;
-use rusqlite::{Result, Transaction, params};
 use serde::Serialize;
-use serde_json::Value;
-use zip::ZipArchive;
 
 use crate::{CJDicError, db::Database};
 
-fn blake3_hex(s: &str) -> String {
-    format!("{}", hash(s.as_bytes()))
-}
-
-#[derive(Serialize, Debug)]
-pub struct YomitanZipImportResult {
-    exists: bool,
-    load: bool,
-    error: Option<String>,
-}
-
 #[derive(Serialize, Debug)]
 pub struct YomitanRow {
     term: String,
@@ -42,362 +25,6 @@ impl YomitanDatabase {
         Self { db }
     }
 
-    pub fn create_schema(&self) -> Result<()> {
-        let conn = self.db.conn.lock().unwrap();
-
-        conn.execute_batch(
-        "
-        PRAGMA journal_mode = WAL;
-        PRAGMA foreign_keys = ON;
-        PRAGMA synchronous  = NORMAL;
-        PRAGMA cache_size   = -65536;
-        PRAGMA temp_store   = MEMORY;
-
-        CREATE TABLE IF NOT EXISTS schema_meta (
-            key   TEXT PRIMARY KEY,
-            value TEXT NOT NULL
-        );
-        CREATE TABLE IF NOT EXISTS dictionaries (
-            id           INTEGER PRIMARY KEY,
-            title        TEXT    NOT NULL,
-            revision     TEXT    NOT NULL,
-            author       TEXT,
-            url          TEXT,
-            description  TEXT,
-            sort_order   INTEGER NOT NULL DEFAULT 0,
-            installed_at TEXT    NOT NULL DEFAULT (datetime('now')),
-            lang         TEXT    NOT NULL,
-            UNIQUE (title, revision)
-        );
-        CREATE TABLE IF NOT EXISTS glossaries (
-            id      INTEGER PRIMARY KEY,
-            hash    TEXT NOT NULL UNIQUE,
-            content TEXT NOT NULL
-        );
-        CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
-        CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
-        CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE);
-        CREATE TABLE IF NOT EXISTS terms (
-            id INTEGER PRIMARY KEY,
-            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
-            term TEXT NOT NULL,
-            reading TEXT NOT NULL,
-            def_tags_id INTEGER,
-            rules_id INTEGER,
-            score INTEGER NOT NULL DEFAULT 0,
-            glossary_id INTEGER NOT NULL REFERENCES glossaries(id),
-            sequence INTEGER,
-            term_tags_id INTEGER
-        );
-        CREATE TABLE IF NOT EXISTS term_meta (
-            id INTEGER PRIMARY KEY,
-            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
-            term TEXT NOT NULL,
-            mode TEXT NOT NULL,
-            reading TEXT,
-            data TEXT NOT NULL
-        );
-        CREATE TABLE IF NOT EXISTS tags (
-            id INTEGER PRIMARY KEY,
-            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
-            name TEXT NOT NULL,
-            category TEXT,
-            sort_order INTEGER NOT NULL DEFAULT 0,
-            notes TEXT,
-            score INTEGER NOT NULL DEFAULT 0,
-            UNIQUE (dict_id, name)
-        );
-        CREATE TABLE IF NOT EXISTS kanji (
-            id INTEGER PRIMARY KEY,
-            dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
-            kanji TEXT NOT NULL,
-            onyomi TEXT,
-            kunyomi TEXT,
-            tags TEXT,
-            meanings TEXT NOT NULL DEFAULT '[]',
-            stats TEXT NOT NULL DEFAULT '{}'
-        );
-        ",
-    )?;
-
-        conn.execute_batch(
-            "
-        CREATE INDEX IF NOT EXISTS dictionaries_lang ON dictionaries (lang);
-
-        CREATE INDEX IF NOT EXISTS terms_term ON terms (term);
-        CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading);
-        ",
-        )?;
-
-        let mut stmt = conn.prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?;
-        if stmt.exists([])? == false {
-            conn.prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")?
-                .execute([])?;
-            conn.prepare(
-                "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))",
-            )?
-            .execute([])?;
-        }
-
-        Ok(())
-    }
-
-    pub fn import_bundled_zip_file(
-        &mut self,
-        zip_path: PathBuf,
-        lang: &str,
-    ) -> anyhow::Result<YomitanZipImportResult, CJDicError> {
-        self.create_schema()?;
-
-        let f =
-            File::open(&zip_path).with_context(|| format!("opening zip {}", zip_path.display()))?;
-        let mut archive = ZipArchive::new(f).with_context(|| "reading zip archive")?;
-
-        let index_file = match archive.by_name("index.json") {
-            Ok(mut f) => {
-                let mut s = String::new();
-                f.read_to_string(&mut s)?;
-                serde_json::from_str::<Value>(&s)?
-            }
-            Err(e) => {
-                // TODO: proper error handling
-                eprintln!("{}", e);
-                return Ok(YomitanZipImportResult {
-                    exists: false,
-                    load: false,
-                    error: Some(format!("{}", e)),
-                });
-            }
-        };
-
-        let title = index_file
-            .get("title")
-            .and_then(Value::as_str)
-            .unwrap_or("")
-            .to_string();
-        let revision = index_file
-            .get("revision")
-            .and_then(Value::as_str)
-            .unwrap_or("")
-            .to_string();
-
-        let mut conn = self.db.conn.lock().unwrap();
-
-        // Skip if already installed
-        let exists: bool = conn
-            .prepare("SELECT 1 FROM dictionaries WHERE title = ?1 AND revision = ?2")?
-            .exists([title, revision])?;
-        if exists {
-            return Ok(YomitanZipImportResult {
-                exists: true,
-                load: false,
-                error: None,
-            });
-        }
-
-        let tx = conn.transaction()?;
-        tx.execute(
-            "INSERT INTO dictionaries (title, revision, author, url, description, lang) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
-            params![
-                index_file.get("title").and_then(Value::as_str),
-                index_file.get("revision").and_then(Value::as_str),
-                index_file.get("author").and_then(Value::as_str),
-                index_file.get("url").and_then(Value::as_str),
-                index_file.get("description").and_then(Value::as_str),
-                lang,
-            ],
-        )?;
-
-        let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?;
-
-        // prepare statements
-        let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)";
-        let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1";
-        let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)";
-        let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1";
-        let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)";
-        let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1";
-        let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)";
-        let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1";
-        let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)";
-        let insert_meta =
-            "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
-        let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)";
-
-        let mut glossary_cache: HashMap<String, i64> = HashMap::new();
-        let mut def_cache: HashMap<String, i64> = HashMap::new();
-        let mut term_tags_cache: HashMap<String, i64> = HashMap::new();
-        let mut rules_cache: HashMap<String, i64> = HashMap::new();
-
-        // term banks
-        let mut bank_i = 1;
-        loop {
-            let name = format!("term_bank_{}.json", bank_i);
-            match archive.by_name(&name) {
-                Ok(mut f) => {
-                    let mut s = String::new();
-                    f.read_to_string(&mut s)?;
-                    let entries: Vec<Value> = serde_json::from_str(&s)?;
-                    for e in entries {
-                        let term = e.get(0).and_then(Value::as_str).unwrap_or("");
-                        let reading = e.get(1).and_then(Value::as_str).unwrap_or("");
-                        let def_tags = e
-                            .get(2)
-                            .and_then(Value::as_str)
-                            .map(|s| s.trim())
-                            .filter(|s| !s.is_empty())
-                            .map(|s| s.to_string());
-                        let rules = e
-                            .get(3)
-                            .and_then(Value::as_str)
-                            .map(|s| s.trim())
-                            .filter(|s| !s.is_empty())
-                            .map(|s| s.to_string());
-                        let score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
-                        let glossary_val = e.get(5).cloned().unwrap_or(Value::Null);
-                        let sequence = e.get(6).and_then(Value::as_i64);
-                        let term_tags = e
-                            .get(7)
-                            .and_then(Value::as_str)
-                            .map(|s| s.trim())
-                            .filter(|s| !s.is_empty())
-                            .map(|s| s.to_string());
-
-                        let glossary_json = serde_json::to_string(&glossary_val)?;
-                        let hash = blake3_hex(&glossary_json);
-                        let glossary_id = if let Some(&id) = glossary_cache.get(&hash) {
-                            id
-                        } else {
-                            tx.execute(insert_glossary, params![hash, glossary_json])?;
-                            let id: i64 =
-                                tx.query_row(select_glossary, params![hash], |r| r.get(0))?;
-                            glossary_cache.insert(hash.clone(), id);
-                            id
-                        };
-
-                        let def_id = if let Some(s) = def_tags.as_deref() {
-                            intern(&tx, insert_def, select_def, &mut def_cache, s)?
-                        } else {
-                            0
-                        };
-                        let rules_id = if let Some(s) = rules.as_deref() {
-                            intern(&tx, insert_rules, select_rules, &mut rules_cache, s)?
-                        } else {
-                            0
-                        };
-                        let term_tags_id = if let Some(s) = term_tags.as_deref() {
-                            intern(
-                                &tx,
-                                insert_term_tags,
-                                select_term_tags,
-                                &mut term_tags_cache,
-                                s,
-                            )?
-                        } else {
-                            0
-                        };
-
-                        tx.execute(
-                            insert_term,
-                            params![
-                                dict_id,
-                                term,
-                                reading,
-                                if def_id != 0 {
-                                    Some(def_id)
-                                } else {
-                                    Option::<i64>::None
-                                },
-                                if rules_id != 0 {
-                                    Some(rules_id)
-                                } else {
-                                    Option::<i64>::None
-                                },
-                                score,
-                                glossary_id,
-                                sequence,
-                                if term_tags_id != 0 {
-                                    Some(term_tags_id)
-                                } else {
-                                    Option::<i64>::None
-                                },
-                            ],
-                        )?;
-                    }
-                    bank_i += 1;
-                    continue;
-                }
-                Err(_) => break,
-            }
-        }
-
-        // term_meta banks
-        let mut meta_i = 1;
-        loop {
-            let name = format!("term_meta_bank_{}.json", meta_i);
-            match archive.by_name(&name) {
-                Ok(mut f) => {
-                    let mut s = String::new();
-                    f.read_to_string(&mut s)?;
-                    let entries: Vec<Value> = serde_json::from_str(&s)?;
-                    for e in entries {
-                        let term = e.get(0).and_then(Value::as_str).unwrap_or("");
-                        let mode = e.get(1).and_then(Value::as_str).unwrap_or("");
-                        let data = e.get(2).cloned().unwrap_or(Value::Null);
-                        let reading = data
-                            .get("reading")
-                            .and_then(Value::as_str)
-                            .map(|s| s.to_string());
-                        tx.execute(
-                            insert_meta,
-                            params![dict_id, term, mode, reading, serde_json::to_string(&data)?],
-                        )?;
-                    }
-                    meta_i += 1;
-                    continue;
-                }
-                Err(_) => break,
-            }
-        }
-
-        // tag banks
-        let mut tag_i = 1;
-        loop {
-            let name = format!("tag_bank_{}.json", tag_i);
-            match archive.by_name(&name) {
-                Ok(mut f) => {
-                    let mut s = String::new();
-                    f.read_to_string(&mut s)?;
-                    let entries: Vec<Value> = serde_json::from_str(&s)?;
-                    for e in entries {
-                        let name = e.get(0).and_then(Value::as_str).unwrap_or("");
-                        let category = e.get(1).and_then(Value::as_str);
-                        let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0);
-                        let notes = e.get(3).and_then(Value::as_str);
-                        let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
-                        tx.execute(
-                            insert_tag,
-                            params![dict_id, name, category, sort_order, notes, tag_score],
-                        )?;
-                    }
-                    tag_i += 1;
-                    continue;
-                }
-                Err(_) => break,
-            }
-        }
-
-        tx.commit()?;
-
-        conn.execute_batch("VACUUM;")?;
-
-        Ok(YomitanZipImportResult {
-            exists: true,
-            load: true,
-            error: None,
-        })
-    }
-
     pub fn search_yomitan(
         &self,
         q_term: &str,
@@ -420,12 +47,12 @@ impl YomitanDatabase {
             t.sequence,
             COALESCE(tt.tags,  '')  AS term_tags,
             d.title                 AS dict_title
-        FROM terms t
+        FROM yomitan.terms              t
-        JOIN  glossaries    g  ON g.id  = t.glossary_id
+        JOIN yomitan.glossaries         g  ON g.id  = t.glossary_id
-        JOIN  dictionaries  d  ON d.id  = t.dict_id
+        JOIN yomitan.dictionaries       d  ON d.id  = t.dict_id
-        LEFT JOIN def_tag_sets  dt ON dt.id = t.def_tags_id
+        LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
-        LEFT JOIN rule_sets      r ON r.id  = t.rules_id
+        LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
-        LEFT JOIN term_tag_sets tt ON tt.id = t.term_tags_id
+        LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id
         WHERE t.term GLOB ?1 {} t.reading GLOB ?2
         ORDER BY t.score DESC
         LIMIT ?3 OFFSET ?4
@@ -457,19 +84,3 @@ impl YomitanDatabase {
         Ok(out)
     }
 }
-
-fn intern(
-    tx: &Transaction,
-    insert_sql: &str,
-    select_sql: &str,
-    cache: &mut HashMap<String, i64>,
-    value: &str,
-) -> rusqlite::Result<i64> {
-    if let Some(&v) = cache.get(value) {
-        return Ok(v);
-    }
-    tx.execute(insert_sql, [value])?;
-    let id: i64 = tx.query_row(select_sql, [value], |r| r.get(0))?;
-    cache.insert(value.to_string(), id);
-    Ok(id)
-}

src-core/src/db/yomitan_writer.rs

@@ -0,0 +1,404 @@
+use std::{collections::HashMap, fs::File, io::Read, path::PathBuf};
+
+use anyhow::Context;
+use blake3::hash;
+use rusqlite::{Connection, Result, Transaction, params};
+use serde::Serialize;
+use serde_json::Value;
+use zip::ZipArchive;
+
+use crate::CJDicError;
+
+fn blake3_hex(s: &str) -> String {
+    format!("{}", hash(s.as_bytes()))
+}
+
+#[derive(Serialize, Debug)]
+pub struct YomitanZipImportResult {
+    pub exists: bool,
+    pub load: bool,
+    pub error: Option<String>,
+}
+
+pub struct YomitanWriter {
+    conn: Connection,
+}
+
+impl YomitanWriter {
+    pub fn new(conn: Connection) -> Self {
+        Self { conn }
+    }
+
+    pub fn create_schema(&self) -> Result<()> {
+        self.conn.execute_batch(
+        "
+            PRAGMA journal_mode = WAL;
+            PRAGMA foreign_keys = ON;
+            PRAGMA synchronous  = NORMAL;
+            PRAGMA cache_size   = -65536;
+            PRAGMA temp_store   = MEMORY;
+
+            CREATE TABLE IF NOT EXISTS schema_meta (
+                key   TEXT PRIMARY KEY,
+                value TEXT NOT NULL
+            );
+            CREATE TABLE IF NOT EXISTS dictionaries (
+                id           INTEGER PRIMARY KEY,
+                title        TEXT    NOT NULL,
+                revision     TEXT    NOT NULL,
+                author       TEXT,
+                url          TEXT,
+                description  TEXT,
+                sort_order   INTEGER NOT NULL DEFAULT 0,
+                installed_at TEXT    NOT NULL DEFAULT (datetime('now')),
+                lang         TEXT    NOT NULL,
+                UNIQUE (title, revision)
+            );
+            CREATE TABLE IF NOT EXISTS glossaries (
+                id      INTEGER PRIMARY KEY,
+                hash    TEXT NOT NULL UNIQUE,
+                content TEXT NOT NULL
+            );
+            CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
+            CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
+            CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE);
+            CREATE TABLE IF NOT EXISTS terms (
+                id INTEGER PRIMARY KEY,
+                dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+                term TEXT NOT NULL,
+                reading TEXT NOT NULL,
+                def_tags_id INTEGER,
+                rules_id INTEGER,
+                score INTEGER NOT NULL DEFAULT 0,
+                glossary_id INTEGER NOT NULL REFERENCES glossaries(id),
+                sequence INTEGER,
+                term_tags_id INTEGER
+            );
+            CREATE TABLE IF NOT EXISTS term_meta (
+                id INTEGER PRIMARY KEY,
+                dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+                term TEXT NOT NULL,
+                mode TEXT NOT NULL,
+                reading TEXT,
+                data TEXT NOT NULL
+            );
+            CREATE TABLE IF NOT EXISTS tags (
+                id INTEGER PRIMARY KEY,
+                dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+                name TEXT NOT NULL,
+                category TEXT,
+                sort_order INTEGER NOT NULL DEFAULT 0,
+                notes TEXT,
+                score INTEGER NOT NULL DEFAULT 0,
+                UNIQUE (dict_id, name)
+            );
+            CREATE TABLE IF NOT EXISTS kanji (
+                id INTEGER PRIMARY KEY,
+                dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
+                kanji TEXT NOT NULL,
+                onyomi TEXT,
+                kunyomi TEXT,
+                tags TEXT,
+                meanings TEXT NOT NULL DEFAULT '[]',
+                stats TEXT NOT NULL DEFAULT '{}'
+            );
+            ",
+        )?;
+
+        self.conn.execute_batch(
+            "
+        CREATE INDEX IF NOT EXISTS dictionaries_lang ON dictionaries (lang);
+
+        CREATE INDEX IF NOT EXISTS terms_term ON terms (term);
+        CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading);
+        ",
+        )?;
+
+        let mut stmt = self
+            .conn
+            .prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?;
+        if stmt.exists([])? == false {
+            self.conn
+                .prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")?
+                .execute([])?;
+            self.conn
+                .prepare(
+                    "INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))",
+                )?
+                .execute([])?;
+        }
+
+        Ok(())
+    }
+
+    pub fn import_bundled_zip_file(
+        &mut self,
+        zip_file: PathBuf,
+        lang: &str,
+    ) -> anyhow::Result<YomitanZipImportResult, CJDicError> {
+        self.create_schema()?;
+
+        let f =
+            File::open(&zip_file).with_context(|| format!("opening zip {}", zip_file.display()))?;
+        let mut archive = ZipArchive::new(f).with_context(|| "reading zip archive")?;
+
+        let index_file = match archive.by_name("index.json") {
+            Ok(mut f) => {
+                let mut s = String::new();
+                f.read_to_string(&mut s)?;
+                serde_json::from_str::<Value>(&s)?
+            }
+            Err(e) => {
+                // TODO: proper error handling
+                eprintln!("{}", e);
+                return Ok(YomitanZipImportResult {
+                    exists: false,
+                    load: false,
+                    error: Some(format!("{}", e)),
+                });
+            }
+        };
+
+        let title = index_file
+            .get("title")
+            .and_then(Value::as_str)
+            .unwrap_or("")
+            .to_string();
+        let revision = index_file
+            .get("revision")
+            .and_then(Value::as_str)
+            .unwrap_or("")
+            .to_string();
+
+        // Skip if already installed
+        let exists: bool = self
+            .conn
+            .prepare("SELECT 1 FROM dictionaries WHERE title = ?1 AND revision = ?2")?
+            .exists([title, revision])?;
+        if exists {
+            return Ok(YomitanZipImportResult {
+                exists: true,
+                load: false,
+                error: None,
+            });
+        }
+
+        let tx = self.conn.transaction()?;
+        tx.execute(
+            "INSERT INTO dictionaries (title, revision, author, url, description, lang) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+            params![
+                index_file.get("title").and_then(Value::as_str),
+                index_file.get("revision").and_then(Value::as_str),
+                index_file.get("author").and_then(Value::as_str),
+                index_file.get("url").and_then(Value::as_str),
+                index_file.get("description").and_then(Value::as_str),
+                lang,
+            ],
+        )?;
+
+        let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?;
+
+        // prepare statements
+        let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)";
+        let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1";
+        let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)";
+        let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1";
+        let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)";
+        let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1";
+        let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)";
+        let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1";
+        let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)";
+        let insert_meta =
+            "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
+        let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)";
+
+        let mut glossary_cache: HashMap<String, i64> = HashMap::new();
+        let mut def_cache: HashMap<String, i64> = HashMap::new();
+        let mut term_tags_cache: HashMap<String, i64> = HashMap::new();
+        let mut rules_cache: HashMap<String, i64> = HashMap::new();
+
+        // term banks
+        let mut bank_i = 1;
+        loop {
+            let name = format!("term_bank_{}.json", bank_i);
+            match archive.by_name(&name) {
+                Ok(mut f) => {
+                    let mut s = String::new();
+                    f.read_to_string(&mut s)?;
+                    let entries: Vec<Value> = serde_json::from_str(&s)?;
+                    for e in entries {
+                        let term = e.get(0).and_then(Value::as_str).unwrap_or("");
+                        let reading = e.get(1).and_then(Value::as_str).unwrap_or("");
+                        let def_tags = e
+                            .get(2)
+                            .and_then(Value::as_str)
+                            .map(|s| s.trim())
+                            .filter(|s| !s.is_empty())
+                            .map(|s| s.to_string());
+                        let rules = e
+                            .get(3)
+                            .and_then(Value::as_str)
+                            .map(|s| s.trim())
+                            .filter(|s| !s.is_empty())
+                            .map(|s| s.to_string());
+                        let score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
+                        let glossary_val = e.get(5).cloned().unwrap_or(Value::Null);
+                        let sequence = e.get(6).and_then(Value::as_i64);
+                        let term_tags = e
+                            .get(7)
+                            .and_then(Value::as_str)
+                            .map(|s| s.trim())
+                            .filter(|s| !s.is_empty())
+                            .map(|s| s.to_string());
+
+                        let glossary_json = serde_json::to_string(&glossary_val)?;
+                        let hash = blake3_hex(&glossary_json);
+                        let glossary_id = if let Some(&id) = glossary_cache.get(&hash) {
+                            id
+                        } else {
+                            tx.execute(insert_glossary, params![hash, glossary_json])?;
+                            let id: i64 =
+                                tx.query_row(select_glossary, params![hash], |r| r.get(0))?;
+                            glossary_cache.insert(hash.clone(), id);
+                            id
+                        };
+
+                        let def_id = if let Some(s) = def_tags.as_deref() {
+                            intern(&tx, insert_def, select_def, &mut def_cache, s)?
+                        } else {
+                            0
+                        };
+                        let rules_id = if let Some(s) = rules.as_deref() {
+                            intern(&tx, insert_rules, select_rules, &mut rules_cache, s)?
+                        } else {
+                            0
+                        };
+                        let term_tags_id = if let Some(s) = term_tags.as_deref() {
+                            intern(
+                                &tx,
+                                insert_term_tags,
+                                select_term_tags,
+                                &mut term_tags_cache,
+                                s,
+                            )?
+                        } else {
+                            0
+                        };
+
+                        tx.execute(
+                            insert_term,
+                            params![
+                                dict_id,
+                                term,
+                                reading,
+                                if def_id != 0 {
+                                    Some(def_id)
+                                } else {
+                                    Option::<i64>::None
+                                },
+                                if rules_id != 0 {
+                                    Some(rules_id)
+                                } else {
+                                    Option::<i64>::None
+                                },
+                                score,
+                                glossary_id,
+                                sequence,
+                                if term_tags_id != 0 {
+                                    Some(term_tags_id)
+                                } else {
+                                    Option::<i64>::None
+                                },
+                            ],
+                        )?;
+                    }
+                    bank_i += 1;
+                    continue;
+                }
+                Err(_) => break,
+            }
+        }
+
+        // term_meta banks
+        let mut meta_i = 1;
+        loop {
+            let name = format!("term_meta_bank_{}.json", meta_i);
+            match archive.by_name(&name) {
+                Ok(mut f) => {
+                    let mut s = String::new();
+                    f.read_to_string(&mut s)?;
+                    let entries: Vec<Value> = serde_json::from_str(&s)?;
+                    for e in entries {
+                        let term = e.get(0).and_then(Value::as_str).unwrap_or("");
+                        let mode = e.get(1).and_then(Value::as_str).unwrap_or("");
+                        let data = e.get(2).cloned().unwrap_or(Value::Null);
+                        let reading = data
+                            .get("reading")
+                            .and_then(Value::as_str)
+                            .map(|s| s.to_string());
+                        tx.execute(
+                            insert_meta,
+                            params![dict_id, term, mode, reading, serde_json::to_string(&data)?],
+                        )?;
+                    }
+                    meta_i += 1;
+                    continue;
+                }
+                Err(_) => break,
+            }
+        }
+
+        // tag banks
+        let mut tag_i = 1;
+        loop {
+            let name = format!("tag_bank_{}.json", tag_i);
+            match archive.by_name(&name) {
+                Ok(mut f) => {
+                    let mut s = String::new();
+                    f.read_to_string(&mut s)?;
+                    let entries: Vec<Value> = serde_json::from_str(&s)?;
+                    for e in entries {
+                        let name = e.get(0).and_then(Value::as_str).unwrap_or("");
+                        let category = e.get(1).and_then(Value::as_str);
+                        let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0);
+                        let notes = e.get(3).and_then(Value::as_str);
+                        let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
+                        tx.execute(
+                            insert_tag,
+                            params![dict_id, name, category, sort_order, notes, tag_score],
+                        )?;
+                    }
+                    tag_i += 1;
+                    continue;
+                }
+                Err(_) => break,
+            }
+        }
+
+        tx.commit()?;
+
+        self.conn.execute_batch("VACUUM;")?;
+
+        Ok(YomitanZipImportResult {
+            exists: true,
+            load: true,
+            error: None,
+        })
+    }
+}
+
+fn intern(
+    tx: &Transaction,
+    insert_sql: &str,
+    select_sql: &str,
+    cache: &mut HashMap<String, i64>,
+    value: &str,
+) -> rusqlite::Result<i64> {
+    if let Some(&v) = cache.get(value) {
+        return Ok(v);
+    }
+    tx.execute(insert_sql, [value])?;
+    let id: i64 = tx.query_row(select_sql, [value], |r| r.get(0))?;
+    cache.insert(value.to_string(), id);
+    Ok(id)
+}

src-core/src/lib.rs

@@ -1,7 +1,7 @@
-pub mod api;
+mod api;
 pub mod db;
-pub mod error;
+mod error;
-pub mod models;
+mod models;
 pub mod service;
 
 pub use error::CJDicError;

src-core/src/service.rs

@@ -1,14 +1,20 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
-use crate::{db::Database, db::YomitanRow, error::CJDicError, models::Entry};
+use rusqlite::Connection;
+
+use crate::{
+    db::{Database, YomitanRow, YomitanWriter, YomitanZipImportResult},
+    error::CJDicError,
+    models::Entry,
+};
 
 pub struct AppService {
     db: Database,
 }
 
 impl AppService {
-    pub fn new<P: AsRef<Path>>(db_path: P) -> Result<Self, CJDicError> {
+    pub fn new<P: AsRef<Path>>(db_dir: P) -> Result<Self, CJDicError> {
-        let db = Database::new(db_path)?;
+        let db = Database::new(db_dir)?;
         db.init()?;
         Ok(Self { db })
     }
@@ -31,7 +37,22 @@ impl AppService {
     ) -> Result<Vec<YomitanRow>, CJDicError> {
         Ok(self
             .db
-            .yomitan()
+            .yomitan()?
             .search_yomitan(q_term, q_reading, limit, offset)?)
     }
+
+    pub fn get_yomitan_writer(&self) -> Result<YomitanWriter, CJDicError> {
+        let conn = Connection::open(self.db.dir.join("yomitan.db"))?;
+        let writer = YomitanWriter::new(conn);
+        writer.create_schema()?;
+        Ok(writer)
+    }
+
+    pub fn import_yomitan_zip_file(
+        writer: &mut YomitanWriter,
+        zip_file: PathBuf,
+        lang: &str,
+    ) -> Result<YomitanZipImportResult, CJDicError> {
+        Ok(writer.import_bundled_zip_file(zip_file, lang)?)
+    }
 }

src-tauri/src/lib.rs

@@ -14,9 +14,9 @@ pub fn run() {
             let app_dir = app.path().app_config_dir()?;
             create_dir_all(&app_dir)?;
 
-            // let db_path = app_dir.join("cjdic.db");
+            // db_path is app_dir;
-            let db_path = Path::new(r"D:\Projects\cjdic2\test.db");
+            let db_dir = Path::new(r"D:\Projects\cjdic2\tmp\save-db");
-            let service = AppService::new(&db_path)?;
+            let service = AppService::new(&db_dir)?;
             app.manage(service);
 
             Ok(())