polv/cjdic2
1
0 Fork 0
mirror of https://github.com/patarapolw/cjdic2.git synced 2026-11-09 17:39:35 +00:00
Code Releases 2 Activity

move glossary (def text) to a separate db file

Browse Source
This commit is contained in:
polv committed 2026-03-07 08:10:48 +07:00
1 parent 729f6635fc
commit afeb43fc2c
15 files changed
+687 -299
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src-core/src/db/yomitan_writer.rs
+411 -235
View File
@@ -1,16 +1,20 @@
use blake3::hash;
use rusqlite::{Connection, Result, Transaction, params};
use rusqlite::{Connection, OptionalExtension, Result, Transaction, params};
use serde::Serialize;
use serde_json::Value;
use std::{
borrow::Cow,
collections::HashMap,
env::current_dir,
io::{Cursor, Read},
path::{Path, PathBuf},
};
use zip::ZipArchive;
use crate::{CJDicError, Timer};
use crate::{
CJDicError, Timer,
db::{YOMITAN_DBFILE, YOMITAN_GLOSSARY_DBFILE},
};
fn blake3_hex(s: &str) -> String {
format!("{}", hash(s.as_bytes()))
@@ -39,11 +43,11 @@ impl ZipSource for PathBuf {
}
#[derive(Serialize, Debug, Clone)]
pub struct YomitanZipImportProgress {
pub struct YomitanProgress {
pub message: String,
pub current: u32,
pub total: u32,
pub steps: u64,
pub current: usize,
pub total: usize,
pub steps: usize,
}
#[derive(Serialize, Debug)]
@@ -54,11 +58,15 @@ pub struct YomitanZipImportResult {
}
pub struct YomitanWriter {
dir: PathBuf,
conn: Connection,
}
impl YomitanWriter {
pub fn new(conn: Connection) -> Result<Self> {
pub fn new(db_dir: impl AsRef<Path>) -> Result<Self> {
let dir = db_dir.as_ref().to_path_buf();
let conn = Connection::open(dir.join(YOMITAN_DBFILE))?;
conn.execute_batch(
"
PRAGMA journal_mode = WAL;
@@ -69,15 +77,13 @@ impl YomitanWriter {
",
)?;
Ok(Self { conn })
Ok(Self { dir, conn })
}
pub fn cleanup(&self) -> Result<()> {
self.conn.execute_batch("VACUUM")?;
Ok(())
}
pub fn create_schema(&mut self) -> Result<()> {
pub fn create_schema(
&mut self,
progress_callback: impl Fn(YomitanProgress),
) -> Result<(), CJDicError> {
self.conn.execute_batch(
"
CREATE TABLE IF NOT EXISTS schema_meta (
@@ -97,9 +103,9 @@ impl YomitanWriter {
lang TEXT NOT NULL, -- custom
UNIQUE (title, revision)
);
CREATE TABLE IF NOT EXISTS glossaries (
CREATE TABLE IF NOT EXISTS glossaries ( -- not used. moved to a new file
id INTEGER PRIMARY KEY,
hash TEXT NOT NULL UNIQUE,
hash TEXT NOT NULL UNIQUE, -- maybe used in the future for excerpts
content TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
@@ -168,14 +174,89 @@ impl YomitanWriter {
)?;
self.create_materialized_views()?;
self.attach_glossary()?;
let schema_version = "2";
let current_schema_version = {
let mut stmt = self
.conn
.prepare("SELECT value FROM schema_meta WHERE key = 'schema_version'")?;
stmt.query_one([], |r| r.get::<_, String>(0)).optional()?
};
if let Some(v) = current_schema_version {
if v < "2".to_string() {
let total = self
.conn
.query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))?
as usize;
let message = &format!("migrate to {}", YOMITAN_GLOSSARY_DBFILE);
progress_callback(YomitanProgress {
message: message.to_string(),
current: 0,
total,
steps: 0,
});
let _timer = Timer::new(message.to_string());
let tx = self.conn.transaction()?;
{
let mut stmt = tx.prepare("SELECT id, hash, content FROM glossaries")?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})?;
let mut stmt_write = tx.prepare(
"INSERT INTO glossary.glossaries (id, hash, b) VALUES (?1, ?2, ?3)",
)?;
for (i, row) in rows.enumerate() {
let (id, hash, glossary_json) = row?;
stmt_write.execute(params![
id,
hash,
zstd::encode_all(glossary_json.as_bytes(), 3)?
])?;
if i % 1000 == 0 {
progress_callback(YomitanProgress {
message: message.to_string(),
current: i,
total,
steps: 0,
});
}
}
}
tx.commit()?;
self.hash_glossary()?;
let _timer = Timer::new(format!("cleanup for {}", YOMITAN_GLOSSARY_DBFILE));
self.conn.execute_batch(
"
UPDATE glossaries SET content = '[]';
VACUUM;
",
)?;
}
let mut stmt = self
.conn
.prepare("SELECT 1 FROM schema_meta WHERE key = 'schema_version'")?;
if stmt.exists([])? == false {
self.conn
.prepare("INSERT INTO schema_meta (key, value) VALUES ('schema_version', '1')")?
.execute([])?;
.prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")?
.execute(["schema_version", schema_version])?;
} else {
self.conn
.prepare("INSERT INTO schema_meta (key, value) VALUES (?1, ?2)")?
.execute(["schema_version", schema_version])?;
self.conn
.prepare(
"INSERT INTO schema_meta (key, value) VALUES ('created_at', datetime('now'))",
@@ -263,6 +344,86 @@ impl YomitanWriter {
Ok(())
}
fn attach_glossary(&mut self) -> Result<(), CJDicError> {
let dir = self.dir.clone();
let path = if self.dir.is_absolute() {
dir
} else {
current_dir()?.join(dir)
};
let db_file = YOMITAN_GLOSSARY_DBFILE;
let schema_name = "glossary";
let path = path.join(db_file).to_string_lossy().replace(r"\", r"/");
#[cfg(windows)]
let path = if !path.starts_with("/") {
format!("/{}", path)
} else {
path.to_string()
};
let uri = format!("file:{}", path);
self.conn
.execute(&format!("ATTACH DATABASE ?1 AS {}", schema_name), [uri])?;
self.conn.execute_batch(&format!(
r"
PRAGMA {}.mmap_size = 3000000000; -- 3_000_000_000
",
schema_name
))?;
self.create_schema_glossary()?;
Ok(())
}
fn create_schema_glossary(&self) -> Result<()> {
self.conn.execute_batch(&format!(
"
CREATE TABLE IF NOT EXISTS glossary.meta (
key TEXT PRIMARY KEY,
value BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS glossary.glossaries (
id INTEGER PRIMARY KEY,
hash TEXT NOT NULL UNIQUE,
b BLOB NOT NULL
);
"
))?;
Ok(())
}
fn hash_glossary(&self) -> Result<(), CJDicError> {
let max_id: i64 =
self.conn
.query_row("SELECT MAX(id) FROM glossary.glossaries", [], |r| r.get(0))?;
let step = max_id / 1000;
let samples: Vec<Vec<u8>> = self
.conn
.prepare("SELECT b FROM glossary.glossaries WHERE id % ?1 = 0 LIMIT 1000")?
.query_map(params![step], |r| r.get::<_, Vec<u8>>(0))?
.filter_map(|r| r.ok())
.collect();
let dict_size = 112640; // 110KB default
let dictionary = zstd::dict::from_samples(&samples, dict_size)?;
self.conn.execute(
"INSERT OR REPLACE INTO glossary.meta (key, value) VALUES (?1, ?2)",
params!["zstd_dict", dictionary],
)?;
Ok(())
}
pub fn import_dictionary_zip_file<Z, Callback>(
&mut self,
zip_file: &Z,
@@ -271,7 +432,7 @@ impl YomitanWriter {
) -> anyhow::Result<YomitanZipImportResult, CJDicError>
where
Z: ZipSource,
Callback: Fn(YomitanZipImportProgress),
Callback: Fn(YomitanProgress),
{
let bundle_name = zip_file.file_name();
let _timer = Timer::new(format!("Importing {}", bundle_name));
@@ -298,8 +459,8 @@ impl YomitanWriter {
}
};
let mut steps: u64 = 0;
progress_callback(YomitanZipImportProgress {
let mut steps: usize = 0;
progress_callback(YomitanProgress {
message: format!("Opening {}", bundle_name),
current: 0,
total: 0,
@@ -351,247 +512,260 @@ impl YomitanWriter {
}
}
let tx = self.conn.transaction()?;
tx.execute(
"INSERT INTO dictionaries
(title, revision, author, url, description, lang, bundle_name) VALUES
( ?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params![
title,
revision,
index_file.get("author").and_then(Value::as_str),
index_file.get("url").and_then(Value::as_str),
index_file.get("description").and_then(Value::as_str),
lang,
bundle_name,
],
)?;
self.attach_glossary()?;
let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?;
{
let tx = self.conn.transaction()?;
tx.execute(
"INSERT INTO dictionaries
(title, revision, author, url, description, lang, bundle_name) VALUES
( ?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params![
title,
revision,
index_file.get("author").and_then(Value::as_str),
index_file.get("url").and_then(Value::as_str),
index_file.get("description").and_then(Value::as_str),
lang,
bundle_name,
],
)?;
// prepare statements
let insert_glossary = "INSERT OR IGNORE INTO glossaries (hash, content) VALUES (?1, ?2)";
let select_glossary = "SELECT id FROM glossaries WHERE hash = ?1";
let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)";
let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1";
let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)";
let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1";
let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)";
let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1";
let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)";
let insert_meta =
"INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)";
let dict_id: i64 = tx.query_row("SELECT last_insert_rowid()", [], |r| r.get(0))?;
let mut glossary_cache: HashMap<String, i64> = HashMap::new();
let mut def_cache: HashMap<String, i64> = HashMap::new();
let mut term_tags_cache: HashMap<String, i64> = HashMap::new();
let mut rules_cache: HashMap<String, i64> = HashMap::new();
// prepare statements
let insert_glossary =
"INSERT OR IGNORE INTO glossary.glossaries (hash, b) VALUES (?1, ?2)";
let select_glossary = "SELECT id FROM glossary.glossaries WHERE hash = ?1";
let insert_def = "INSERT OR IGNORE INTO def_tag_sets (tags) VALUES (?1)";
let select_def = "SELECT id FROM def_tag_sets WHERE tags = ?1";
let insert_term_tags = "INSERT OR IGNORE INTO term_tag_sets (tags) VALUES (?1)";
let select_term_tags = "SELECT id FROM term_tag_sets WHERE tags = ?1";
let insert_rules = "INSERT OR IGNORE INTO rule_sets (rules) VALUES (?1)";
let select_rules = "SELECT id FROM rule_sets WHERE rules = ?1";
let insert_term = "INSERT INTO terms (dict_id, term, reading, def_tags_id, rules_id, score, glossary_id, sequence, term_tags_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)";
let insert_meta = "INSERT INTO term_meta (dict_id, term, mode, reading, data) VALUES (?1,?2,?3,?4,?5)";
let insert_tag = "INSERT OR IGNORE INTO tags (dict_id, name, category, sort_order, notes, score) VALUES (?1,?2,?3,?4,?5,?6)";
let mut n_bank: u32 = 0;
let mut glossary_cache: HashMap<String, i64> = HashMap::new();
let mut def_cache: HashMap<String, i64> = HashMap::new();
let mut term_tags_cache: HashMap<String, i64> = HashMap::new();
let mut rules_cache: HashMap<String, i64> = HashMap::new();
for filename in archive.file_names() {
let starts_with = "term_bank_";
let mut n_bank: usize = 0;
if filename.starts_with(starts_with) {
let text = filename.to_string();
let start_idx = starts_with.len();
for filename in archive.file_names() {
let starts_with = "term_bank_";
if let Some(stop_idx) = text[start_idx..].find(".") {
let sub = &text[start_idx..(start_idx + stop_idx)];
if let Some(i_bank) = sub.parse::<u32>().ok() {
if i_bank > n_bank {
n_bank = i_bank;
if filename.starts_with(starts_with) {
let text = filename.to_string();
let start_idx = starts_with.len();
if let Some(stop_idx) = text[start_idx..].find(".") {
let sub = &text[start_idx..(start_idx + stop_idx)];
if let Some(i_bank) = sub.parse::<usize>().ok() {
if i_bank > n_bank {
n_bank = i_bank;
}
}
}
}
}
}
// term banks
let mut bank_i = 1;
loop {
let name = format!("term_bank_{}.json", bank_i);
match archive.by_name(&name) {
Ok(mut f) => {
let mut s = String::new();
f.read_to_string(&mut s)?;
let entries: Vec<Value> = serde_json::from_str(&s)?;
// term banks
let mut bank_i = 1;
loop {
let name = format!("term_bank_{}.json", bank_i);
match archive.by_name(&name) {
Ok(mut f) => {
let mut s = String::new();
f.read_to_string(&mut s)?;
let entries: Vec<Value> = serde_json::from_str(&s)?;
steps += entries.len() as u64;
progress_callback(YomitanZipImportProgress {
message: format!("Reading term banks"),
current: bank_i,
total: n_bank,
steps,
});
steps += entries.len();
progress_callback(YomitanProgress {
message: format!("Reading term banks"),
current: bank_i,
total: n_bank,
steps,
});
for e in entries {
let term = e.get(0).and_then(Value::as_str).unwrap_or("");
let reading = e.get(1).and_then(Value::as_str).unwrap_or("");
let def_tags = e
.get(2)
.and_then(Value::as_str)
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let rules = e
.get(3)
.and_then(Value::as_str)
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
let glossary_val = e.get(5).cloned().unwrap_or(Value::Null);
let sequence = e.get(6).and_then(Value::as_i64);
let term_tags = e
.get(7)
.and_then(Value::as_str)
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
for e in entries {
let term = e.get(0).and_then(Value::as_str).unwrap_or("");
let reading = e.get(1).and_then(Value::as_str).unwrap_or("");
let def_tags = e
.get(2)
.and_then(Value::as_str)
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let rules = e
.get(3)
.and_then(Value::as_str)
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
let glossary_val = e.get(5).cloned().unwrap_or(Value::Null);
let sequence = e.get(6).and_then(Value::as_i64);
let term_tags = e
.get(7)
.and_then(Value::as_str)
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let glossary_json = serde_json::to_string(&glossary_val)?;
let hash = blake3_hex(&glossary_json);
let glossary_id = if let Some(&id) = glossary_cache.get(&hash) {
id
} else {
tx.execute(insert_glossary, params![hash, glossary_json])?;
let id: i64 =
tx.query_row(select_glossary, params![hash], |r| r.get(0))?;
glossary_cache.insert(hash.clone(), id);
id
};
let glossary_json = serde_json::to_string(&glossary_val)?;
let hash = blake3_hex(&glossary_json);
let glossary_id = if let Some(&id) = glossary_cache.get(&hash) {
id
} else {
tx.execute(
insert_glossary,
params![hash, zstd::encode_all(glossary_json.as_bytes(), 3)?],
)?;
let id: i64 =
tx.query_row(select_glossary, params![hash], |r| r.get(0))?;
glossary_cache.insert(hash.clone(), id);
id
};
let def_id = if let Some(s) = def_tags.as_deref() {
intern(&tx, insert_def, select_def, &mut def_cache, s)?
} else {
0
};
let rules_id = if let Some(s) = rules.as_deref() {
intern(&tx, insert_rules, select_rules, &mut rules_cache, s)?
} else {
0
};
let term_tags_id = if let Some(s) = term_tags.as_deref() {
intern(
&tx,
insert_term_tags,
select_term_tags,
&mut term_tags_cache,
s,
)?
} else {
0
};
let def_id = if let Some(s) = def_tags.as_deref() {
intern(&tx, insert_def, select_def, &mut def_cache, s)?
} else {
0
};
let rules_id = if let Some(s) = rules.as_deref() {
intern(&tx, insert_rules, select_rules, &mut rules_cache, s)?
} else {
0
};
let term_tags_id = if let Some(s) = term_tags.as_deref() {
intern(
&tx,
insert_term_tags,
select_term_tags,
&mut term_tags_cache,
s,
)?
} else {
0
};
tx.execute(
insert_term,
params![
dict_id,
term,
reading,
if def_id != 0 {
Some(def_id)
} else {
Option::<i64>::None
},
if rules_id != 0 {
Some(rules_id)
} else {
Option::<i64>::None
},
score,
glossary_id,
sequence,
if term_tags_id != 0 {
Some(term_tags_id)
} else {
Option::<i64>::None
},
],
)?;
tx.execute(
insert_term,
params![
dict_id,
term,
reading,
if def_id != 0 {
Some(def_id)
} else {
Option::<i64>::None
},
if rules_id != 0 {
Some(rules_id)
} else {
Option::<i64>::None
},
score,
glossary_id,
sequence,
if term_tags_id != 0 {
Some(term_tags_id)
} else {
Option::<i64>::None
},
],
)?;
}
bank_i += 1;
continue;
}
bank_i += 1;
continue;
Err(_) => break,
}
Err(_) => break,
}
}
steps += 1;
progress_callback(YomitanZipImportProgress {
message: format!("Reading term meta banks"),
current: 0,
total: 0,
steps,
});
steps += 1;
progress_callback(YomitanProgress {
message: format!("Reading term meta banks"),
current: 0,
total: 0,
steps,
});
// term_meta banks
let mut meta_i = 1;
loop {
let name = format!("term_meta_bank_{}.json", meta_i);
match archive.by_name(&name) {
Ok(mut f) => {
let mut s = String::new();
f.read_to_string(&mut s)?;
let entries: Vec<Value> = serde_json::from_str(&s)?;
for e in entries {
let term = e.get(0).and_then(Value::as_str).unwrap_or("");
let mode = e.get(1).and_then(Value::as_str).unwrap_or("");
let data = e.get(2).cloned().unwrap_or(Value::Null);
let reading = data
.get("reading")
.and_then(Value::as_str)
.map(|s| s.to_string());
tx.execute(
insert_meta,
params![dict_id, term, mode, reading, serde_json::to_string(&data)?],
)?;
// term_meta banks
let mut meta_i = 1;
loop {
let name = format!("term_meta_bank_{}.json", meta_i);
match archive.by_name(&name) {
Ok(mut f) => {
let mut s = String::new();
f.read_to_string(&mut s)?;
let entries: Vec<Value> = serde_json::from_str(&s)?;
for e in entries {
let term = e.get(0).and_then(Value::as_str).unwrap_or("");
let mode = e.get(1).and_then(Value::as_str).unwrap_or("");
let data = e.get(2).cloned().unwrap_or(Value::Null);
let reading = data
.get("reading")
.and_then(Value::as_str)
.map(|s| s.to_string());
tx.execute(
insert_meta,
params![
dict_id,
term,
mode,
reading,
serde_json::to_string(&data)?
],
)?;
}
meta_i += 1;
continue;
}
meta_i += 1;
continue;
Err(_) => break,
}
Err(_) => break,
}
}
steps += 1;
progress_callback(YomitanZipImportProgress {
message: format!("Reading tag banks"),
current: 0,
total: 0,
steps,
});
steps += 1;
progress_callback(YomitanProgress {
message: format!("Reading tag banks"),
current: 0,
total: 0,
steps,
});
// tag banks
let mut tag_i = 1;
loop {
let name = format!("tag_bank_{}.json", tag_i);
match archive.by_name(&name) {
Ok(mut f) => {
let mut s = String::new();
f.read_to_string(&mut s)?;
let entries: Vec<Value> = serde_json::from_str(&s)?;
for e in entries {
let name = e.get(0).and_then(Value::as_str).unwrap_or("");
let category = e.get(1).and_then(Value::as_str);
let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0);
let notes = e.get(3).and_then(Value::as_str);
let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
tx.execute(
insert_tag,
params![dict_id, name, category, sort_order, notes, tag_score],
)?;
// tag banks
let mut tag_i = 1;
loop {
let name = format!("tag_bank_{}.json", tag_i);
match archive.by_name(&name) {
Ok(mut f) => {
let mut s = String::new();
f.read_to_string(&mut s)?;
let entries: Vec<Value> = serde_json::from_str(&s)?;
for e in entries {
let name = e.get(0).and_then(Value::as_str).unwrap_or("");
let category = e.get(1).and_then(Value::as_str);
let sort_order = e.get(2).and_then(Value::as_i64).unwrap_or(0);
let notes = e.get(3).and_then(Value::as_str);
let tag_score = e.get(4).and_then(Value::as_i64).unwrap_or(0);
tx.execute(
insert_tag,
params![dict_id, name, category, sort_order, notes, tag_score],
)?;
}
tag_i += 1;
continue;
}
tag_i += 1;
continue;
Err(_) => break,
}
Err(_) => break,
}
}
tx.commit()?;
tx.commit()?;
}
self.conn.execute_batch(
"
@@ -600,6 +774,8 @@ impl YomitanWriter {
",
)?; // < 1 ms
self.hash_glossary()?;
// self.conn.execute_batch("VACUUM;")?; // Not needed, twice time taken than import
Ok(YomitanZipImportResult {