polv/cjdic2
1
0 Fork 0
mirror of https://github.com/patarapolw/cjdic2.git synced 2026-11-09 17:39:35 +00:00
Code Releases 2 Activity

improve search.db

Browse Source
This commit is contained in:
polv committed 2026-03-09 17:00:54 +07:00
1 parent 189853cc23
commit 7b75d6302e
16 files changed
+641 -429
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src-core/Cargo.toml
+2 -1
View File
@@ -14,6 +14,7 @@ thiserror = "2.0.18"
zip = "8.1.0"
blake3 = "1.8.3"
anyhow = "1.0.102"
vibrato = "0.5.2"
vibrato-rkyv = "0.7.5"
zstd = "0.13.3"
enum-map = "2.7.3"
wana_kana = "4.0.0"
src-core/src/db/search.rs
+129 -79
View File
@@ -1,54 +1,40 @@
use rusqlite::Connection;
use std::collections::HashSet;
use rusqlite::{Connection, params};
use wana_kana::utils::katakana_to_hiragana;
use crate::{
CJDicError,
db::{DBSCHEMA, DbChild},
CJDicError, Timer, YomitanProgress,
db::{DBFILE, DBSCHEMA, DbChild, yomitan_writer::normalize_term},
tokenizer::Tokenizer,
};
pub(super) fn normalize_reading(s: &str) -> String {
s.chars()
.map(|c| {
if c >= 'ア' && c <= 'ン' {
// Katakana range
if let Some(x) = char::from_u32(c as u32 - 0x60) {
x // Convert to Hiragana
} else {
c // Ensure it's a valid char
}
} else {
c // Return the character as is if it's not in the range
}
})
.collect()
katakana_to_hiragana(s)
}
pub struct SearchDatabase;
// {
// db: Database,
//}
pub struct SearchDatabase<'a> {
conn: &'a mut Connection,
}
// pub struct SearchEntry {
// id: i64,
// term: String,
// reading: String,
// }
impl<'a> SearchDatabase<'a> {
pub(crate) fn new(conn: &'a mut Connection) -> Self {
Self { conn }
}
impl SearchDatabase {
// pub(crate) fn new(db: Database) -> Self {
// Self { db }
// }
pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> {
conn.execute_batch(
pub fn create_schema(&self) -> Result<(), CJDicError> {
self.conn.execute_batch(
&format!("
CREATE TABLE IF NOT EXISTS {0}.terms (
id INTEGER PRIMARY KEY, -- yomitan.terms.id OR yomitan dups OR -user.terms.id
term TEXT NOT NULL, -- normalized
reading TEXT NOT NULL -- normalized
reading TEXT NOT NULL, -- normalized
max_score INTEGER
);
CREATE INDEX IF NOT EXISTS {0}.terms_term ON terms (term);
CREATE INDEX IF NOT EXISTS {0}.terms_reading ON terms (reading);
CREATE INDEX IF NOT EXISTS {0}.terms_max_score ON terms (max_score DESC);
CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 (
term,
@@ -61,55 +47,119 @@ impl SearchDatabase {
Ok(())
}
// pub fn insert_many(
// &self,
// entries: Vec<SearchEntry>,
// tokenizer: Tokenizer,
// ) -> Result<(), CJDicError> {
// let mut conn = self.db.conn.lock()?;
// let tx = conn.transaction()?;
// {
// let mut stmt =
// tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?;
// for it in entries.iter() {
// stmt.execute(params![
// it.id,
// normalize_term(&it.term),
// normalize_reading(&it.reading)
// ])?;
// }
pub fn reset_db(&self) -> Result<(), CJDicError> {
self.conn.execute_batch(&format!(
"
DELETE FROM {0}.terms;
DELETE FROM {0}.terms_ft;
",
DBSCHEMA[DbChild::Search]
))?;
// let exclude_pos =
// HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
Ok(())
}
// let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?;
// for it in entries {
// let mut segmented: Vec<String> = vec![];
// for t in tokenizer.tokenize(it.term) {
// let mut base = t.surface;
// let mut pos = None;
pub fn regenerate_yomitan(
&mut self,
yomitan_schema: &str,
tokenizer: Tokenizer,
progress_callback: impl Fn(YomitanProgress),
) -> Result<(), CJDicError> {
let total = self.conn.query_row(
&format!("SELECT COALESCE(MAX(id), 0) FROM {}.terms", yomitan_schema),
[],
|r| r.get::<_, i64>(0),
)? as usize;
// for (i, dt) in t.details.iter().enumerate() {
// match i {
// 0 => base = dt.to_string(),
// 6 => pos = Some(dt),
// _ => {}
// }
// }
if total == 0 {
return Ok(());
}
// if let Some(p) = pos {
// if exclude_pos.contains(p.as_str()) {
// continue;
// }
// }
// segmented.push(base);
// }
let message = &format!("Generating {}", DBFILE[DbChild::Search]);
progress_callback(YomitanProgress {
message: message.to_string(),
current: 0,
total,
steps: 0,
});
// stmt.execute(params![it.id, segmented.join(" ")])?;
// }
// }
// tx.commit()?;
let _timer = Timer::new(message.to_string());
// Ok(())
// }
let tx = self.conn.transaction()?;
{
let mut stmt = tx.prepare(&format!(
"SELECT t.id, t.term, t.reading, tr.max_score
FROM {0}.terms t
LEFT JOIN {0}.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
",
yomitan_schema
))?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
r.get::<_, Option<i64>>(3)?,
))
})?;
let mut stmt_write = tx.prepare(&format!(
"INSERT INTO {}.terms (id, term, reading, max_score) VALUES (?1, ?2, ?3, ?4)",
DBSCHEMA[DbChild::Search]
))?;
let exclude_pos =
HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
let mut stmt_write_ft = tx.prepare(&format!(
"INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)",
DBSCHEMA[DbChild::Search]
))?;
for (i, row) in rows.enumerate() {
let (id, term, reading, max_score) = row?;
stmt_write.execute(params![
id,
normalize_term(&term),
normalize_reading(&reading),
max_score
])?;
let mut segmented: Vec<String> = vec![];
for t in tokenizer.tokenize(term) {
let mut base = t.surface;
let mut pos = None;
for (i, dt) in t.details.iter().enumerate() {
match i {
0 => base = dt.to_string(),
6 => pos = Some(dt),
_ => {}
}
}
if let Some(p) = pos {
if exclude_pos.contains(p.as_str()) {
continue;
}
}
segmented.push(base);
}
stmt_write_ft.execute(params![id, segmented.join(" ")])?;
if i % 10_000 == 0 {
progress_callback(YomitanProgress {
message: message.to_string(),
current: i,
total,
steps: i,
});
}
}
}
tx.commit()?;
Ok(())
}
}
src-core/src/db/yomitan.rs
+25 -52
View File
@@ -50,7 +50,16 @@ impl YomitanDatabase {
limit: u32,
offset: u32,
) -> Result<Vec<YomitanRow>, CJDicError> {
let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading));
let q_label = if q_term == q_reading {
q_term
} else {
&format!("{} ({})", q_term, q_reading)
};
let _timer = Timer::new(format!(
"search_yomitan: {} offset={} limit={}",
q_label, offset, limit
));
// must be run before self.db.conn.lock()
let mut decompressor = self.decompressor()?;
@@ -80,53 +89,15 @@ impl YomitanDatabase {
let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
let conn = self.db.conn.lock()?;
conn.execute_batch(
"
CREATE TEMP TABLE IF NOT EXISTS yomitan_lookup_keys (
term,
reading,
score,
title,
glossary_id,
def_tags_id,
rules_id,
term_tags_id,
sequence
);
DELETE FROM yomitan_lookup_keys;
",
)?;
// let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading));
let sql_string = format!(
"
INSERT INTO yomitan_lookup_keys
SELECT
t.term,
t.reading,
t.score,
d.title,
t.glossary_id,
t.def_tags_id,
rules_id,
term_tags_id,
sequence
FROM search.terms s
JOIN yomitan.terms t ON t.id = s.id
JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
JOIN yomitan.dictionaries d ON d.id = t.dict_id
WHERE s.term {eq1} ?1 {and_or} s.reading {eq2} ?2
ORDER BY tr.max_score DESC, d.sort_order DESC
LIMIT ?3 OFFSET ?4
"
);
// println!("{}", sql_string);
// still the most time bottleneck step, several seconds
conn.execute(&sql_string, params![q_term_norm, q_reading, limit, offset])?;
let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading));
let mut stmt = conn.prepare(
let mut stmt = conn.prepare(&format!(
"
WITH s AS (
SELECT * FROM search.terms
WHERE term {eq1} ?1 {and_or} reading {eq2} ?2
ORDER BY max_score DESC
LIMIT ?3 OFFSET ?4
)
SELECT
t.term,
t.reading,
@@ -136,15 +107,17 @@ impl YomitanDatabase {
COALESCE(g.b, '[]') AS glossary,
t.sequence,
COALESCE(tt.tags, '') AS term_tags,
t.title AS dict_title
FROM yomitan_lookup_keys t
LEFT JOIN glossary.glossaries g ON g.id = t.glossary_id
d.title AS dict_title
FROM s
JOIN yomitan.terms t ON t.id = s.id
LEFT JOIN yomitan.dictionaries d ON d.id = t.dict_id
LEFT JOIN yomitan.def_tag_sets dt ON dt.id = t.def_tags_id
LEFT JOIN yomitan.rule_sets r ON r.id = t.rules_id
LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id
",
)?;
let rows = stmt.query_map(params![], |r| {
LEFT JOIN glossary.glossaries g ON g.id = t.glossary_id
"
))?;
let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| {
Ok(YomitanRow {
term: r.get(0)?,
reading: r.get(1)?,
src-core/src/db/yomitan_writer.rs
+62 -127
View File
@@ -13,10 +13,8 @@ use zip::ZipArchive;
use crate::{
CJDicError, Timer,
db::{
DBFILE, DBSCHEMA, DbChild,
search::{SearchDatabase, normalize_reading},
},
db::{DBFILE, DBSCHEMA, DbChild, search::SearchDatabase},
tokenizer::Tokenizer,
};
fn blake3_hex(s: &str) -> String {
@@ -85,6 +83,7 @@ impl YomitanWriter {
pub fn create_schema(
&mut self,
tokenizer: Tokenizer,
progress_callback: impl Fn(YomitanProgress),
) -> Result<(), CJDicError> {
self.conn.execute_batch(
@@ -107,53 +106,52 @@ impl YomitanWriter {
UNIQUE (title, revision)
);
CREATE TABLE IF NOT EXISTS glossaries ( -- not used. moved to a new file
id INTEGER PRIMARY KEY,
hash TEXT NOT NULL UNIQUE, -- maybe used in the future for excerpts
content TEXT NOT NULL
id INTEGER PRIMARY KEY,
hash TEXT NOT NULL UNIQUE, -- maybe used in the future for excerpts
content TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE);
CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE);
CREATE TABLE IF NOT EXISTS terms (
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
term TEXT NOT NULL,
reading TEXT NOT NULL,
def_tags_id INTEGER,
rules_id INTEGER,
score INTEGER NOT NULL DEFAULT 0,
glossary_id INTEGER NOT NULL REFERENCES glossaries(id),
sequence INTEGER,
term_tags_id INTEGER,
term_norm TEXT -- custom
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
term TEXT NOT NULL,
reading TEXT NOT NULL,
def_tags_id INTEGER,
rules_id INTEGER,
score INTEGER NOT NULL DEFAULT 0,
glossary_id INTEGER NOT NULL REFERENCES glossaries(id),
sequence INTEGER,
term_tags_id INTEGER
);
CREATE TABLE IF NOT EXISTS term_meta (
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
term TEXT NOT NULL,
mode TEXT NOT NULL,
reading TEXT,
data TEXT NOT NULL
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
term TEXT NOT NULL,
mode TEXT NOT NULL,
reading TEXT,
data TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
name TEXT NOT NULL,
category TEXT,
sort_order INTEGER NOT NULL DEFAULT 0,
notes TEXT,
score INTEGER NOT NULL DEFAULT 0,
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
name TEXT NOT NULL,
category TEXT,
sort_order INTEGER NOT NULL DEFAULT 0,
notes TEXT,
score INTEGER NOT NULL DEFAULT 0,
UNIQUE (dict_id, name)
);
CREATE TABLE IF NOT EXISTS kanji (
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
kanji TEXT NOT NULL,
onyomi TEXT,
kunyomi TEXT,
tags TEXT,
meanings TEXT NOT NULL DEFAULT '[]',
stats TEXT NOT NULL DEFAULT '{}'
id INTEGER PRIMARY KEY,
dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE,
kanji TEXT NOT NULL,
onyomi TEXT,
kunyomi TEXT,
tags TEXT,
meanings TEXT NOT NULL DEFAULT '[]',
stats TEXT NOT NULL DEFAULT '{}'
);
",
)?;
@@ -164,8 +162,6 @@ impl YomitanWriter {
CREATE INDEX IF NOT EXISTS dictionaries_lang ON dictionaries (lang);
CREATE INDEX IF NOT EXISTS terms_dict_id ON terms (dict_id);
CREATE INDEX IF NOT EXISTS terms_term_norm ON terms (term_norm);
CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading);
CREATE INDEX IF NOT EXISTS terms_term_reading_score ON terms(term, reading, score DESC);
CREATE INDEX IF NOT EXISTS term_meta_dict_id ON term_meta (dict_id);
@@ -180,8 +176,26 @@ impl YomitanWriter {
self.attach_db(DbChild::YomitanGlossary)?;
self.attach_db(DbChild::Search)?;
let schema_version = "2026-03-08";
let search_max_id = {
let mut stmt = self.conn.prepare(&format!(
"SELECT COALESCE(MAX(id), 0) FROM {}.terms",
DBSCHEMA[DbChild::Search]
))?;
let mut rows = stmt.query([])?;
if let Some(r) = rows.next()? {
r.get(0)?
} else {
0
}
};
if search_max_id == 0 {
let mut search_db = SearchDatabase::new(&mut self.conn);
search_db.reset_db()?;
search_db.regenerate_yomitan("main", tokenizer.clone(), &progress_callback)?;
}
let schema_version = "2026-03-08";
let current_schema_version = {
let mut stmt = self
.conn
@@ -189,14 +203,14 @@ impl YomitanWriter {
stmt.query_one([], |r| r.get::<_, String>(0)).optional()?
};
if let Some(v) = current_schema_version {
let v = v.as_str();
if v < "2" {
let total = self
.conn
.query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))?
as usize;
let total =
self.conn
.query_row("SELECT COALESCE(MAX(id), 0) FROM glossaries", [], |r| {
r.get::<_, i64>(0)
})? as usize;
let message = &format!("Migrate to {}", DBFILE[DbChild::YomitanGlossary]);
progress_callback(YomitanProgress {
@@ -270,59 +284,6 @@ impl YomitanWriter {
)?;
}
if v < "2026-03-08" {
let total = self
.conn
.query_row("SELECT MAX(id) FROM terms", [], |r| r.get::<_, i64>(0))?
as usize;
let message = &format!("Generating {}", DBFILE[DbChild::Search]);
progress_callback(YomitanProgress {
message: message.to_string(),
current: 0,
total,
steps: 0,
});
let _timer = Timer::new(message.to_string());
let tx = self.conn.transaction()?;
{
let mut stmt = tx.prepare("SELECT id, term, reading FROM terms")?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})?;
let mut stmt_write = tx.prepare(
"INSERT INTO search.terms (id, term, reading) VALUES (?1, ?2, ?3)",
)?;
for (i, row) in rows.enumerate() {
let (id, term, reading) = row?;
stmt_write.execute(params![
id,
normalize_term(&term),
normalize_reading(&reading)
])?;
if i % 10_000 == 0 {
progress_callback(YomitanProgress {
message: message.to_string(),
current: i,
total,
steps: i,
});
}
}
}
tx.commit()?;
}
if v < schema_version {
self.conn
.prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")?
@@ -390,32 +351,6 @@ impl YomitanWriter {
}
}
let tx = self.conn.transaction()?;
{
let mut stmt = tx.prepare("SELECT rowid, term FROM terms WHERE term_norm IS NULL")?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
})?;
let mut update_stmt = tx.prepare("UPDATE terms SET term_norm = ?1 WHERE rowid = ?2")?;
for row in rows {
let (rowid, term) = row?;
let norm = normalize_term(&term); // your Unicode normalizer
update_stmt.execute((&norm, rowid))?;
}
}
tx.commit()?;
// 5 sec
self.conn.execute_batch(
"
CREATE INDEX IF NOT EXISTS terms_term_norm ON terms (term_norm);
",
)?;
// 1 sec
Ok(())
}
@@ -452,7 +387,7 @@ impl YomitanWriter {
match db_child {
DbChild::YomitanGlossary => self.create_schema_glossary()?,
DbChild::Search => SearchDatabase::create_schema(&self.conn)?,
DbChild::Search => SearchDatabase::new(&mut self.conn).create_schema()?,
_ => (),
}
src-core/src/error.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use std::{
use rusqlite::Connection;
use serde::Serialize;
use thiserror::Error;
use vibrato::{Tokenizer, errors::VibratoError};
use vibrato_rkyv::{Tokenizer, errors::VibratoError};
use zip::result::ZipError;
#[derive(Debug, Error, Serialize)]
src-core/src/service.rs
+2 -2
View File
@@ -121,7 +121,7 @@ impl AppService {
progress_callback: impl Fn(YomitanProgress),
) -> Result<YomitanWriter, CJDicError> {
let mut writer = YomitanWriter::new(self.db.dir.clone())?;
writer.create_schema(&progress_callback)?;
writer.create_schema(self.tokenizer.clone(), &progress_callback)?;
Ok(writer)
}
@@ -187,7 +187,7 @@ impl AppService {
{
let mut writer = YomitanWriter::new(self.db.dir.clone())?;
writer.create_schema(&import_callback)?;
writer.create_schema(self.tokenizer.clone(), &import_callback)?;
}
{
src-core/src/tokenizer.rs
+7 -5
View File
@@ -1,6 +1,7 @@
use std::{fs::File, path::Path};
use std::path::Path;
use serde::Serialize;
use vibrato_rkyv::CacheStrategy;
use crate::CJDicError;
@@ -20,16 +21,17 @@ pub struct TokenizeSegment {
pub details: Vec<String>,
}
#[derive(Clone)]
pub struct Tokenizer {
ja_tokenizer: vibrato::Tokenizer,
ja_tokenizer: vibrato_rkyv::Tokenizer,
}
impl Tokenizer {
pub fn new(vibrato_dict: impl AsRef<Path>) -> Result<Self, CJDicError> {
let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
let dictionary = vibrato::Dictionary::read(reader)?;
let dictionary =
vibrato_rkyv::Dictionary::from_zstd(vibrato_dict, CacheStrategy::GlobalCache)?;
Ok(Self {
ja_tokenizer: vibrato::Tokenizer::new(dictionary),
ja_tokenizer: vibrato_rkyv::Tokenizer::new(dictionary),
})
}
src-core/src/util.rs
-14
View File
@@ -1,13 +1,10 @@
#[cfg(debug_assertions)]
use std::time::Instant;
#[cfg(debug_assertions)]
pub struct Timer {
start: Instant,
label: String,
}
#[cfg(debug_assertions)]
impl Timer {
pub fn new(label: String) -> Self {
Self {
@@ -17,19 +14,8 @@ impl Timer {
}
}
#[cfg(debug_assertions)]
impl Drop for Timer {
fn drop(&mut self) {
println!("[{}] {:.2?}", self.label, self.start.elapsed())
}
}
#[cfg(not(debug_assertions))]
pub struct Timer;
#[cfg(not(debug_assertions))]
impl Timer {
pub fn new(_label: String) -> Self {
Self
}
}