polv/cjdic2
1
0 Fork 0
mirror of https://github.com/patarapolw/cjdic2.git synced 2026-11-09 17:39:35 +00:00
Code Releases 2 Activity

improve search.db

Browse Source
This commit is contained in:
polv committed 2026-03-09 17:00:54 +07:00
1 parent 189853cc23
commit 7b75d6302e
16 files changed
+641 -429
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src-core/src/db/search.rs
+129 -79
View File
@@ -1,54 +1,40 @@
use rusqlite::Connection;
use std::collections::HashSet;
use rusqlite::{Connection, params};
use wana_kana::utils::katakana_to_hiragana;
use crate::{
CJDicError,
db::{DBSCHEMA, DbChild},
CJDicError, Timer, YomitanProgress,
db::{DBFILE, DBSCHEMA, DbChild, yomitan_writer::normalize_term},
tokenizer::Tokenizer,
};
pub(super) fn normalize_reading(s: &str) -> String {
s.chars()
.map(|c| {
if c >= 'ア' && c <= 'ン' {
// Katakana range
if let Some(x) = char::from_u32(c as u32 - 0x60) {
x // Convert to Hiragana
} else {
c // Ensure it's a valid char
}
} else {
c // Return the character as is if it's not in the range
}
})
.collect()
katakana_to_hiragana(s)
}
pub struct SearchDatabase;
// {
// db: Database,
//}
pub struct SearchDatabase<'a> {
conn: &'a mut Connection,
}
// pub struct SearchEntry {
// id: i64,
// term: String,
// reading: String,
// }
impl<'a> SearchDatabase<'a> {
pub(crate) fn new(conn: &'a mut Connection) -> Self {
Self { conn }
}
impl SearchDatabase {
// pub(crate) fn new(db: Database) -> Self {
// Self { db }
// }
pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> {
conn.execute_batch(
pub fn create_schema(&self) -> Result<(), CJDicError> {
self.conn.execute_batch(
&format!("
CREATE TABLE IF NOT EXISTS {0}.terms (
id INTEGER PRIMARY KEY, -- yomitan.terms.id OR yomitan dups OR -user.terms.id
term TEXT NOT NULL, -- normalized
reading TEXT NOT NULL -- normalized
reading TEXT NOT NULL, -- normalized
max_score INTEGER
);
CREATE INDEX IF NOT EXISTS {0}.terms_term ON terms (term);
CREATE INDEX IF NOT EXISTS {0}.terms_reading ON terms (reading);
CREATE INDEX IF NOT EXISTS {0}.terms_max_score ON terms (max_score DESC);
CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 (
term,
@@ -61,55 +47,119 @@ impl SearchDatabase {
Ok(())
}
// pub fn insert_many(
// &self,
// entries: Vec<SearchEntry>,
// tokenizer: Tokenizer,
// ) -> Result<(), CJDicError> {
// let mut conn = self.db.conn.lock()?;
// let tx = conn.transaction()?;
// {
// let mut stmt =
// tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?;
// for it in entries.iter() {
// stmt.execute(params![
// it.id,
// normalize_term(&it.term),
// normalize_reading(&it.reading)
// ])?;
// }
pub fn reset_db(&self) -> Result<(), CJDicError> {
self.conn.execute_batch(&format!(
"
DELETE FROM {0}.terms;
DELETE FROM {0}.terms_ft;
",
DBSCHEMA[DbChild::Search]
))?;
// let exclude_pos =
// HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
Ok(())
}
// let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?;
// for it in entries {
// let mut segmented: Vec<String> = vec![];
// for t in tokenizer.tokenize(it.term) {
// let mut base = t.surface;
// let mut pos = None;
pub fn regenerate_yomitan(
&mut self,
yomitan_schema: &str,
tokenizer: Tokenizer,
progress_callback: impl Fn(YomitanProgress),
) -> Result<(), CJDicError> {
let total = self.conn.query_row(
&format!("SELECT COALESCE(MAX(id), 0) FROM {}.terms", yomitan_schema),
[],
|r| r.get::<_, i64>(0),
)? as usize;
// for (i, dt) in t.details.iter().enumerate() {
// match i {
// 0 => base = dt.to_string(),
// 6 => pos = Some(dt),
// _ => {}
// }
// }
if total == 0 {
return Ok(());
}
// if let Some(p) = pos {
// if exclude_pos.contains(p.as_str()) {
// continue;
// }
// }
// segmented.push(base);
// }
let message = &format!("Generating {}", DBFILE[DbChild::Search]);
progress_callback(YomitanProgress {
message: message.to_string(),
current: 0,
total,
steps: 0,
});
// stmt.execute(params![it.id, segmented.join(" ")])?;
// }
// }
// tx.commit()?;
let _timer = Timer::new(message.to_string());
// Ok(())
// }
let tx = self.conn.transaction()?;
{
let mut stmt = tx.prepare(&format!(
"SELECT t.id, t.term, t.reading, tr.max_score
FROM {0}.terms t
LEFT JOIN {0}.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
",
yomitan_schema
))?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
r.get::<_, Option<i64>>(3)?,
))
})?;
let mut stmt_write = tx.prepare(&format!(
"INSERT INTO {}.terms (id, term, reading, max_score) VALUES (?1, ?2, ?3, ?4)",
DBSCHEMA[DbChild::Search]
))?;
let exclude_pos =
HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
let mut stmt_write_ft = tx.prepare(&format!(
"INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)",
DBSCHEMA[DbChild::Search]
))?;
for (i, row) in rows.enumerate() {
let (id, term, reading, max_score) = row?;
stmt_write.execute(params![
id,
normalize_term(&term),
normalize_reading(&reading),
max_score
])?;
let mut segmented: Vec<String> = vec![];
for t in tokenizer.tokenize(term) {
let mut base = t.surface;
let mut pos = None;
for (i, dt) in t.details.iter().enumerate() {
match i {
0 => base = dt.to_string(),
6 => pos = Some(dt),
_ => {}
}
}
if let Some(p) = pos {
if exclude_pos.contains(p.as_str()) {
continue;
}
}
segmented.push(base);
}
stmt_write_ft.execute(params![id, segmented.join(" ")])?;
if i % 10_000 == 0 {
progress_callback(YomitanProgress {
message: message.to_string(),
current: i,
total,
steps: i,
});
}
}
}
tx.commit()?;
Ok(())
}
}