polv/cjdic2
1
0 Fork 0
mirror of https://github.com/patarapolw/cjdic2.git synced 2026-11-08 17:35:18 +00:00
Code Releases 2 Activity

normalize reading

Browse Source
This commit is contained in:
polv committed 2026-03-08 23:43:32 +07:00
1 parent a97da3fa1d
commit 189853cc23
10 files changed
+248 -56
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src-core/examples/ja_tokenize.rs
+1 -1
View File
@@ -15,7 +15,7 @@ fn main() -> Result<(), anyhow::Error> {
println!(
"{:#?}",
service.ja_tokenize(arg_it.next().unwrap_or(
service.tokenize(arg_it.next().unwrap_or(
"武器･防具≪エルガー武器商会≫の裏の鍛冶場を調べ「錆びた鍵」を入手します ".to_string()
))?
);
src-core/src/db.rs
+2 -2
View File
@@ -40,14 +40,14 @@ pub(crate) static DBFILE: LazyLock<EnumMap<DbChild, &str>> = LazyLock::new(|| {
enum_map! {
DbChild::Yomitan => "yomitan.db",
DbChild::YomitanGlossary => "yomitan-glossary.db",
DbChild::Search => "reading.db"
DbChild::Search => "search.db"
}
});
pub(crate) static DBSCHEMA: LazyLock<EnumMap<DbChild, &str>> = LazyLock::new(|| {
enum_map! {
DbChild::Yomitan => "yomitan",
DbChild::YomitanGlossary => "glossary",
DbChild::Search => "reading"
DbChild::Search => "search"
}
});
src-core/src/db/search.rs
+99 -1
View File
@@ -1,4 +1,10 @@
#[allow(unused)]
use rusqlite::Connection;
use crate::{
CJDicError,
db::{DBSCHEMA, DbChild},
};
pub(super) fn normalize_reading(s: &str) -> String {
s.chars()
.map(|c| {
@@ -15,3 +21,95 @@ pub(super) fn normalize_reading(s: &str) -> String {
})
.collect()
}
pub struct SearchDatabase;
// {
// db: Database,
//}
// pub struct SearchEntry {
// id: i64,
// term: String,
// reading: String,
// }
impl SearchDatabase {
// pub(crate) fn new(db: Database) -> Self {
// Self { db }
// }
pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> {
conn.execute_batch(
&format!("
CREATE TABLE IF NOT EXISTS {0}.terms (
id INTEGER PRIMARY KEY, -- yomitan.terms.id OR yomitan dups OR -user.terms.id
term TEXT NOT NULL, -- normalized
reading TEXT NOT NULL -- normalized
);
CREATE INDEX IF NOT EXISTS {0}.terms_term ON terms (term);
CREATE INDEX IF NOT EXISTS {0}.terms_reading ON terms (reading);
CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 (
term,
content='',
tokenize=unicode61
);
", DBSCHEMA[DbChild::Search]),
)?;
Ok(())
}
// pub fn insert_many(
// &self,
// entries: Vec<SearchEntry>,
// tokenizer: Tokenizer,
// ) -> Result<(), CJDicError> {
// let mut conn = self.db.conn.lock()?;
// let tx = conn.transaction()?;
// {
// let mut stmt =
// tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?;
// for it in entries.iter() {
// stmt.execute(params![
// it.id,
// normalize_term(&it.term),
// normalize_reading(&it.reading)
// ])?;
// }
// let exclude_pos =
// HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
// let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?;
// for it in entries {
// let mut segmented: Vec<String> = vec![];
// for t in tokenizer.tokenize(it.term) {
// let mut base = t.surface;
// let mut pos = None;
// for (i, dt) in t.details.iter().enumerate() {
// match i {
// 0 => base = dt.to_string(),
// 6 => pos = Some(dt),
// _ => {}
// }
// }
// if let Some(p) = pos {
// if exclude_pos.contains(p.as_str()) {
// continue;
// }
// }
// segmented.push(base);
// }
// stmt.execute(params![it.id, segmented.join(" ")])?;
// }
// }
// tx.commit()?;
// Ok(())
// }
}
src-core/src/db/yomitan.rs
+3 -2
View File
@@ -111,10 +111,11 @@ impl YomitanDatabase {
rules_id,
term_tags_id,
sequence
FROM yomitan.terms t
FROM search.terms s
JOIN yomitan.terms t ON t.id = s.id
JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading
JOIN yomitan.dictionaries d ON d.id = t.dict_id
WHERE t.term_norm {eq1} ?1 {and_or} t.reading {eq2} ?2
WHERE s.term {eq1} ?1 {and_or} s.reading {eq2} ?2
ORDER BY tr.max_score DESC, d.sort_order DESC
LIMIT ?3 OFFSET ?4
"
src-core/src/db/yomitan_writer.rs
+62 -4
View File
@@ -13,7 +13,10 @@ use zip::ZipArchive;
use crate::{
CJDicError, Timer,
db::{DBFILE, DBSCHEMA, DbChild},
db::{
DBFILE, DBSCHEMA, DbChild,
search::{SearchDatabase, normalize_reading},
},
};
fn blake3_hex(s: &str) -> String {
@@ -177,7 +180,7 @@ impl YomitanWriter {
self.attach_db(DbChild::YomitanGlossary)?;
self.attach_db(DbChild::Search)?;
let schema_version = "2026-03-07";
let schema_version = "2026-03-08";
let current_schema_version = {
let mut stmt = self
@@ -188,7 +191,8 @@ impl YomitanWriter {
};
if let Some(v) = current_schema_version {
if v < "2".to_string() {
let v = v.as_str();
if v < "2" {
let total = self
.conn
.query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))?
@@ -266,7 +270,60 @@ impl YomitanWriter {
)?;
}
if v < schema_version.to_string() {
if v < "2026-03-08" {
let total = self
.conn
.query_row("SELECT MAX(id) FROM terms", [], |r| r.get::<_, i64>(0))?
as usize;
let message = &format!("Generating {}", DBFILE[DbChild::Search]);
progress_callback(YomitanProgress {
message: message.to_string(),
current: 0,
total,
steps: 0,
});
let _timer = Timer::new(message.to_string());
let tx = self.conn.transaction()?;
{
let mut stmt = tx.prepare("SELECT id, term, reading FROM terms")?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})?;
let mut stmt_write = tx.prepare(
"INSERT INTO search.terms (id, term, reading) VALUES (?1, ?2, ?3)",
)?;
for (i, row) in rows.enumerate() {
let (id, term, reading) = row?;
stmt_write.execute(params![
id,
normalize_term(&term),
normalize_reading(&reading)
])?;
if i % 10_000 == 0 {
progress_callback(YomitanProgress {
message: message.to_string(),
current: i,
total,
steps: i,
});
}
}
}
tx.commit()?;
}
if v < schema_version {
self.conn
.prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")?
.execute(["schema_version", schema_version])?;
@@ -395,6 +452,7 @@ impl YomitanWriter {
match db_child {
DbChild::YomitanGlossary => self.create_schema_glossary()?,
DbChild::Search => SearchDatabase::create_schema(&self.conn)?,
_ => (),
}
src-core/src/error.rs
+11 -1
View File
@@ -6,7 +6,7 @@ use std::{
use rusqlite::Connection;
use serde::Serialize;
use thiserror::Error;
use vibrato::errors::VibratoError;
use vibrato::{Tokenizer, errors::VibratoError};
use zip::result::ZipError;
#[derive(Debug, Error, Serialize)]
@@ -39,6 +39,9 @@ pub enum CJDicError {
#[error("ConnectionMutexGuardError: {0}")]
ConnectionMutexGuardError(String),
#[error("TokenizerMutexGuardError: {0}")]
TokenizerMutexGuardError(String),
#[error("Not found")]
NotFound,
@@ -101,3 +104,10 @@ impl From<PoisonError<MutexGuard<'_, Connection>>> for CJDicError {
CJDicError::ConnectionMutexGuardError(e.to_string())
}
}
impl From<PoisonError<MutexGuard<'_, Tokenizer>>> for CJDicError {
fn from(e: PoisonError<MutexGuard<'_, Tokenizer>>) -> Self {
eprintln!("{e:#}");
CJDicError::TokenizerMutexGuardError(e.to_string())
}
}
src-core/src/lib.rs
+3 -1
View File
@@ -4,11 +4,13 @@ extern crate enum_map;
mod db;
mod error;
mod service;
mod tokenizer;
mod util;
pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource};
pub use error::CJDicError;
pub use service::{AppService, SqlParam, TokenizeSegment};
pub use service::{AppService, SqlParam};
pub use tokenizer::TokenizeSegment;
pub use util::Timer;
pub fn add(left: u64, right: u64) -> u64 {
src-core/src/service.rs
+10 -43
View File
@@ -1,12 +1,10 @@
use std::{
collections::{HashMap, HashSet},
fs::File,
path::Path,
};
use rusqlite::{Connection, params_from_iter};
use serde::{Deserialize, Serialize};
use vibrato::{Dictionary, Tokenizer};
use crate::{
Timer, ZipSource,
@@ -15,6 +13,7 @@ use crate::{
YomitanZipImportResult,
},
error::CJDicError,
tokenizer::{TokenizeSegment, Tokenizer},
};
#[derive(Serialize, Debug, Clone)]
@@ -33,55 +32,23 @@ pub enum SqlParam {
Bool(bool),
}
#[derive(Debug, Serialize)]
pub struct TokenizeSegment {
surface: String,
/// 0 品詞 Part-of-speech
/// 1 品詞細分類1 Part-of-speech subcategory 1
/// 2 品詞細分類2 Part-of-speech subcategory 2
/// 3 品詞細分類3 Part-of-speech subcategory 3
/// 4 活用形 Conjugation form
/// 5 活用型 Conjugation type
/// 6 原形 Base form
/// 7 読み Reading
/// 8 発音 Pronunciation
/// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
details: Vec<String>,
}
pub struct AppService {
db: Database,
ja_tokenizer: Tokenizer,
tokenizer: Tokenizer,
}
impl AppService {
pub fn new<P: AsRef<Path>>(db_dir: P, vibrato_dict: P) -> Result<Self, CJDicError> {
pub fn new(
db_dir: impl AsRef<Path>,
vibrato_dict: impl AsRef<Path>,
) -> Result<Self, CJDicError> {
let db = Database::new(db_dir)?;
let ja_tokenizer = {
let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
let dictionary = Dictionary::read(reader)?;
Tokenizer::new(dictionary)
};
Ok(Self { db, ja_tokenizer })
let tokenizer = Tokenizer::new(vibrato_dict)?;
Ok(Self { db, tokenizer })
}
pub fn ja_tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
let mut worker = self.ja_tokenizer.new_worker();
worker.reset_sentence(text);
worker.tokenize();
let mut output: Vec<TokenizeSegment> = vec![];
for token in worker.token_iter() {
let surface = token.surface().to_string();
let details = token.feature().split(",").map(|s| s.to_string()).collect();
output.push(TokenizeSegment { surface, details });
}
Ok(output)
pub fn tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
Ok(self.tokenizer.tokenize(text))
}
pub fn execute_sql(
src-core/src/tokenizer.rs
+56
View File
@@ -0,0 +1,56 @@
use std::{fs::File, path::Path};
use serde::Serialize;
use crate::CJDicError;
#[derive(Debug, Serialize)]
pub struct TokenizeSegment {
pub surface: String,
/// 0 品詞 Part-of-speech
/// 1 品詞細分類1 Part-of-speech subcategory 1
/// 2 品詞細分類2 Part-of-speech subcategory 2
/// 3 品詞細分類3 Part-of-speech subcategory 3
/// 4 活用形 Conjugation form
/// 5 活用型 Conjugation type
/// 6 原形 Base form
/// 7 読み Reading
/// 8 発音 Pronunciation
/// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
pub details: Vec<String>,
}
pub struct Tokenizer {
ja_tokenizer: vibrato::Tokenizer,
}
impl Tokenizer {
pub fn new(vibrato_dict: impl AsRef<Path>) -> Result<Self, CJDicError> {
let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
let dictionary = vibrato::Dictionary::read(reader)?;
Ok(Self {
ja_tokenizer: vibrato::Tokenizer::new(dictionary),
})
}
pub fn tokenize(&self, text: String) -> Vec<TokenizeSegment> {
self.ja_tokenize(text)
}
fn ja_tokenize(&self, text: String) -> Vec<TokenizeSegment> {
let mut worker = self.ja_tokenizer.new_worker();
worker.reset_sentence(text);
worker.tokenize();
let mut output: Vec<TokenizeSegment> = vec![];
for token in worker.token_iter() {
let surface = token.surface().to_string();
let details = token.feature().split(",").map(|s| s.to_string()).collect();
output.push(TokenizeSegment { surface, details });
}
output
}
}
src-tauri/src/command.rs
+1 -1
View File
@@ -251,5 +251,5 @@ pub async fn tokenize(
text: String,
state: tauri::State<'_, AppService>,
) -> Result<Vec<TokenizeSegment>, CJDicError> {
state.ja_tokenize(text)
state.tokenize(text)
}