diff --git a/.vscode/settings.json b/.vscode/settings.json index b3bd36c..e3b16d1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,10 @@ "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true }, + "[rust]": { + "editor.tabSize": 4, + "editor.formatOnSave": true + }, "files.associations": { "tauri*.conf.json5": "jsonc" }, diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index a1a3df0..ea98543 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -673,8 +673,11 @@ dependencies = [ "tauri-plugin-http", "tauri-plugin-opener", "tauri-plugin-sql", + "thiserror 2.0.18", + "tokio", "vibrato-rkyv", "zip", + "zstd", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index f9a28be..fb6f55d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -29,3 +29,6 @@ tauri-plugin-clipboard-manager = "2" zip = "8.1.0" blake3 = "1.8.3" vibrato-rkyv = "0.7.5" +tokio = { version = "1.50.0", features = ["io-util"] } +thiserror = "2.0.18" +zstd = "0.13.3" diff --git a/src-tauri/src/command.rs b/src-tauri/src/command.rs new file mode 100644 index 0000000..59ac896 --- /dev/null +++ b/src-tauri/src/command.rs @@ -0,0 +1,110 @@ +use serde::Serialize; +use std::{fs::remove_file, path::PathBuf}; +use tauri::{AppHandle, Emitter, Manager}; +use tauri_plugin_http::reqwest::Client; +use tokio::io::AsyncWriteExt; + +use crate::{error::CJDicError, service::AppService}; + +#[tauri::command] +pub async fn tokenize( + text: String, + state: tauri::State<'_, AppService>, +) -> Result, CJDicError> { + state.tokenize(text) +} + +#[tauri::command] +pub async fn normalize( + text: String, + state: tauri::State<'_, AppService>, +) -> Result, CJDicError> { + state.normalize(text) +} + +#[derive(Serialize, Clone)] +struct DownloadProgress { + url: String, + filepath: String, + content_length: u64, + downloaded: u64, +} + +#[tauri::command] +pub async fn download_url(app: AppHandle, url: &str, filepath: &str) -> Result { + download_url_local(url, &app.path().app_data_dir()?, filepath, |progress| { + app.emit("download-url-progress", progress).unwrap(); + }) + .await +} + +async fn download_url_local( + url: &str, + rootdir: &PathBuf, + filepath: &str, + callback: impl Fn(DownloadProgress), +) -> Result { + let client = Client::new(); + let file_pf = rootdir.join(filepath); + + let temp_pf = rootdir.join(format!("{}.dl-tmp", filepath)); + let temp_path = temp_pf.as_path(); + // Check existing partial download size + let existing_size = tokio::fs::metadata(temp_path) + .await + .map(|m| m.len()) + .unwrap_or(0); + + let mut request = client.get(url); + // Request only the remaining bytes + if existing_size > 0 { + request = request.header("Range", format!("bytes={}-", existing_size)); + } + + // Making the asynchronous GET request + let mut response = request.send().await?; + + // Check if the request was successful + if response.status().is_success() { + let content_length = response.content_length().unwrap_or(0); + let mut file = if existing_size > 0 { + tokio::fs::OpenOptions::new() + .append(true) + .open(temp_path) + .await? + } else { + tokio::fs::File::create(&file_pf).await? + }; + + // Downloading the file in chunks + let mut downloaded: u64 = 0; + while let Some(chunk) = response.chunk().await? { + file.write_all(&chunk).await?; + downloaded += chunk.len() as u64; + + callback(DownloadProgress { + url: url.to_owned(), + filepath: filepath.to_owned(), + content_length, + downloaded, + }); + } + + // Verify download completeness + if content_length > 0 && downloaded < content_length { + // Delete incomplete file + drop(file); // ensure file handle is released before deleting + remove_file(&file_pf).ok(); + return Err(CJDicError::Error(format!( + "Incomplete download: expected {} bytes, got {}", + content_length, downloaded + ))); + } + + println!("File downloaded successfully to {:?}", &file_pf); + } else { + eprintln!("Failed to download file: {:?}", response.status()); + } + + Ok(true) +} diff --git a/src-tauri/src/core/mod.rs b/src-tauri/src/core/mod.rs deleted file mode 100644 index d4b5ac0..0000000 --- a/src-tauri/src/core/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -use vibrato_rkyv::Tokenizer; - -pub struct AppService { - tokenizer: Tokenizer, -} \ No newline at end of file diff --git a/src-tauri/src/error.rs b/src-tauri/src/error.rs new file mode 100644 index 0000000..bd1f124 --- /dev/null +++ b/src-tauri/src/error.rs @@ -0,0 +1,90 @@ +use std::{backtrace::Backtrace, string::FromUtf8Error}; + +use serde::Serialize; +use thiserror::Error; +use vibrato_rkyv::errors::VibratoError; +use zip::result::ZipError; + +#[derive(Debug, Error, Serialize)] +#[serde(tag = "kind", content = "message")] +pub enum CJDicError { + #[error("SerializationError: {0}")] + SerializationError(String), + + #[error("IOError: {0}")] + IOError(String), + + #[error("ZipError: {0}")] + ZipError(String), + + #[error("VibratoError: {0}")] + VibratoError(String), + + #[error("FromUtf8Error: {0}")] + FromUtf8Error(String), + + #[error("TauriError: {0}")] + TauriError(String), + + #[error("TauriReqwestError: {0}")] + TauriReqwestError(String), + + #[error("Error: {0}")] + Error(String), +} + +impl From for CJDicError { + fn from(e: serde_json::Error) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::SerializationError(e.to_string()) + } +} + +impl From for CJDicError { + fn from(e: std::io::Error) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::IOError(e.to_string()) + } +} + +impl From for CJDicError { + fn from(e: ZipError) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::ZipError(e.to_string()) + } +} + +impl From for CJDicError { + fn from(e: VibratoError) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::VibratoError(e.to_string()) + } +} + +impl From for CJDicError { + fn from(e: FromUtf8Error) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::FromUtf8Error(e.to_string()) + } +} + +impl From for CJDicError { + fn from(e: tauri::Error) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::TauriError(e.to_string()) + } +} + +impl From for CJDicError { + fn from(e: tauri_plugin_http::reqwest::Error) -> Self { + let bt = Backtrace::capture(); + eprintln!("{e:#}\n{bt}"); + CJDicError::TauriReqwestError(e.to_string()) + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7fad01c..53ee97c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,4 +1,17 @@ -mod core; +use std::fs::{create_dir_all, File}; +use tauri::Manager; +use tauri_plugin_fs::FsExt; + +use crate::command::*; +use crate::service::AppService; +use crate::yomitan::search::YomitanSearch; + +mod tokenizer; +mod yomitan; + +mod command; +mod error; +mod service; // Learn more about Tauri commands at https://tauri.app/develop/calling-rust/ #[tauri::command] @@ -11,10 +24,51 @@ pub fn run() { tauri::Builder::default() .plugin(tauri_plugin_clipboard_manager::init()) .plugin(tauri_plugin_fs::init()) - .plugin(tauri_plugin_sql::Builder::new().build()) + .plugin( + tauri_plugin_sql::Builder::default() + .add_migrations("sqlite:search.db", YomitanSearch::migrations()) + .build(), + ) .plugin(tauri_plugin_http::init()) .plugin(tauri_plugin_opener::init()) - .invoke_handler(tauri::generate_handler![greet]) + .setup(|app| { + let config_dir = app.path().app_config_dir()?; + create_dir_all(&config_dir)?; + + let data_dir = app.path().app_data_dir()?; + create_dir_all(&data_dir.join("yomitan/ja"))?; + + let cache_dir = app.path().app_cache_dir()?; + let vibrato_dic_dir = cache_dir.join("vibrato"); + create_dir_all(&vibrato_dic_dir)?; + + let vibrato_dic_path = vibrato_dic_dir.join("system.dic"); + if !vibrato_dic_path.exists() { + let zst_path = app.path().resolve( + "resources/mecab-ipadic/system.dic.zst", + tauri::path::BaseDirectory::Resource, + )?; + + // Use Tauri's fs abstraction — works on both Android and desktop + let bytes = app.fs().read(&zst_path)?; + + let mut decoder = zstd::Decoder::new(bytes.as_slice())?; + let mut out = File::create(&vibrato_dic_path)?; + std::io::copy(&mut decoder, &mut out)?; + } + + // db_path is at config_dir; + let service = AppService::new(&vibrato_dic_path)?; + app.manage(service); + + Ok(()) + }) + .invoke_handler(tauri::generate_handler![ + greet, + tokenize, + normalize, + download_url + ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } diff --git a/src-tauri/src/service.rs b/src-tauri/src/service.rs new file mode 100644 index 0000000..b8253a5 --- /dev/null +++ b/src-tauri/src/service.rs @@ -0,0 +1,23 @@ +use std::path::Path; + +use crate::{error::CJDicError, tokenizer::ja::Tokenizer}; + +pub struct AppService { + pub ja_tokenizer: Tokenizer, +} + +impl AppService { + pub fn new(vibrato_dic: impl AsRef) -> Result { + Ok(Self { + ja_tokenizer: Tokenizer::new(vibrato_dic)?, + }) + } + + pub fn tokenize(&self, text: String) -> Result, CJDicError> { + self.ja_tokenizer.tokenize(text) + } + + pub fn normalize(&self, text: String) -> Result, CJDicError> { + self.ja_tokenizer.normalize(text) + } +} diff --git a/src-tauri/src/tokenizer/ja.rs b/src-tauri/src/tokenizer/ja.rs new file mode 100644 index 0000000..bf10b01 --- /dev/null +++ b/src-tauri/src/tokenizer/ja.rs @@ -0,0 +1,95 @@ +use std::{collections::HashSet, fs::File, io::BufReader, path::Path}; + +use serde::Serialize; + +use crate::error::CJDicError; + +#[derive(Debug, Serialize)] +pub struct TokenizeSegment { + pub surface: String, + /// ```txt + /// 0 品詞 Part-of-speech + /// 1 品詞細分類1 Part-of-speech subcategory 1 + /// 2 品詞細分類2 Part-of-speech subcategory 2 + /// 3 品詞細分類3 Part-of-speech subcategory 3 + /// 4 活用形 Conjugation form + /// 5 活用型 Conjugation type + /// 6 原形 Base form + /// 7 読み Reading + /// 8 発音 Pronunciation + /// ``` + /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html + pub details: Vec, +} + +#[derive(Clone)] +pub struct Tokenizer { + tokenizer: vibrato_rkyv::Tokenizer, + exclude_pos: HashSet, +} + +impl Tokenizer { + pub fn new(vibrato_dic: impl AsRef) -> Result { + let reader = BufReader::new(File::open(vibrato_dic)?); + let dictionary = vibrato_rkyv::Dictionary::read(reader)?; + Ok(Self { + tokenizer: vibrato_rkyv::Tokenizer::new(dictionary), + exclude_pos: HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string())), + }) + } + + pub fn tokenize(&self, text: String) -> Result, CJDicError> { + let mut segmented: Vec = vec![]; + for t in self.ja_tokenize(text)? { + segmented.push(t.surface); + } + Ok(segmented) + } + + fn ja_tokenize(&self, text: String) -> Result, CJDicError> { + let mut worker = self.tokenizer.new_worker(); + worker.reset_sentence(text); + worker.tokenize(); + + let mut output: Vec = vec![]; + + for token in worker.token_iter() { + let surface = token.surface().to_string(); + let details = token.feature().split(",").map(|s| s.to_string()).collect(); + + output.push(TokenizeSegment { surface, details }); + } + + Ok(output) + } + + pub fn normalize(&self, text: String) -> Result, CJDicError> { + let mut segmented: Vec = vec![]; + for t in self.ja_tokenize(text)? { + let mut surface = t.surface; + let mut pos = None; + + for (i, dt) in t.details.iter().enumerate() { + match i { + 0 => pos = Some(dt), + 6 => { + let base = dt.to_string(); + if base != "*" { + surface = base + } + } + _ => {} + } + } + + if let Some(p) = pos { + if self.exclude_pos.contains(p.as_str()) { + continue; + } + } + segmented.push(surface); + } + + Ok(segmented) + } +} diff --git a/src-tauri/src/tokenizer/mod.rs b/src-tauri/src/tokenizer/mod.rs new file mode 100644 index 0000000..8315321 --- /dev/null +++ b/src-tauri/src/tokenizer/mod.rs @@ -0,0 +1 @@ +pub mod ja; diff --git a/src-tauri/src/yomitan/mod.rs b/src-tauri/src/yomitan/mod.rs new file mode 100644 index 0000000..20acaae --- /dev/null +++ b/src-tauri/src/yomitan/mod.rs @@ -0,0 +1,2 @@ +pub mod search; +pub mod unzip; diff --git a/src-tauri/src/yomitan/search.rs b/src-tauri/src/yomitan/search.rs new file mode 100644 index 0000000..638095e --- /dev/null +++ b/src-tauri/src/yomitan/search.rs @@ -0,0 +1,75 @@ +use tauri_plugin_sql::{Migration, MigrationKind}; + +pub struct YomitanSearch; + +impl YomitanSearch { + pub fn migrations() -> Vec { + vec![ + Migration { + version: 1, + description: "create yomitan dict-version tables", + sql: " + CREATE TABLE dict ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + revision TEXT NOT NULL, + author TEXT, + url TEXT, + description TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + installed_at TEXT NOT NULL DEFAULT (datetime('now')), + bundle_name TEXT, -- custom + lang TEXT NOT NULL, -- custom + asset_count INTEGER, -- custom + source TEXT, -- custom + UNIQUE (title, revision) + ); + ", + kind: MigrationKind::Up, + }, + Migration { + version: 1, + description: "create yomitan ja query tables", + sql: " + CREATE TABLE ja_search ( + id INTEGER NOT NULL, -- yomitan.terms.id OR yomitan dups OR -user.terms.id + dict_id INTEGER NOT NULL REFERENCES dict(id) ON DELETE CASCADE, + term TEXT NOT NULL, -- normalized + term_seg TEXT, -- segmented and filtered for fillers + reading TEXT NOT NULL, -- normalized + frequency INTEGER, + PRIMARY KEY (id, dict_id) + ); + + CREATE INDEX ja_search_term ON ja_search (term); + CREATE INDEX ja_search_reading ON ja_search (reading); + CREATE INDEX ja_search_frequency ON ja_search (frequency DESC); + + CREATE VIRTUAL TABLE IF NOT EXISTS ja_search_ft USING fts5 ( + term_seg, + content='ja_search', + content_rowid='rowid', + tokenize=unicode61 + ); + + CREATE TRIGGER t_ja_search_ft_after_insert + AFTER INSERT ON ja_search BEGIN + INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg); + END; + + CREATE TRIGGER t_ja_search_ft_after_delete + AFTER DELETE ON ja_search BEGIN + INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg); + END; + + CREATE TRIGGER t_ja_search_ft_after_update + AFTER UPDATE ON ja_search BEGIN + INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg); + INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg); + END; + ", + kind: MigrationKind::Up + } + ] + } +} diff --git a/src-tauri/src/yomitan/unzip.rs b/src-tauri/src/yomitan/unzip.rs new file mode 100644 index 0000000..e69de29