backend

TODO: sql loading/parsing from frontend

.vscode/settings.json

@@ -12,6 +12,10 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true
   },
+  "[rust]": {
+    "editor.tabSize": 4,
+    "editor.formatOnSave": true
+  },
   "files.associations": {
     "tauri*.conf.json5": "jsonc"
   },

src-tauri/Cargo.lock

@@ -673,8 +673,11 @@ dependencies = [
  "tauri-plugin-http",
  "tauri-plugin-opener",
  "tauri-plugin-sql",
+ "thiserror 2.0.18",
+ "tokio",
  "vibrato-rkyv",
  "zip",
+ "zstd",
 ]
 
 [[package]]

src-tauri/Cargo.toml

@@ -29,3 +29,6 @@ tauri-plugin-clipboard-manager = "2"
 zip = "8.1.0"
 blake3 = "1.8.3"
 vibrato-rkyv = "0.7.5"
+tokio = { version = "1.50.0", features = ["io-util"] }
+thiserror = "2.0.18"
+zstd = "0.13.3"

src-tauri/src/command.rs

@@ -0,0 +1,110 @@
+use serde::Serialize;
+use std::{fs::remove_file, path::PathBuf};
+use tauri::{AppHandle, Emitter, Manager};
+use tauri_plugin_http::reqwest::Client;
+use tokio::io::AsyncWriteExt;
+
+use crate::{error::CJDicError, service::AppService};
+
+#[tauri::command]
+pub async fn tokenize(
+    text: String,
+    state: tauri::State<'_, AppService>,
+) -> Result<Vec<String>, CJDicError> {
+    state.tokenize(text)
+}
+
+#[tauri::command]
+pub async fn normalize(
+    text: String,
+    state: tauri::State<'_, AppService>,
+) -> Result<Vec<String>, CJDicError> {
+    state.normalize(text)
+}
+
+#[derive(Serialize, Clone)]
+struct DownloadProgress {
+    url: String,
+    filepath: String,
+    content_length: u64,
+    downloaded: u64,
+}
+
+#[tauri::command]
+pub async fn download_url(app: AppHandle, url: &str, filepath: &str) -> Result<bool, CJDicError> {
+    download_url_local(url, &app.path().app_data_dir()?, filepath, |progress| {
+        app.emit("download-url-progress", progress).unwrap();
+    })
+    .await
+}
+
+async fn download_url_local(
+    url: &str,
+    rootdir: &PathBuf,
+    filepath: &str,
+    callback: impl Fn(DownloadProgress),
+) -> Result<bool, CJDicError> {
+    let client = Client::new();
+    let file_pf = rootdir.join(filepath);
+
+    let temp_pf = rootdir.join(format!("{}.dl-tmp", filepath));
+    let temp_path = temp_pf.as_path();
+    // Check existing partial download size
+    let existing_size = tokio::fs::metadata(temp_path)
+        .await
+        .map(|m| m.len())
+        .unwrap_or(0);
+
+    let mut request = client.get(url);
+    // Request only the remaining bytes
+    if existing_size > 0 {
+        request = request.header("Range", format!("bytes={}-", existing_size));
+    }
+
+    // Making the asynchronous GET request
+    let mut response = request.send().await?;
+
+    // Check if the request was successful
+    if response.status().is_success() {
+        let content_length = response.content_length().unwrap_or(0);
+        let mut file = if existing_size > 0 {
+            tokio::fs::OpenOptions::new()
+                .append(true)
+                .open(temp_path)
+                .await?
+        } else {
+            tokio::fs::File::create(&file_pf).await?
+        };
+
+        // Downloading the file in chunks
+        let mut downloaded: u64 = 0;
+        while let Some(chunk) = response.chunk().await? {
+            file.write_all(&chunk).await?;
+            downloaded += chunk.len() as u64;
+
+            callback(DownloadProgress {
+                url: url.to_owned(),
+                filepath: filepath.to_owned(),
+                content_length,
+                downloaded,
+            });
+        }
+
+        // Verify download completeness
+        if content_length > 0 && downloaded < content_length {
+            // Delete incomplete file
+            drop(file); // ensure file handle is released before deleting
+            remove_file(&file_pf).ok();
+            return Err(CJDicError::Error(format!(
+                "Incomplete download: expected {} bytes, got {}",
+                content_length, downloaded
+            )));
+        }
+
+        println!("File downloaded successfully to {:?}", &file_pf);
+    } else {
+        eprintln!("Failed to download file: {:?}", response.status());
+    }
+
+    Ok(true)
+}

src-tauri/src/core/mod.rs

@@ -1,5 +0,0 @@
-use vibrato_rkyv::Tokenizer;
-
-pub struct AppService {
-    tokenizer: Tokenizer,
-}

src-tauri/src/error.rs

@@ -0,0 +1,90 @@
+use std::{backtrace::Backtrace, string::FromUtf8Error};
+
+use serde::Serialize;
+use thiserror::Error;
+use vibrato_rkyv::errors::VibratoError;
+use zip::result::ZipError;
+
+#[derive(Debug, Error, Serialize)]
+#[serde(tag = "kind", content = "message")]
+pub enum CJDicError {
+    #[error("SerializationError: {0}")]
+    SerializationError(String),
+
+    #[error("IOError: {0}")]
+    IOError(String),
+
+    #[error("ZipError: {0}")]
+    ZipError(String),
+
+    #[error("VibratoError: {0}")]
+    VibratoError(String),
+
+    #[error("FromUtf8Error: {0}")]
+    FromUtf8Error(String),
+
+    #[error("TauriError: {0}")]
+    TauriError(String),
+
+    #[error("TauriReqwestError: {0}")]
+    TauriReqwestError(String),
+
+    #[error("Error: {0}")]
+    Error(String),
+}
+
+impl From<serde_json::Error> for CJDicError {
+    fn from(e: serde_json::Error) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::SerializationError(e.to_string())
+    }
+}
+
+impl From<std::io::Error> for CJDicError {
+    fn from(e: std::io::Error) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::IOError(e.to_string())
+    }
+}
+
+impl From<ZipError> for CJDicError {
+    fn from(e: ZipError) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::ZipError(e.to_string())
+    }
+}
+
+impl From<VibratoError> for CJDicError {
+    fn from(e: VibratoError) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::VibratoError(e.to_string())
+    }
+}
+
+impl From<FromUtf8Error> for CJDicError {
+    fn from(e: FromUtf8Error) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::FromUtf8Error(e.to_string())
+    }
+}
+
+impl From<tauri::Error> for CJDicError {
+    fn from(e: tauri::Error) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::TauriError(e.to_string())
+    }
+}
+
+impl From<tauri_plugin_http::reqwest::Error> for CJDicError {
+    fn from(e: tauri_plugin_http::reqwest::Error) -> Self {
+        let bt = Backtrace::capture();
+        eprintln!("{e:#}\n{bt}");
+        CJDicError::TauriReqwestError(e.to_string())
+    }
+}

src-tauri/src/lib.rs

@@ -1,4 +1,17 @@
-mod core;
+use std::fs::{create_dir_all, File};
+use tauri::Manager;
+use tauri_plugin_fs::FsExt;
+
+use crate::command::*;
+use crate::service::AppService;
+use crate::yomitan::search::YomitanSearch;
+
+mod tokenizer;
+mod yomitan;
+
+mod command;
+mod error;
+mod service;
 
 // Learn more about Tauri commands at https://tauri.app/develop/calling-rust/
 #[tauri::command]
@@ -11,10 +24,51 @@ pub fn run() {
     tauri::Builder::default()
         .plugin(tauri_plugin_clipboard_manager::init())
         .plugin(tauri_plugin_fs::init())
-        .plugin(tauri_plugin_sql::Builder::new().build())
+        .plugin(
+            tauri_plugin_sql::Builder::default()
+                .add_migrations("sqlite:search.db", YomitanSearch::migrations())
+                .build(),
+        )
         .plugin(tauri_plugin_http::init())
         .plugin(tauri_plugin_opener::init())
-        .invoke_handler(tauri::generate_handler![greet])
+        .setup(|app| {
+            let config_dir = app.path().app_config_dir()?;
+            create_dir_all(&config_dir)?;
+
+            let data_dir = app.path().app_data_dir()?;
+            create_dir_all(&data_dir.join("yomitan/ja"))?;
+
+            let cache_dir = app.path().app_cache_dir()?;
+            let vibrato_dic_dir = cache_dir.join("vibrato");
+            create_dir_all(&vibrato_dic_dir)?;
+
+            let vibrato_dic_path = vibrato_dic_dir.join("system.dic");
+            if !vibrato_dic_path.exists() {
+                let zst_path = app.path().resolve(
+                    "resources/mecab-ipadic/system.dic.zst",
+                    tauri::path::BaseDirectory::Resource,
+                )?;
+
+                // Use Tauri's fs abstraction — works on both Android and desktop
+                let bytes = app.fs().read(&zst_path)?;
+
+                let mut decoder = zstd::Decoder::new(bytes.as_slice())?;
+                let mut out = File::create(&vibrato_dic_path)?;
+                std::io::copy(&mut decoder, &mut out)?;
+            }
+
+            // db_path is at config_dir;
+            let service = AppService::new(&vibrato_dic_path)?;
+            app.manage(service);
+
+            Ok(())
+        })
+        .invoke_handler(tauri::generate_handler![
+            greet,
+            tokenize,
+            normalize,
+            download_url
+        ])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");
 }

src-tauri/src/service.rs

@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use crate::{error::CJDicError, tokenizer::ja::Tokenizer};
+
+pub struct AppService {
+    pub ja_tokenizer: Tokenizer,
+}
+
+impl AppService {
+    pub fn new(vibrato_dic: impl AsRef<Path>) -> Result<Self, CJDicError> {
+        Ok(Self {
+            ja_tokenizer: Tokenizer::new(vibrato_dic)?,
+        })
+    }
+
+    pub fn tokenize(&self, text: String) -> Result<Vec<String>, CJDicError> {
+        self.ja_tokenizer.tokenize(text)
+    }
+
+    pub fn normalize(&self, text: String) -> Result<Vec<String>, CJDicError> {
+        self.ja_tokenizer.normalize(text)
+    }
+}

src-tauri/src/tokenizer/ja.rs

@@ -0,0 +1,95 @@
+use std::{collections::HashSet, fs::File, io::BufReader, path::Path};
+
+use serde::Serialize;
+
+use crate::error::CJDicError;
+
+#[derive(Debug, Serialize)]
+pub struct TokenizeSegment {
+    pub surface: String,
+    /// ```txt
+    /// 0	品詞	Part-of-speech
+    /// 1	品詞細分類1	Part-of-speech subcategory 1
+    /// 2	品詞細分類2	Part-of-speech subcategory 2
+    /// 3	品詞細分類3	Part-of-speech subcategory 3
+    /// 4	活用形	Conjugation form
+    /// 5	活用型	Conjugation type
+    /// 6	原形	Base form
+    /// 7	読み	Reading
+    /// 8	発音	Pronunciation
+    /// ```
+    /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
+    pub details: Vec<String>,
+}
+
+#[derive(Clone)]
+pub struct Tokenizer {
+    tokenizer: vibrato_rkyv::Tokenizer,
+    exclude_pos: HashSet<String>,
+}
+
+impl Tokenizer {
+    pub fn new(vibrato_dic: impl AsRef<Path>) -> Result<Self, CJDicError> {
+        let reader = BufReader::new(File::open(vibrato_dic)?);
+        let dictionary = vibrato_rkyv::Dictionary::read(reader)?;
+        Ok(Self {
+            tokenizer: vibrato_rkyv::Tokenizer::new(dictionary),
+            exclude_pos: HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string())),
+        })
+    }
+
+    pub fn tokenize(&self, text: String) -> Result<Vec<String>, CJDicError> {
+        let mut segmented: Vec<String> = vec![];
+        for t in self.ja_tokenize(text)? {
+            segmented.push(t.surface);
+        }
+        Ok(segmented)
+    }
+
+    fn ja_tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
+        let mut worker = self.tokenizer.new_worker();
+        worker.reset_sentence(text);
+        worker.tokenize();
+
+        let mut output: Vec<TokenizeSegment> = vec![];
+
+        for token in worker.token_iter() {
+            let surface = token.surface().to_string();
+            let details = token.feature().split(",").map(|s| s.to_string()).collect();
+
+            output.push(TokenizeSegment { surface, details });
+        }
+
+        Ok(output)
+    }
+
+    pub fn normalize(&self, text: String) -> Result<Vec<String>, CJDicError> {
+        let mut segmented: Vec<String> = vec![];
+        for t in self.ja_tokenize(text)? {
+            let mut surface = t.surface;
+            let mut pos = None;
+
+            for (i, dt) in t.details.iter().enumerate() {
+                match i {
+                    0 => pos = Some(dt),
+                    6 => {
+                        let base = dt.to_string();
+                        if base != "*" {
+                            surface = base
+                        }
+                    }
+                    _ => {}
+                }
+            }
+
+            if let Some(p) = pos {
+                if self.exclude_pos.contains(p.as_str()) {
+                    continue;
+                }
+            }
+            segmented.push(surface);
+        }
+
+        Ok(segmented)
+    }
+}

src-tauri/src/tokenizer/mod.rs

@@ -0,0 +1 @@
+pub mod ja;

src-tauri/src/yomitan/mod.rs

@@ -0,0 +1,2 @@
+pub mod search;
+pub mod unzip;

src-tauri/src/yomitan/search.rs

@@ -0,0 +1,75 @@
+use tauri_plugin_sql::{Migration, MigrationKind};
+
+pub struct YomitanSearch;
+
+impl YomitanSearch {
+    pub fn migrations() -> Vec<Migration> {
+      vec![
+        Migration {
+            version: 1,
+            description: "create yomitan dict-version tables",
+            sql: "
+            CREATE TABLE dict (
+                id           INTEGER PRIMARY KEY,
+                title        TEXT    NOT NULL,
+                revision     TEXT    NOT NULL,
+                author       TEXT,
+                url          TEXT,
+                description  TEXT,
+                sort_order   INTEGER NOT NULL DEFAULT 0,
+                installed_at TEXT    NOT NULL DEFAULT (datetime('now')),
+                bundle_name  TEXT,                  -- custom
+                lang         TEXT    NOT NULL,      -- custom
+                asset_count  INTEGER,               -- custom
+                source       TEXT,                  -- custom
+                UNIQUE (title, revision)
+            );
+            ",
+            kind: MigrationKind::Up,
+        },
+        Migration {
+            version: 1,
+            description: "create yomitan ja query tables",
+            sql: "
+            CREATE TABLE ja_search (
+                id          INTEGER NOT NULL,       -- yomitan.terms.id OR yomitan dups OR -user.terms.id
+                dict_id     INTEGER NOT NULL REFERENCES dict(id) ON DELETE CASCADE,
+                term        TEXT NOT NULL,          -- normalized
+                term_seg    TEXT,                   -- segmented and filtered for fillers
+                reading     TEXT NOT NULL,          -- normalized
+                frequency   INTEGER,
+                PRIMARY KEY (id, dict_id)
+            );
+
+            CREATE INDEX ja_search_term       ON ja_search (term);
+            CREATE INDEX ja_search_reading    ON ja_search (reading);
+            CREATE INDEX ja_search_frequency  ON ja_search (frequency DESC);
+
+            CREATE VIRTUAL TABLE IF NOT EXISTS ja_search_ft USING fts5 (
+                term_seg,
+                content='ja_search',
+                content_rowid='rowid',
+                tokenize=unicode61
+            );
+
+            CREATE TRIGGER t_ja_search_ft_after_insert
+            AFTER INSERT ON ja_search BEGIN
+                INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg);
+            END;
+
+            CREATE TRIGGER t_ja_search_ft_after_delete
+            AFTER DELETE ON ja_search BEGIN
+                INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg);
+            END;
+
+            CREATE TRIGGER t_ja_search_ft_after_update
+            AFTER UPDATE ON ja_search BEGIN
+                INSERT INTO ja_search_ft (ja_search_ft, rowid, term_seg) VALUES ('delete', OLD.rowid, OLD.term_seg);
+                INSERT INTO ja_search_ft (rowid, term_seg) VALUES (NEW.rowid, NEW.term_seg);
+            END;
+            ",
+            kind: MigrationKind::Up
+        }
+    ]
+    }
+}