word segmentation and clipboard first word detector

src-core/Cargo.toml

@@ -15,3 +15,5 @@ zip = "8.1.0"
 blake3 = "1.8.3"
 anyhow = "1.0.102"
 log = "0.4.29"
+vibrato = "0.5.2"
+zstd = "0.13.3"

src-core/examples/add_yomitan_dict_folder.rs

@@ -9,8 +9,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
     let mut writer = service.get_yomitan_writer()?;
 
     let zip_dir = Path::new("src-tauri/resources/yomitan");

src-core/examples/common/mod.rs

@@ -3,3 +3,7 @@ use std::path::PathBuf;
 pub fn get_db_dir() -> PathBuf {
     PathBuf::from("tmp/save-db")
 }
+
+pub fn get_vibrato_dict_dir() -> PathBuf {
+    PathBuf::from("src-tauri/resources/ipadic-mecab/system.dic.zst")
+}

src-core/examples/ja_tokenize.rs

@@ -0,0 +1,19 @@
+use cjdic2_core::AppService;
+
+mod common;
+use common::get_db_dir;
+
+use crate::common::get_vibrato_dict_dir;
+
+fn main() -> Result<(), anyhow::Error> {
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
+
+    println!(
+        "{:#?}",
+        service.ja_tokenize(
+            "予約キャンセル・変更は【配信日前日の19時頃】までに行ってください。".to_string()
+        )?
+    );
+
+    Ok(())
+}

src-core/examples/remove_one_yomitan_dict.rs

@@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
 
     {
         let _timer = Timer::new("remove one".to_string());

src-core/examples/search_yomitan.rs

@@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
 
     {
         let _timer = Timer::new("Search".to_string());

src-core/examples/yomitan_update_schema.rs

@@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
 
     {
         let _timer = Timer::new("update schema".to_string());

src-core/src/error.rs

@@ -1,5 +1,6 @@
 use serde::Serialize;
 use thiserror::Error;
+use vibrato::errors::VibratoError;
 use zip::result::ZipError;
 
 #[derive(Debug, Error, Serialize)]
@@ -23,6 +24,9 @@ pub enum CJDicError {
     #[error("Zip error: {0}")]
     ZipError(String),
 
+    #[error("Vibrato error: {0}")]
+    VibratoError(String),
+
     #[error("Not found")]
     NotFound,
 
@@ -64,3 +68,10 @@ impl From<ZipError> for CJDicError {
         CJDicError::ZipError(e.to_string())
     }
 }
+
+impl From<VibratoError> for CJDicError {
+    fn from(e: VibratoError) -> Self {
+        log::error!("{e:#}");
+        CJDicError::VibratoError(e.to_string())
+    }
+}

src-core/src/lib.rs

@@ -5,7 +5,7 @@ mod util;
 
 pub use db::{YomitanRow, YomitanZipImportProgress, YomitanZipImportResult, ZipSource};
 pub use error::CJDicError;
-pub use service::{AppService, SqlParam};
+pub use service::{AppService, SqlParam, TokenizeSegment};
 pub use util::Timer;
 
 pub fn add(left: u64, right: u64) -> u64 {

src-core/src/service.rs

@@ -1,10 +1,12 @@
 use std::{
     collections::{HashMap, HashSet},
+    fs::File,
     path::Path,
 };
 
 use rusqlite::{Connection, params_from_iter};
 use serde::{Deserialize, Serialize};
+use vibrato::{Dictionary, Tokenizer};
 
 use crate::{
     Timer, ZipSource,
@@ -31,14 +33,55 @@ pub enum SqlParam {
     Bool(bool),
 }
 
+#[derive(Debug, Serialize)]
+pub struct TokenizeSegment {
+    surface: String,
+    /// 0	品詞	Part-of-speech
+    /// 1	品詞細分類1	Part-of-speech subcategory 1
+    /// 2	品詞細分類2	Part-of-speech subcategory 2
+    /// 3	品詞細分類3	Part-of-speech subcategory 3
+    /// 4	活用形	Conjugation form
+    /// 5	活用型	Conjugation type
+    /// 6	原形	Base form
+    /// 7	読み	Reading
+    /// 8	発音	Pronunciation
+    /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
+    details: Vec<String>,
+}
+
 pub struct AppService {
     db: Database,
+    ja_tokenizer: Tokenizer,
 }
 
 impl AppService {
-    pub fn new<P: AsRef<Path>>(db_dir: P) -> Result<Self, CJDicError> {
+    pub fn new<P: AsRef<Path>>(db_dir: P, vibrato_dict: P) -> Result<Self, CJDicError> {
         let db = Database::new(db_dir)?;
-        Ok(Self { db })
+
+        let ja_tokenizer = {
+            let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
+            let dictionary = Dictionary::read(reader)?;
+            Tokenizer::new(dictionary)
+        };
+
+        Ok(Self { db, ja_tokenizer })
+    }
+
+    pub fn ja_tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
+        let mut worker = self.ja_tokenizer.new_worker();
+        worker.reset_sentence(text);
+        worker.tokenize();
+
+        let mut output: Vec<TokenizeSegment> = vec![];
+
+        for token in worker.token_iter() {
+            let surface = token.surface().to_string();
+            let details = token.feature().split(",").map(|s| s.to_string()).collect();
+
+            output.push(TokenizeSegment { surface, details });
+        }
+
+        Ok(output)
     }
 
     pub fn execute_sql(