word segmentation and clipboard first word detector

Cargo.lock

@@ -19,6 +19,17 @@ dependencies = [
  "cpufeatures",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
+dependencies = [
+ "getrandom 0.2.17",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.4"
@@ -43,6 +54,12 @@ dependencies = [
  "alloc-no-stdlib",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -79,6 +96,50 @@ dependencies = [
  "x11rb",
 ]
 
+[[package]]
+name = "argmin"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "760a49d596b18b881d2fe6e9e6da4608fa64d4a7653ef5cd43bfaa4da018d596"
+dependencies = [
+ "anyhow",
+ "argmin-math",
+ "instant",
+ "num-traits",
+ "paste",
+ "rand 0.8.5",
+ "rand_xoshiro",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "argmin-math"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d93a0d0269b60bd1cd674de70314e3f0da97406cf8c1936ce760d2a46e0f13fe"
+dependencies = [
+ "anyhow",
+ "cfg-if",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "rand 0.8.5",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "argmin-observer-slog"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83d798c8ab61e6a55d161775f2ae8c42e56c15b746398878c36234575e6839aa"
+dependencies = [
+ "anyhow",
+ "argmin",
+ "slog",
+ "slog-async",
+ "slog-term",
+]
+
 [[package]]
 name = "arrayref"
 version = "0.3.9"
@@ -269,6 +330,26 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "bincode"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
+dependencies = [
+ "bincode_derive",
+ "serde",
+ "unty",
+]
+
+[[package]]
+name = "bincode_derive"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
+dependencies = [
+ "virtue",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -517,8 +598,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118"
 dependencies = [
  "iana-time-zone",
+ "js-sys",
  "num-traits",
  "serde",
+ "wasm-bindgen",
  "windows-link 0.2.1",
 ]
 
@@ -561,7 +644,9 @@ dependencies = [
  "serde",
  "serde_json",
  "thiserror 2.0.18",
+ "vibrato",
  "zip",
+ "zstd",
 ]
 
 [[package]]
@@ -710,6 +795,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crawdad"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87fbd1ecd2ed790e11c8fbe034f9b3e7687404818d1bdfd8218d26ec645ec7c5"
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -777,6 +868,15 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "csv-core"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "ctor"
 version = "0.2.9"
@@ -1052,6 +1152,15 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
+[[package]]
+name = "erased-serde"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "erased-serde"
 version = "0.4.9"
@@ -1692,6 +1801,9 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
 
 [[package]]
 name = "hashbrown"
@@ -1699,6 +1811,8 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
+ "allocator-api2",
+ "equivalent",
  "foldhash 0.1.5",
 ]
 
@@ -2071,6 +2185,15 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.11.0"
@@ -2096,6 +2219,17 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "is-terminal"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "is-wsl"
 version = "0.4.0"
@@ -2510,12 +2644,30 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -2857,6 +3009,12 @@ dependencies = [
  "windows-link 0.2.1",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "pathdiff"
 version = "0.2.3"
@@ -3446,6 +3604,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "raw-window-handle"
 version = "0.6.2"
@@ -3622,6 +3789,20 @@ dependencies = [
  "thiserror 2.0.18",
 ]
 
+[[package]]
+name = "rucrf"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5f17d147c82c4a3d9f84274769a5a301b22b4f006b3d0f8a06570bac0acdc91"
+dependencies = [
+ "argmin",
+ "argmin-math",
+ "argmin-observer-slog",
+ "bincode",
+ "crossbeam-channel",
+ "hashbrown 0.15.5",
+]
+
 [[package]]
 name = "rusqlite"
 version = "0.38.0"
@@ -3822,7 +4003,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058"
 dependencies = [
- "erased-serde",
+ "erased-serde 0.4.9",
  "serde",
  "serde_core",
  "typeid",
@@ -4038,6 +4219,44 @@ version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
+[[package]]
+name = "slog"
+version = "2.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b3b8565691b22d2bdfc066426ed48f837fc0c5f2c8cad8d9718f7f99d6995c1"
+dependencies = [
+ "anyhow",
+ "erased-serde 0.3.31",
+ "rustversion",
+ "serde_core",
+]
+
+[[package]]
+name = "slog-async"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84"
+dependencies = [
+ "crossbeam-channel",
+ "slog",
+ "take_mut",
+ "thread_local",
+]
+
+[[package]]
+name = "slog-term"
+version = "2.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cb1fc680b38eed6fad4c02b3871c09d2c81db8c96aa4e9c0a34904c830f09b5"
+dependencies = [
+ "chrono",
+ "is-terminal",
+ "slog",
+ "term",
+ "thread_local",
+ "time",
+]
+
 [[package]]
 name = "smallvec"
 version = "1.15.1"
@@ -4244,6 +4463,12 @@ dependencies = [
  "version-compare",
 ]
 
+[[package]]
+name = "take_mut"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
+
 [[package]]
 name = "tao"
 version = "0.34.5"
@@ -4640,6 +4865,15 @@ dependencies = [
  "utf-8",
 ]
 
+[[package]]
+name = "term"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -4680,6 +4914,15 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "thread_local"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "tiff"
 version = "0.10.3"
@@ -5105,6 +5348,12 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
+[[package]]
+name = "unty"
+version = "0.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
+
 [[package]]
 name = "url"
 version = "2.5.8"
@@ -5172,6 +5421,26 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "vibrato"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b02279291f68f435d84407eae0a99a8ac3dea4c2ff1f9efc86ab2431d13606bd"
+dependencies = [
+ "bincode",
+ "crawdad",
+ "csv-core",
+ "hashbrown 0.12.3",
+ "regex",
+ "rucrf",
+]
+
+[[package]]
+name = "virtue"
+version = "0.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
+
 [[package]]
 name = "vswhom"
 version = "0.1.0"

src-core/Cargo.toml

@@ -15,3 +15,5 @@ zip = "8.1.0"
 blake3 = "1.8.3"
 anyhow = "1.0.102"
 log = "0.4.29"
+vibrato = "0.5.2"
+zstd = "0.13.3"

src-core/examples/add_yomitan_dict_folder.rs

@@ -9,8 +9,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
     let mut writer = service.get_yomitan_writer()?;
 
     let zip_dir = Path::new("src-tauri/resources/yomitan");

src-core/examples/common/mod.rs

@@ -3,3 +3,7 @@ use std::path::PathBuf;
 pub fn get_db_dir() -> PathBuf {
     PathBuf::from("tmp/save-db")
 }
+
+pub fn get_vibrato_dict_dir() -> PathBuf {
+    PathBuf::from("src-tauri/resources/ipadic-mecab/system.dic.zst")
+}

src-core/examples/ja_tokenize.rs

@@ -0,0 +1,19 @@
+use cjdic2_core::AppService;
+
+mod common;
+use common::get_db_dir;
+
+use crate::common::get_vibrato_dict_dir;
+
+fn main() -> Result<(), anyhow::Error> {
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
+
+    println!(
+        "{:#?}",
+        service.ja_tokenize(
+            "予約キャンセル・変更は【配信日前日の19時頃】までに行ってください。".to_string()
+        )?
+    );
+
+    Ok(())
+}

src-core/examples/remove_one_yomitan_dict.rs

@@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
 
     {
         let _timer = Timer::new("remove one".to_string());

src-core/examples/search_yomitan.rs

@@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
 
     {
         let _timer = Timer::new("Search".to_string());

src-core/examples/yomitan_update_schema.rs

@@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer};
 mod common;
 use common::get_db_dir;
 
+use crate::common::get_vibrato_dict_dir;
+
 fn main() -> Result<(), anyhow::Error> {
-    let service = AppService::new(get_db_dir())?;
+    let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?;
 
     {
         let _timer = Timer::new("update schema".to_string());

src-core/src/error.rs

@@ -1,5 +1,6 @@
 use serde::Serialize;
 use thiserror::Error;
+use vibrato::errors::VibratoError;
 use zip::result::ZipError;
 
 #[derive(Debug, Error, Serialize)]
@@ -23,6 +24,9 @@ pub enum CJDicError {
     #[error("Zip error: {0}")]
     ZipError(String),
 
+    #[error("Vibrato error: {0}")]
+    VibratoError(String),
+
     #[error("Not found")]
     NotFound,
 
@@ -64,3 +68,10 @@ impl From<ZipError> for CJDicError {
         CJDicError::ZipError(e.to_string())
     }
 }
+
+impl From<VibratoError> for CJDicError {
+    fn from(e: VibratoError) -> Self {
+        log::error!("{e:#}");
+        CJDicError::VibratoError(e.to_string())
+    }
+}

src-core/src/lib.rs

@@ -5,7 +5,7 @@ mod util;
 
 pub use db::{YomitanRow, YomitanZipImportProgress, YomitanZipImportResult, ZipSource};
 pub use error::CJDicError;
-pub use service::{AppService, SqlParam};
+pub use service::{AppService, SqlParam, TokenizeSegment};
 pub use util::Timer;
 
 pub fn add(left: u64, right: u64) -> u64 {

src-core/src/service.rs

@@ -1,10 +1,12 @@
 use std::{
     collections::{HashMap, HashSet},
+    fs::File,
     path::Path,
 };
 
 use rusqlite::{Connection, params_from_iter};
 use serde::{Deserialize, Serialize};
+use vibrato::{Dictionary, Tokenizer};
 
 use crate::{
     Timer, ZipSource,
@@ -31,14 +33,55 @@ pub enum SqlParam {
     Bool(bool),
 }
 
+#[derive(Debug, Serialize)]
+pub struct TokenizeSegment {
+    surface: String,
+    /// 0	品詞	Part-of-speech
+    /// 1	品詞細分類1	Part-of-speech subcategory 1
+    /// 2	品詞細分類2	Part-of-speech subcategory 2
+    /// 3	品詞細分類3	Part-of-speech subcategory 3
+    /// 4	活用形	Conjugation form
+    /// 5	活用型	Conjugation type
+    /// 6	原形	Base form
+    /// 7	読み	Reading
+    /// 8	発音	Pronunciation
+    /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html
+    details: Vec<String>,
+}
+
 pub struct AppService {
     db: Database,
+    ja_tokenizer: Tokenizer,
 }
 
 impl AppService {
-    pub fn new<P: AsRef<Path>>(db_dir: P) -> Result<Self, CJDicError> {
+    pub fn new<P: AsRef<Path>>(db_dir: P, vibrato_dict: P) -> Result<Self, CJDicError> {
         let db = Database::new(db_dir)?;
-        Ok(Self { db })
+
+        let ja_tokenizer = {
+            let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?;
+            let dictionary = Dictionary::read(reader)?;
+            Tokenizer::new(dictionary)
+        };
+
+        Ok(Self { db, ja_tokenizer })
+    }
+
+    pub fn ja_tokenize(&self, text: String) -> Result<Vec<TokenizeSegment>, CJDicError> {
+        let mut worker = self.ja_tokenizer.new_worker();
+        worker.reset_sentence(text);
+        worker.tokenize();
+
+        let mut output: Vec<TokenizeSegment> = vec![];
+
+        for token in worker.token_iter() {
+            let surface = token.surface().to_string();
+            let details = token.feature().split(",").map(|s| s.to_string()).collect();
+
+            output.push(TokenizeSegment { surface, details });
+        }
+
+        Ok(output)
     }
 
     pub fn execute_sql(

src-react/src/pages/SearchPage.tsx

@@ -94,7 +94,9 @@ function SearchPage() {
   }
 
   async function runSearch(isNew = true) {
-    let norm_q = q.trim();
+    let norm_q = q.split(" ")[0].trim();
+    const isFromSplit = norm_q !== q.trim();
+
     if (isAutoKana) {
       norm_q = norm_q.replace(/n$/i, "ん").replace(/[a-z]$/i, "");
     }
@@ -132,6 +134,22 @@ function SearchPage() {
     });
 
     if (isNew) {
+      if (!result.length && !isFromSplit && ender === "" && q.length > 2) {
+        const segs = await invoke<{ surface: string }[]>("tokenize", {
+          text: q,
+        });
+        if (segs.length > 1) {
+          const newQ =
+            segs.shift()!.surface +
+            " " +
+            segs.map((s) => s.surface).join("") +
+            " ";
+
+          setQ(newQ);
+          return;
+        }
+      }
+
       setEntries(result);
     } else {
       setEntries([...entries, ...result]);

src-tauri/resources/.gitignore

@@ -4,3 +4,5 @@
 jmdict.db
 
 manifest.json
+# vibrato dicts from https://github.com/daac-tools/vibrato/releases
+system.dic.zst

src-tauri/resources/ipadic-mecab/COPYING

@@ -0,0 +1,73 @@
+Copyright 2000, 2001, 2002, 2003 Nara Institute of Science
+and Technology.
+Copyright 2023, LegalOn Technologies, Inc.
+All Rights Reserved.
+
+Use, reproduction, and distribution of this software is permitted.
+Any copy of this software, whether in its original form or modified,
+must include both the above copyright notice and the following
+paragraphs.
+
+Nara Institute of Science and Technology (NAIST),
+the copyright holders, disclaims all warranties with regard to this
+software, including all implied warranties of merchantability and
+fitness, in no event shall NAIST be liable for
+any special, indirect or consequential damages or any damages
+whatsoever resulting from loss of use, data or profits, whether in an
+action of contract, negligence or other tortuous action, arising out
+of or in connection with the use or performance of this software.
+
+A large portion of the dictionary entries
+originate from ICOT Free Software.  The following conditions for ICOT
+Free Software applies to the current dictionary as well.
+
+Each User may also freely distribute the Program, whether in its
+original form or modified, to any third party or parties, PROVIDED
+that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear
+on, or be attached to, the Program, which is distributed substantially
+in the same form as set out herein and that such intended
+distribution, if actually made, will neither violate or otherwise
+contravene any of the laws and regulations of the countries having
+jurisdiction over the User or the intended distribution itself.
+
+NO WARRANTY
+
+The program was produced on an experimental basis in the course of the
+research and development conducted during the project and is provided
+to users as so produced on an experimental basis.  Accordingly, the
+program is provided without any warranty whatsoever, whether express,
+implied, statutory or otherwise.  The term "warranty" used herein
+includes, but is not limited to, any warranty of the quality,
+performance, merchantability and fitness for a particular purpose of
+the program and the nonexistence of any infringement or violation of
+any right of any third party.
+
+Each user of the program will agree and understand, and be deemed to
+have agreed and understood, that there is no warranty whatsoever for
+the program and, accordingly, the entire risk arising from or
+otherwise connected with the program is assumed by the user.
+
+Therefore, neither ICOT, the copyright holder, or any other
+organization that participated in or was otherwise related to the
+development of the program and their respective officials, directors,
+officers and other employees shall be held liable for any and all
+damages, including, without limitation, general, special, incidental
+and consequential damages, arising out of or otherwise in connection
+with the use or inability to use the program or any product, material
+or result produced or otherwise obtained by using the program,
+regardless of whether they have been advised of, or otherwise had
+knowledge of, the possibility of such damages at any time during the
+project or thereafter.  Each user will be deemed to have agreed to the
+foregoing by his or her commencement of use of the program.  The term
+"use" as used herein includes, but is not limited to, the use,
+modification, copying and distribution of the program and the
+production of secondary products from the program.
+
+In the case where the program, whether in its original form or
+modified, was distributed or delivered to or received by a user from
+any person, organization or entity other than ICOT, unless it makes or
+grants independently of ICOT any specific warranty to the user in
+writing, such person, organization or entity, will also be exempted
+from and not be held liable to the user for any such damages as noted
+above as far as the program is concerned.
+÷÷

src-tauri/resources/ipadic-mecab/NOTICE

@@ -0,0 +1,7 @@
+This software includes a binary version of data from
+
+  http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz,
+
+where the connection ids are remapped using CORE data in BCCWJ (except the PN category)
+
+  https://clrd.ninjal.ac.jp/bccwj/.

src-tauri/src/command.rs

@@ -3,7 +3,9 @@ use std::{
     path::PathBuf,
 };
 
-use cjdic2_core::{AppService, CJDicError, SqlParam, YomitanRow, YomitanZipImportResult};
+use cjdic2_core::{
+    AppService, CJDicError, SqlParam, TokenizeSegment, YomitanRow, YomitanZipImportResult,
+};
 use serde::{Deserialize, Serialize};
 use tauri::{AppHandle, Emitter, Manager};
 use tauri_plugin_http::reqwest::Client;
@@ -242,3 +244,11 @@ pub async fn download_url(
     .await
     .map_err(|e| CJDicError::Error(e.to_string()))
 }
+
+#[tauri::command]
+pub async fn tokenize(
+    text: String,
+    state: tauri::State<'_, AppService>,
+) -> Result<Vec<TokenizeSegment>, CJDicError> {
+    state.ja_tokenize(text)
+}

src-tauri/src/lib.rs

@@ -1,8 +1,9 @@
-use std::fs::create_dir_all;
+use std::fs::{self, create_dir_all};
 
 use crate::command::*;
 use cjdic2_core::*;
 use tauri::Manager;
+use tauri_plugin_fs::FsExt;
 
 mod command;
 
@@ -17,9 +18,27 @@ pub fn run() {
             let app_dir = app.path().app_config_dir()?;
             create_dir_all(&app_dir)?;
 
+            let dict_path = {
+                // Resolve the resource path (returns asset://localhost/ URI on Android)
+                let resource_path = app.path().resolve(
+                    "resources/ipadic-mecab/system.dic.zst",
+                    tauri::path::BaseDirectory::Resource,
+                )?;
+
+                // Use Tauri's fs abstraction — works on both Android and desktop
+                let bytes = app.fs().read(&resource_path)?;
+
+                // Write to app's cache dir, which IS a real writable filesystem path on Android
+                let cache_dir = app.path().cache_dir()?;
+                create_dir_all(&cache_dir)?;
+                let dict_path = cache_dir.join("system.dic.zst");
+                fs::write(&dict_path, &bytes)?;
+
+                dict_path
+            };
+
             // db_path is app_dir;
-            let db_dir = app_dir.as_path();
+            let service = AppService::new(&app_dir, &dict_path)?;
-            let service = AppService::new(&db_dir)?;
             app.manage(service);
 
             let app_data_dir = app.path().app_data_dir()?;
@@ -34,6 +53,7 @@ pub fn run() {
             search_yomitan,
             execute_sql,
             download_url,
+            tokenize,
         ])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");

src-tauri/tauri.conf.json

@@ -23,6 +23,9 @@
     }
   },
   "bundle": {
+    "resources": [
+      "resources/**/system.dic.zst"
+    ],
     "active": true,
     "targets": "all",
     "icon": [