diff --git a/Cargo.lock b/Cargo.lock index 979ff32..a34ae01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,6 +19,17 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -43,6 +54,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -79,6 +96,50 @@ dependencies = [ "x11rb", ] +[[package]] +name = "argmin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760a49d596b18b881d2fe6e9e6da4608fa64d4a7653ef5cd43bfaa4da018d596" +dependencies = [ + "anyhow", + "argmin-math", + "instant", + "num-traits", + "paste", + "rand 0.8.5", + "rand_xoshiro", + "thiserror 1.0.69", +] + +[[package]] +name = "argmin-math" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93a0d0269b60bd1cd674de70314e3f0da97406cf8c1936ce760d2a46e0f13fe" +dependencies = [ + "anyhow", + "cfg-if", + "num-complex", + "num-integer", + "num-traits", + "rand 0.8.5", + "thiserror 1.0.69", +] + +[[package]] +name = "argmin-observer-slog" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d798c8ab61e6a55d161775f2ae8c42e56c15b746398878c36234575e6839aa" +dependencies = [ + "anyhow", + "argmin", + "slog", + "slog-async", + "slog-term", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -269,6 +330,26 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -517,8 +598,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link 0.2.1", ] @@ -561,7 +644,9 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", + "vibrato", "zip", + "zstd", ] [[package]] @@ -710,6 +795,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crawdad" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fbd1ecd2ed790e11c8fbe034f9b3e7687404818d1bdfd8218d26ec645ec7c5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -777,6 +868,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.2.9" @@ -1052,6 +1152,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +dependencies = [ + "serde", +] + [[package]] name = "erased-serde" version = "0.4.9" @@ -1692,6 +1801,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -1699,6 +1811,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -2071,6 +2185,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2096,6 +2219,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is-wsl" version = "0.4.0" @@ -2510,12 +2644,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2857,6 +3009,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pathdiff" version = "0.2.3" @@ -3446,6 +3604,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -3622,6 +3789,20 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "rucrf" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f17d147c82c4a3d9f84274769a5a301b22b4f006b3d0f8a06570bac0acdc91" +dependencies = [ + "argmin", + "argmin-math", + "argmin-observer-slog", + "bincode", + "crossbeam-channel", + "hashbrown 0.15.5", +] + [[package]] name = "rusqlite" version = "0.38.0" @@ -3822,7 +4003,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" dependencies = [ - "erased-serde", + "erased-serde 0.4.9", "serde", "serde_core", "typeid", @@ -4038,6 +4219,44 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "slog" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b8565691b22d2bdfc066426ed48f837fc0c5f2c8cad8d9718f7f99d6995c1" +dependencies = [ + "anyhow", + "erased-serde 0.3.31", + "rustversion", + "serde_core", +] + +[[package]] +name = "slog-async" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-term" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cb1fc680b38eed6fad4c02b3871c09d2c81db8c96aa4e9c0a34904c830f09b5" +dependencies = [ + "chrono", + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -4244,6 +4463,12 @@ dependencies = [ "version-compare", ] +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tao" version = "0.34.5" @@ -4640,6 +4865,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "term" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4680,6 +4914,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "tiff" version = "0.10.3" @@ -5105,6 +5348,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -5172,6 +5421,26 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vibrato" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02279291f68f435d84407eae0a99a8ac3dea4c2ff1f9efc86ab2431d13606bd" +dependencies = [ + "bincode", + "crawdad", + "csv-core", + "hashbrown 0.12.3", + "regex", + "rucrf", +] + +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "vswhom" version = "0.1.0" diff --git a/src-core/Cargo.toml b/src-core/Cargo.toml index 78666b1..2c8b230 100644 --- a/src-core/Cargo.toml +++ b/src-core/Cargo.toml @@ -15,3 +15,5 @@ zip = "8.1.0" blake3 = "1.8.3" anyhow = "1.0.102" log = "0.4.29" +vibrato = "0.5.2" +zstd = "0.13.3" diff --git a/src-core/examples/add_yomitan_dict_folder.rs b/src-core/examples/add_yomitan_dict_folder.rs index 96e5af0..e4f4cb9 100644 --- a/src-core/examples/add_yomitan_dict_folder.rs +++ b/src-core/examples/add_yomitan_dict_folder.rs @@ -9,8 +9,10 @@ use cjdic2_core::{AppService, Timer}; mod common; use common::get_db_dir; +use crate::common::get_vibrato_dict_dir; + fn main() -> Result<(), anyhow::Error> { - let service = AppService::new(get_db_dir())?; + let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?; let mut writer = service.get_yomitan_writer()?; let zip_dir = Path::new("src-tauri/resources/yomitan"); diff --git a/src-core/examples/common/mod.rs b/src-core/examples/common/mod.rs index 3058cea..bffcd9b 100644 --- a/src-core/examples/common/mod.rs +++ b/src-core/examples/common/mod.rs @@ -3,3 +3,7 @@ use std::path::PathBuf; pub fn get_db_dir() -> PathBuf { PathBuf::from("tmp/save-db") } + +pub fn get_vibrato_dict_dir() -> PathBuf { + PathBuf::from("src-tauri/resources/ipadic-mecab/system.dic.zst") +} diff --git a/src-core/examples/ja_tokenize.rs b/src-core/examples/ja_tokenize.rs new file mode 100644 index 0000000..43f8ef8 --- /dev/null +++ b/src-core/examples/ja_tokenize.rs @@ -0,0 +1,19 @@ +use cjdic2_core::AppService; + +mod common; +use common::get_db_dir; + +use crate::common::get_vibrato_dict_dir; + +fn main() -> Result<(), anyhow::Error> { + let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?; + + println!( + "{:#?}", + service.ja_tokenize( + "予約キャンセル・変更は【配信日前日の19時頃】までに行ってください。".to_string() + )? + ); + + Ok(()) +} diff --git a/src-core/examples/remove_one_yomitan_dict.rs b/src-core/examples/remove_one_yomitan_dict.rs index 7a9dcfc..3298a8e 100644 --- a/src-core/examples/remove_one_yomitan_dict.rs +++ b/src-core/examples/remove_one_yomitan_dict.rs @@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer}; mod common; use common::get_db_dir; +use crate::common::get_vibrato_dict_dir; + fn main() -> Result<(), anyhow::Error> { - let service = AppService::new(get_db_dir())?; + let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?; { let _timer = Timer::new("remove one".to_string()); diff --git a/src-core/examples/search_yomitan.rs b/src-core/examples/search_yomitan.rs index 12ea1b7..61d9ca9 100644 --- a/src-core/examples/search_yomitan.rs +++ b/src-core/examples/search_yomitan.rs @@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer}; mod common; use common::get_db_dir; +use crate::common::get_vibrato_dict_dir; + fn main() -> Result<(), anyhow::Error> { - let service = AppService::new(get_db_dir())?; + let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?; { let _timer = Timer::new("Search".to_string()); diff --git a/src-core/examples/yomitan_update_schema.rs b/src-core/examples/yomitan_update_schema.rs index 5e4683b..58caad3 100644 --- a/src-core/examples/yomitan_update_schema.rs +++ b/src-core/examples/yomitan_update_schema.rs @@ -4,8 +4,10 @@ use cjdic2_core::{AppService, Timer}; mod common; use common::get_db_dir; +use crate::common::get_vibrato_dict_dir; + fn main() -> Result<(), anyhow::Error> { - let service = AppService::new(get_db_dir())?; + let service = AppService::new(get_db_dir(), get_vibrato_dict_dir())?; { let _timer = Timer::new("update schema".to_string()); diff --git a/src-core/src/error.rs b/src-core/src/error.rs index f744f0c..921e6f1 100644 --- a/src-core/src/error.rs +++ b/src-core/src/error.rs @@ -1,5 +1,6 @@ use serde::Serialize; use thiserror::Error; +use vibrato::errors::VibratoError; use zip::result::ZipError; #[derive(Debug, Error, Serialize)] @@ -23,6 +24,9 @@ pub enum CJDicError { #[error("Zip error: {0}")] ZipError(String), + #[error("Vibrato error: {0}")] + VibratoError(String), + #[error("Not found")] NotFound, @@ -64,3 +68,10 @@ impl From for CJDicError { CJDicError::ZipError(e.to_string()) } } + +impl From for CJDicError { + fn from(e: VibratoError) -> Self { + log::error!("{e:#}"); + CJDicError::VibratoError(e.to_string()) + } +} diff --git a/src-core/src/lib.rs b/src-core/src/lib.rs index 84a79d9..08d088b 100644 --- a/src-core/src/lib.rs +++ b/src-core/src/lib.rs @@ -5,7 +5,7 @@ mod util; pub use db::{YomitanRow, YomitanZipImportProgress, YomitanZipImportResult, ZipSource}; pub use error::CJDicError; -pub use service::{AppService, SqlParam}; +pub use service::{AppService, SqlParam, TokenizeSegment}; pub use util::Timer; pub fn add(left: u64, right: u64) -> u64 { diff --git a/src-core/src/service.rs b/src-core/src/service.rs index def8b15..f46f6df 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -1,10 +1,12 @@ use std::{ collections::{HashMap, HashSet}, + fs::File, path::Path, }; use rusqlite::{Connection, params_from_iter}; use serde::{Deserialize, Serialize}; +use vibrato::{Dictionary, Tokenizer}; use crate::{ Timer, ZipSource, @@ -31,14 +33,55 @@ pub enum SqlParam { Bool(bool), } +#[derive(Debug, Serialize)] +pub struct TokenizeSegment { + surface: String, + /// 0 品詞 Part-of-speech + /// 1 品詞細分類1 Part-of-speech subcategory 1 + /// 2 品詞細分類2 Part-of-speech subcategory 2 + /// 3 品詞細分類3 Part-of-speech subcategory 3 + /// 4 活用形 Conjugation form + /// 5 活用型 Conjugation type + /// 6 原形 Base form + /// 7 読み Reading + /// 8 発音 Pronunciation + /// @see https://lindera.github.io/lindera/dictionaries/ipadic.html + details: Vec, +} + pub struct AppService { db: Database, + ja_tokenizer: Tokenizer, } impl AppService { - pub fn new>(db_dir: P) -> Result { + pub fn new>(db_dir: P, vibrato_dict: P) -> Result { let db = Database::new(db_dir)?; - Ok(Self { db }) + + let ja_tokenizer = { + let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?; + let dictionary = Dictionary::read(reader)?; + Tokenizer::new(dictionary) + }; + + Ok(Self { db, ja_tokenizer }) + } + + pub fn ja_tokenize(&self, text: String) -> Result, CJDicError> { + let mut worker = self.ja_tokenizer.new_worker(); + worker.reset_sentence(text); + worker.tokenize(); + + let mut output: Vec = vec![]; + + for token in worker.token_iter() { + let surface = token.surface().to_string(); + let details = token.feature().split(",").map(|s| s.to_string()).collect(); + + output.push(TokenizeSegment { surface, details }); + } + + Ok(output) } pub fn execute_sql( diff --git a/src-react/src/pages/SearchPage.tsx b/src-react/src/pages/SearchPage.tsx index e9edd8c..25a7846 100644 --- a/src-react/src/pages/SearchPage.tsx +++ b/src-react/src/pages/SearchPage.tsx @@ -94,7 +94,9 @@ function SearchPage() { } async function runSearch(isNew = true) { - let norm_q = q.trim(); + let norm_q = q.split(" ")[0].trim(); + const isFromSplit = norm_q !== q.trim(); + if (isAutoKana) { norm_q = norm_q.replace(/n$/i, "ん").replace(/[a-z]$/i, ""); } @@ -132,6 +134,22 @@ function SearchPage() { }); if (isNew) { + if (!result.length && !isFromSplit && ender === "" && q.length > 2) { + const segs = await invoke<{ surface: string }[]>("tokenize", { + text: q, + }); + if (segs.length > 1) { + const newQ = + segs.shift()!.surface + + " " + + segs.map((s) => s.surface).join("") + + " "; + + setQ(newQ); + return; + } + } + setEntries(result); } else { setEntries([...entries, ...result]); diff --git a/src-tauri/gen/android/app/src/main/assets/resources/ipadic-mecab/system.dic.zst b/src-tauri/gen/android/app/src/main/assets/resources/ipadic-mecab/system.dic.zst new file mode 100644 index 0000000..230307b Binary files /dev/null and b/src-tauri/gen/android/app/src/main/assets/resources/ipadic-mecab/system.dic.zst differ diff --git a/src-tauri/resources/.gitignore b/src-tauri/resources/.gitignore index b02ab73..b5cfdbf 100644 --- a/src-tauri/resources/.gitignore +++ b/src-tauri/resources/.gitignore @@ -4,3 +4,5 @@ jmdict.db manifest.json +# vibrato dicts from https://github.com/daac-tools/vibrato/releases +system.dic.zst diff --git a/src-tauri/resources/ipadic-mecab/COPYING b/src-tauri/resources/ipadic-mecab/COPYING new file mode 100644 index 0000000..85a0c63 --- /dev/null +++ b/src-tauri/resources/ipadic-mecab/COPYING @@ -0,0 +1,73 @@ +Copyright 2000, 2001, 2002, 2003 Nara Institute of Science +and Technology. +Copyright 2023, LegalOn Technologies, Inc. +All Rights Reserved. + +Use, reproduction, and distribution of this software is permitted. +Any copy of this software, whether in its original form or modified, +must include both the above copyright notice and the following +paragraphs. + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. +÷÷ diff --git a/src-tauri/resources/ipadic-mecab/NOTICE b/src-tauri/resources/ipadic-mecab/NOTICE new file mode 100644 index 0000000..9bf64bd --- /dev/null +++ b/src-tauri/resources/ipadic-mecab/NOTICE @@ -0,0 +1,7 @@ +This software includes a binary version of data from + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz, + +where the connection ids are remapped using CORE data in BCCWJ (except the PN category) + + https://clrd.ninjal.ac.jp/bccwj/. diff --git a/src-tauri/src/command.rs b/src-tauri/src/command.rs index 329de6a..20ee618 100644 --- a/src-tauri/src/command.rs +++ b/src-tauri/src/command.rs @@ -3,7 +3,9 @@ use std::{ path::PathBuf, }; -use cjdic2_core::{AppService, CJDicError, SqlParam, YomitanRow, YomitanZipImportResult}; +use cjdic2_core::{ + AppService, CJDicError, SqlParam, TokenizeSegment, YomitanRow, YomitanZipImportResult, +}; use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Emitter, Manager}; use tauri_plugin_http::reqwest::Client; @@ -242,3 +244,11 @@ pub async fn download_url( .await .map_err(|e| CJDicError::Error(e.to_string())) } + +#[tauri::command] +pub async fn tokenize( + text: String, + state: tauri::State<'_, AppService>, +) -> Result, CJDicError> { + state.ja_tokenize(text) +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index b8af246..1d087a3 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,8 +1,9 @@ -use std::fs::create_dir_all; +use std::fs::{self, create_dir_all}; use crate::command::*; use cjdic2_core::*; use tauri::Manager; +use tauri_plugin_fs::FsExt; mod command; @@ -17,9 +18,27 @@ pub fn run() { let app_dir = app.path().app_config_dir()?; create_dir_all(&app_dir)?; + let dict_path = { + // Resolve the resource path (returns asset://localhost/ URI on Android) + let resource_path = app.path().resolve( + "resources/ipadic-mecab/system.dic.zst", + tauri::path::BaseDirectory::Resource, + )?; + + // Use Tauri's fs abstraction — works on both Android and desktop + let bytes = app.fs().read(&resource_path)?; + + // Write to app's cache dir, which IS a real writable filesystem path on Android + let cache_dir = app.path().cache_dir()?; + create_dir_all(&cache_dir)?; + let dict_path = cache_dir.join("system.dic.zst"); + fs::write(&dict_path, &bytes)?; + + dict_path + }; + // db_path is app_dir; - let db_dir = app_dir.as_path(); - let service = AppService::new(&db_dir)?; + let service = AppService::new(&app_dir, &dict_path)?; app.manage(service); let app_data_dir = app.path().app_data_dir()?; @@ -34,6 +53,7 @@ pub fn run() { search_yomitan, execute_sql, download_url, + tokenize, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index f1d48eb..195fa9e 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -23,6 +23,9 @@ } }, "bundle": { + "resources": [ + "resources/**/system.dic.zst" + ], "active": true, "targets": "all", "icon": [