diff --git a/Cargo.lock b/Cargo.lock index 7c054bc..0a60575 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,17 +19,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "version_check", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -98,40 +87,40 @@ dependencies = [ [[package]] name = "argmin" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760a49d596b18b881d2fe6e9e6da4608fa64d4a7653ef5cd43bfaa4da018d596" +checksum = "e7ab7ca97779074715a402e5e8045fae27e7191acaec9b4c5653276316e9e404" dependencies = [ "anyhow", "argmin-math", - "instant", "num-traits", "paste", - "rand 0.8.5", + "rand 0.9.2", "rand_xoshiro", - "thiserror 1.0.69", + "thiserror 2.0.18", + "web-time", ] [[package]] name = "argmin-math" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93a0d0269b60bd1cd674de70314e3f0da97406cf8c1936ce760d2a46e0f13fe" +checksum = "ba6958a87117ff5e1d0d7716856a4303752518012ec4a67a68446b6631a1a54d" dependencies = [ "anyhow", "cfg-if", "num-complex", "num-integer", "num-traits", - "rand 0.8.5", - "thiserror 1.0.69", + "rand 0.9.2", + "thiserror 2.0.18", ] [[package]] name = "argmin-observer-slog" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83d798c8ab61e6a55d161775f2ae8c42e56c15b746398878c36234575e6839aa" +checksum = "0b520dbfd3ae35ee2e1d2e456e48c595abaea98c2e5af244a831f62e733c8469" dependencies = [ "anyhow", "argmin", @@ -318,6 +307,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "base64" version = "0.21.7" @@ -330,26 +341,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -437,6 +428,29 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecheck" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bytemuck" version = "1.25.0" @@ -644,7 +658,8 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "vibrato", + "vibrato-rkyv", + "wana_kana", "zip", "zstd", ] @@ -658,6 +673,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "combine" version = "4.6.7" @@ -796,10 +820,13 @@ dependencies = [ ] [[package]] -name = "crawdad" -version = "0.3.0" +name = "crawdad-rkyv" +version = "0.4.0-rkyv.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fbd1ecd2ed790e11c8fbe034f9b3e7687404818d1bdfd8218d26ec645ec7c5" +checksum = "320b8e73b919dc92070dc4abc789a8ae043d27a5b00ed39d6bea8d9c66857f68" +dependencies = [ + "rkyv", +] [[package]] name = "crc32fast" @@ -1090,6 +1117,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "embed-resource" version = "3.0.6" @@ -1286,6 +1319,17 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1363,6 +1407,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futf" version = "0.1.5" @@ -1380,6 +1430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1821,9 +1872,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash", -] [[package]] name = "hashbrown" @@ -1831,8 +1879,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", - "equivalent", "foldhash 0.1.5", ] @@ -1842,6 +1888,8 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -2205,15 +2253,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "ipnet" version = "2.11.0" @@ -2260,6 +2299,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2442,6 +2490,7 @@ checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ "bitflags 2.11.0", "libc", + "redox_syscall 0.7.3", ] [[package]] @@ -2457,9 +2506,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -2503,6 +2552,17 @@ dependencies = [ "sha2", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "mac" version = "0.1.1" @@ -2546,6 +2606,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -2613,6 +2682,26 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "ndk" version = "0.9.0" @@ -2949,6 +3038,12 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" @@ -3024,7 +3119,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link 0.2.1", ] @@ -3404,6 +3499,26 @@ version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "publicsuffix" version = "2.3.0" @@ -3470,6 +3585,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3514,6 +3630,15 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.7.3" @@ -3626,11 +3751,11 @@ dependencies = [ [[package]] name = "rand_xoshiro" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.5", ] [[package]] @@ -3648,6 +3773,15 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -3708,6 +3842,15 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -3759,21 +3902,31 @@ checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", + "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -3799,6 +3952,36 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a30e631b7f4a03dee9056b8ef6982e8ba371dd5bedb74d3ec86df4499132c70" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown 0.16.1", + "indexmap 2.13.0", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8100bb34c0a1d0f907143db3149e6b4eea3c33b9ee8b189720168e818303986f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "rsqlite-vfs" version = "0.1.0" @@ -3810,17 +3993,17 @@ dependencies = [ ] [[package]] -name = "rucrf" -version = "0.3.3" +name = "rucrf-rkyv" +version = "0.3.3-rkyv.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5f17d147c82c4a3d9f84274769a5a301b22b4f006b3d0f8a06570bac0acdc91" +checksum = "6baee03e5fd41176ddcf95bbc4effabea36839df955226fe77e520bd9b1646a0" dependencies = [ "argmin", "argmin-math", "argmin-observer-slog", - "bincode", "crossbeam-channel", - "hashbrown 0.15.5", + "hashbrown 0.16.1", + "rkyv", ] [[package]] @@ -3855,9 +4038,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags 2.11.0", "errno", @@ -3872,6 +4055,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3880,6 +4064,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -3890,12 +4086,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3922,6 +4146,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3979,6 +4212,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.24.0" @@ -4221,6 +4477,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "0.3.11" @@ -4308,7 +4570,7 @@ dependencies = [ "objc2-foundation", "objc2-quartz-core", "raw-window-handle", - "redox_syscall", + "redox_syscall 0.5.18", "tracing", "wasm-bindgen", "web-sys", @@ -4540,6 +4802,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -4863,9 +5136,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.25.0" +version = "3.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", "getrandom 0.4.1", @@ -5368,12 +5641,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - [[package]] name = "url" version = "2.5.8" @@ -5442,25 +5709,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "vibrato" -version = "0.5.2" +name = "vibrato-rkyv" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02279291f68f435d84407eae0a99a8ac3dea4c2ff1f9efc86ab2431d13606bd" +checksum = "b4c69d1126f3ab215599a5d189513135a765b968bc8ce82fd75c7620b589cbdf" dependencies = [ - "bincode", - "crawdad", + "bumpalo", + "crawdad-rkyv", "csv-core", - "hashbrown 0.12.3", + "dirs", + "hashbrown 0.16.1", + "hex", + "log", + "memmap2", "regex", - "rucrf", + "reqwest 0.13.2", + "rkyv", + "rucrf-rkyv", + "sha2", + "tar", + "tempfile", + "thiserror 2.0.18", + "walkdir", + "xz2", + "zstd", ] -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - [[package]] name = "vswhom" version = "0.1.0" @@ -5491,6 +5765,17 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wana_kana" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74666202acfcb4f9b995be2e3e9f7f530deb65e05a1407b8d0b30c9c451238a" +dependencies = [ + "fnv", + "itertools", + "lazy_static", +] + [[package]] name = "want" version = "0.3.1" @@ -5770,6 +6055,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -6489,6 +6783,25 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/src-core/Cargo.toml b/src-core/Cargo.toml index c3e6689..5dabc27 100644 --- a/src-core/Cargo.toml +++ b/src-core/Cargo.toml @@ -14,6 +14,7 @@ thiserror = "2.0.18" zip = "8.1.0" blake3 = "1.8.3" anyhow = "1.0.102" -vibrato = "0.5.2" +vibrato-rkyv = "0.7.5" zstd = "0.13.3" enum-map = "2.7.3" +wana_kana = "4.0.0" diff --git a/src-core/src/db/search.rs b/src-core/src/db/search.rs index 2b2fefd..8d753eb 100644 --- a/src-core/src/db/search.rs +++ b/src-core/src/db/search.rs @@ -1,54 +1,40 @@ -use rusqlite::Connection; +use std::collections::HashSet; + +use rusqlite::{Connection, params}; +use wana_kana::utils::katakana_to_hiragana; use crate::{ - CJDicError, - db::{DBSCHEMA, DbChild}, + CJDicError, Timer, YomitanProgress, + db::{DBFILE, DBSCHEMA, DbChild, yomitan_writer::normalize_term}, + tokenizer::Tokenizer, }; pub(super) fn normalize_reading(s: &str) -> String { - s.chars() - .map(|c| { - if c >= 'ア' && c <= 'ン' { - // Katakana range - if let Some(x) = char::from_u32(c as u32 - 0x60) { - x // Convert to Hiragana - } else { - c // Ensure it's a valid char - } - } else { - c // Return the character as is if it's not in the range - } - }) - .collect() + katakana_to_hiragana(s) } -pub struct SearchDatabase; -// { -// db: Database, -//} +pub struct SearchDatabase<'a> { + conn: &'a mut Connection, +} -// pub struct SearchEntry { -// id: i64, -// term: String, -// reading: String, -// } +impl<'a> SearchDatabase<'a> { + pub(crate) fn new(conn: &'a mut Connection) -> Self { + Self { conn } + } -impl SearchDatabase { - // pub(crate) fn new(db: Database) -> Self { - // Self { db } - // } - - pub fn create_schema(conn: &Connection) -> Result<(), CJDicError> { - conn.execute_batch( + pub fn create_schema(&self) -> Result<(), CJDicError> { + self.conn.execute_batch( &format!(" CREATE TABLE IF NOT EXISTS {0}.terms ( id INTEGER PRIMARY KEY, -- yomitan.terms.id OR yomitan dups OR -user.terms.id term TEXT NOT NULL, -- normalized - reading TEXT NOT NULL -- normalized + reading TEXT NOT NULL, -- normalized + max_score INTEGER ); CREATE INDEX IF NOT EXISTS {0}.terms_term ON terms (term); CREATE INDEX IF NOT EXISTS {0}.terms_reading ON terms (reading); + CREATE INDEX IF NOT EXISTS {0}.terms_max_score ON terms (max_score DESC); CREATE VIRTUAL TABLE IF NOT EXISTS {0}.terms_ft USING fts5 ( term, @@ -61,55 +47,119 @@ impl SearchDatabase { Ok(()) } - // pub fn insert_many( - // &self, - // entries: Vec, - // tokenizer: Tokenizer, - // ) -> Result<(), CJDicError> { - // let mut conn = self.db.conn.lock()?; - // let tx = conn.transaction()?; - // { - // let mut stmt = - // tx.prepare("INSERT INTO terms (id, term, reading) VALUES (?1, ?2, ?3)")?; - // for it in entries.iter() { - // stmt.execute(params![ - // it.id, - // normalize_term(&it.term), - // normalize_reading(&it.reading) - // ])?; - // } + pub fn reset_db(&self) -> Result<(), CJDicError> { + self.conn.execute_batch(&format!( + " + DELETE FROM {0}.terms; + DELETE FROM {0}.terms_ft; + ", + DBSCHEMA[DbChild::Search] + ))?; - // let exclude_pos = - // HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string())); + Ok(()) + } - // let mut stmt = tx.prepare("INSERT INTO terms_ft (rowid, term) VALUES (?1, ?2)")?; - // for it in entries { - // let mut segmented: Vec = vec![]; - // for t in tokenizer.tokenize(it.term) { - // let mut base = t.surface; - // let mut pos = None; + pub fn regenerate_yomitan( + &mut self, + yomitan_schema: &str, + tokenizer: Tokenizer, + progress_callback: impl Fn(YomitanProgress), + ) -> Result<(), CJDicError> { + let total = self.conn.query_row( + &format!("SELECT COALESCE(MAX(id), 0) FROM {}.terms", yomitan_schema), + [], + |r| r.get::<_, i64>(0), + )? as usize; - // for (i, dt) in t.details.iter().enumerate() { - // match i { - // 0 => base = dt.to_string(), - // 6 => pos = Some(dt), - // _ => {} - // } - // } + if total == 0 { + return Ok(()); + } - // if let Some(p) = pos { - // if exclude_pos.contains(p.as_str()) { - // continue; - // } - // } - // segmented.push(base); - // } + let message = &format!("Generating {}", DBFILE[DbChild::Search]); + progress_callback(YomitanProgress { + message: message.to_string(), + current: 0, + total, + steps: 0, + }); - // stmt.execute(params![it.id, segmented.join(" ")])?; - // } - // } - // tx.commit()?; + let _timer = Timer::new(message.to_string()); - // Ok(()) - // } + let tx = self.conn.transaction()?; + { + let mut stmt = tx.prepare(&format!( + "SELECT t.id, t.term, t.reading, tr.max_score + FROM {0}.terms t + LEFT JOIN {0}.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading + ", + yomitan_schema + ))?; + + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, Option>(3)?, + )) + })?; + + let mut stmt_write = tx.prepare(&format!( + "INSERT INTO {}.terms (id, term, reading, max_score) VALUES (?1, ?2, ?3, ?4)", + DBSCHEMA[DbChild::Search] + ))?; + + let exclude_pos = + HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string())); + let mut stmt_write_ft = tx.prepare(&format!( + "INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)", + DBSCHEMA[DbChild::Search] + ))?; + + for (i, row) in rows.enumerate() { + let (id, term, reading, max_score) = row?; + stmt_write.execute(params![ + id, + normalize_term(&term), + normalize_reading(&reading), + max_score + ])?; + + let mut segmented: Vec = vec![]; + for t in tokenizer.tokenize(term) { + let mut base = t.surface; + let mut pos = None; + + for (i, dt) in t.details.iter().enumerate() { + match i { + 0 => base = dt.to_string(), + 6 => pos = Some(dt), + _ => {} + } + } + + if let Some(p) = pos { + if exclude_pos.contains(p.as_str()) { + continue; + } + } + segmented.push(base); + } + + stmt_write_ft.execute(params![id, segmented.join(" ")])?; + + if i % 10_000 == 0 { + progress_callback(YomitanProgress { + message: message.to_string(), + current: i, + total, + steps: i, + }); + } + } + } + tx.commit()?; + + Ok(()) + } } diff --git a/src-core/src/db/yomitan.rs b/src-core/src/db/yomitan.rs index dc9fd43..0c9b610 100644 --- a/src-core/src/db/yomitan.rs +++ b/src-core/src/db/yomitan.rs @@ -50,7 +50,16 @@ impl YomitanDatabase { limit: u32, offset: u32, ) -> Result, CJDicError> { - let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading)); + let q_label = if q_term == q_reading { + q_term + } else { + &format!("{} ({})", q_term, q_reading) + }; + + let _timer = Timer::new(format!( + "search_yomitan: {} offset={} limit={}", + q_label, offset, limit + )); // must be run before self.db.conn.lock() let mut decompressor = self.decompressor()?; @@ -80,53 +89,15 @@ impl YomitanDatabase { let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" }; let conn = self.db.conn.lock()?; - conn.execute_batch( - " - CREATE TEMP TABLE IF NOT EXISTS yomitan_lookup_keys ( - term, - reading, - score, - title, - glossary_id, - def_tags_id, - rules_id, - term_tags_id, - sequence - ); - DELETE FROM yomitan_lookup_keys; - ", - )?; - // let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading)); - let sql_string = format!( - " - INSERT INTO yomitan_lookup_keys - SELECT - t.term, - t.reading, - t.score, - d.title, - t.glossary_id, - t.def_tags_id, - rules_id, - term_tags_id, - sequence - FROM search.terms s - JOIN yomitan.terms t ON t.id = s.id - JOIN yomitan.view_terms_term_rank tr ON tr.term = t.term AND tr.reading = t.reading - JOIN yomitan.dictionaries d ON d.id = t.dict_id - WHERE s.term {eq1} ?1 {and_or} s.reading {eq2} ?2 - ORDER BY tr.max_score DESC, d.sort_order DESC - LIMIT ?3 OFFSET ?4 - " - ); - // println!("{}", sql_string); - // still the most time bottleneck step, several seconds - conn.execute(&sql_string, params![q_term_norm, q_reading, limit, offset])?; - - let _timer = Timer::new(format!("search_yomitan: {} {}", q_term, q_reading)); - let mut stmt = conn.prepare( + let mut stmt = conn.prepare(&format!( " + WITH s AS ( + SELECT * FROM search.terms + WHERE term {eq1} ?1 {and_or} reading {eq2} ?2 + ORDER BY max_score DESC + LIMIT ?3 OFFSET ?4 + ) SELECT t.term, t.reading, @@ -136,15 +107,17 @@ impl YomitanDatabase { COALESCE(g.b, '[]') AS glossary, t.sequence, COALESCE(tt.tags, '') AS term_tags, - t.title AS dict_title - FROM yomitan_lookup_keys t - LEFT JOIN glossary.glossaries g ON g.id = t.glossary_id + d.title AS dict_title + FROM s + JOIN yomitan.terms t ON t.id = s.id + LEFT JOIN yomitan.dictionaries d ON d.id = t.dict_id LEFT JOIN yomitan.def_tag_sets dt ON dt.id = t.def_tags_id LEFT JOIN yomitan.rule_sets r ON r.id = t.rules_id LEFT JOIN yomitan.term_tag_sets tt ON tt.id = t.term_tags_id - ", - )?; - let rows = stmt.query_map(params![], |r| { + LEFT JOIN glossary.glossaries g ON g.id = t.glossary_id + " + ))?; + let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| { Ok(YomitanRow { term: r.get(0)?, reading: r.get(1)?, diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs index 94e7d86..3a33f9e 100644 --- a/src-core/src/db/yomitan_writer.rs +++ b/src-core/src/db/yomitan_writer.rs @@ -13,10 +13,8 @@ use zip::ZipArchive; use crate::{ CJDicError, Timer, - db::{ - DBFILE, DBSCHEMA, DbChild, - search::{SearchDatabase, normalize_reading}, - }, + db::{DBFILE, DBSCHEMA, DbChild, search::SearchDatabase}, + tokenizer::Tokenizer, }; fn blake3_hex(s: &str) -> String { @@ -85,6 +83,7 @@ impl YomitanWriter { pub fn create_schema( &mut self, + tokenizer: Tokenizer, progress_callback: impl Fn(YomitanProgress), ) -> Result<(), CJDicError> { self.conn.execute_batch( @@ -107,53 +106,52 @@ impl YomitanWriter { UNIQUE (title, revision) ); CREATE TABLE IF NOT EXISTS glossaries ( -- not used. moved to a new file - id INTEGER PRIMARY KEY, - hash TEXT NOT NULL UNIQUE, -- maybe used in the future for excerpts - content TEXT NOT NULL + id INTEGER PRIMARY KEY, + hash TEXT NOT NULL UNIQUE, -- maybe used in the future for excerpts + content TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS def_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); CREATE TABLE IF NOT EXISTS term_tag_sets (id INTEGER PRIMARY KEY, tags TEXT NOT NULL UNIQUE); CREATE TABLE IF NOT EXISTS rule_sets (id INTEGER PRIMARY KEY, rules TEXT NOT NULL UNIQUE); CREATE TABLE IF NOT EXISTS terms ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - term TEXT NOT NULL, - reading TEXT NOT NULL, - def_tags_id INTEGER, - rules_id INTEGER, - score INTEGER NOT NULL DEFAULT 0, - glossary_id INTEGER NOT NULL REFERENCES glossaries(id), - sequence INTEGER, - term_tags_id INTEGER, - term_norm TEXT -- custom + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + term TEXT NOT NULL, + reading TEXT NOT NULL, + def_tags_id INTEGER, + rules_id INTEGER, + score INTEGER NOT NULL DEFAULT 0, + glossary_id INTEGER NOT NULL REFERENCES glossaries(id), + sequence INTEGER, + term_tags_id INTEGER ); CREATE TABLE IF NOT EXISTS term_meta ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - term TEXT NOT NULL, - mode TEXT NOT NULL, - reading TEXT, - data TEXT NOT NULL + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + term TEXT NOT NULL, + mode TEXT NOT NULL, + reading TEXT, + data TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS tags ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - name TEXT NOT NULL, - category TEXT, - sort_order INTEGER NOT NULL DEFAULT 0, - notes TEXT, - score INTEGER NOT NULL DEFAULT 0, + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + name TEXT NOT NULL, + category TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + notes TEXT, + score INTEGER NOT NULL DEFAULT 0, UNIQUE (dict_id, name) ); CREATE TABLE IF NOT EXISTS kanji ( - id INTEGER PRIMARY KEY, - dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, - kanji TEXT NOT NULL, - onyomi TEXT, - kunyomi TEXT, - tags TEXT, - meanings TEXT NOT NULL DEFAULT '[]', - stats TEXT NOT NULL DEFAULT '{}' + id INTEGER PRIMARY KEY, + dict_id INTEGER NOT NULL REFERENCES dictionaries(id) ON DELETE CASCADE, + kanji TEXT NOT NULL, + onyomi TEXT, + kunyomi TEXT, + tags TEXT, + meanings TEXT NOT NULL DEFAULT '[]', + stats TEXT NOT NULL DEFAULT '{}' ); ", )?; @@ -164,8 +162,6 @@ impl YomitanWriter { CREATE INDEX IF NOT EXISTS dictionaries_lang ON dictionaries (lang); CREATE INDEX IF NOT EXISTS terms_dict_id ON terms (dict_id); - CREATE INDEX IF NOT EXISTS terms_term_norm ON terms (term_norm); - CREATE INDEX IF NOT EXISTS terms_reading ON terms (reading); CREATE INDEX IF NOT EXISTS terms_term_reading_score ON terms(term, reading, score DESC); CREATE INDEX IF NOT EXISTS term_meta_dict_id ON term_meta (dict_id); @@ -180,8 +176,26 @@ impl YomitanWriter { self.attach_db(DbChild::YomitanGlossary)?; self.attach_db(DbChild::Search)?; - let schema_version = "2026-03-08"; + let search_max_id = { + let mut stmt = self.conn.prepare(&format!( + "SELECT COALESCE(MAX(id), 0) FROM {}.terms", + DBSCHEMA[DbChild::Search] + ))?; + let mut rows = stmt.query([])?; + if let Some(r) = rows.next()? { + r.get(0)? + } else { + 0 + } + }; + if search_max_id == 0 { + let mut search_db = SearchDatabase::new(&mut self.conn); + search_db.reset_db()?; + search_db.regenerate_yomitan("main", tokenizer.clone(), &progress_callback)?; + } + + let schema_version = "2026-03-08"; let current_schema_version = { let mut stmt = self .conn @@ -189,14 +203,14 @@ impl YomitanWriter { stmt.query_one([], |r| r.get::<_, String>(0)).optional()? }; - if let Some(v) = current_schema_version { let v = v.as_str(); if v < "2" { - let total = self - .conn - .query_row("SELECT MAX(id) FROM glossaries", [], |r| r.get::<_, i64>(0))? - as usize; + let total = + self.conn + .query_row("SELECT COALESCE(MAX(id), 0) FROM glossaries", [], |r| { + r.get::<_, i64>(0) + })? as usize; let message = &format!("Migrate to {}", DBFILE[DbChild::YomitanGlossary]); progress_callback(YomitanProgress { @@ -270,59 +284,6 @@ impl YomitanWriter { )?; } - if v < "2026-03-08" { - let total = self - .conn - .query_row("SELECT MAX(id) FROM terms", [], |r| r.get::<_, i64>(0))? - as usize; - - let message = &format!("Generating {}", DBFILE[DbChild::Search]); - progress_callback(YomitanProgress { - message: message.to_string(), - current: 0, - total, - steps: 0, - }); - - let _timer = Timer::new(message.to_string()); - - let tx = self.conn.transaction()?; - { - let mut stmt = tx.prepare("SELECT id, term, reading FROM terms")?; - - let rows = stmt.query_map([], |r| { - Ok(( - r.get::<_, i64>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - )) - })?; - - let mut stmt_write = tx.prepare( - "INSERT INTO search.terms (id, term, reading) VALUES (?1, ?2, ?3)", - )?; - - for (i, row) in rows.enumerate() { - let (id, term, reading) = row?; - stmt_write.execute(params![ - id, - normalize_term(&term), - normalize_reading(&reading) - ])?; - - if i % 10_000 == 0 { - progress_callback(YomitanProgress { - message: message.to_string(), - current: i, - total, - steps: i, - }); - } - } - } - tx.commit()?; - } - if v < schema_version { self.conn .prepare("UPDATE schema_meta SET value = ?2 WHERE key = ?1")? @@ -390,32 +351,6 @@ impl YomitanWriter { } } - let tx = self.conn.transaction()?; - { - let mut stmt = tx.prepare("SELECT rowid, term FROM terms WHERE term_norm IS NULL")?; - - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - })?; - - let mut update_stmt = tx.prepare("UPDATE terms SET term_norm = ?1 WHERE rowid = ?2")?; - - for row in rows { - let (rowid, term) = row?; - let norm = normalize_term(&term); // your Unicode normalizer - update_stmt.execute((&norm, rowid))?; - } - } - tx.commit()?; - // 5 sec - - self.conn.execute_batch( - " - CREATE INDEX IF NOT EXISTS terms_term_norm ON terms (term_norm); - ", - )?; - // 1 sec - Ok(()) } @@ -452,7 +387,7 @@ impl YomitanWriter { match db_child { DbChild::YomitanGlossary => self.create_schema_glossary()?, - DbChild::Search => SearchDatabase::create_schema(&self.conn)?, + DbChild::Search => SearchDatabase::new(&mut self.conn).create_schema()?, _ => (), } diff --git a/src-core/src/error.rs b/src-core/src/error.rs index b429f9a..61517ce 100644 --- a/src-core/src/error.rs +++ b/src-core/src/error.rs @@ -6,7 +6,7 @@ use std::{ use rusqlite::Connection; use serde::Serialize; use thiserror::Error; -use vibrato::{Tokenizer, errors::VibratoError}; +use vibrato_rkyv::{Tokenizer, errors::VibratoError}; use zip::result::ZipError; #[derive(Debug, Error, Serialize)] diff --git a/src-core/src/service.rs b/src-core/src/service.rs index f5adc1c..9dfe310 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -121,7 +121,7 @@ impl AppService { progress_callback: impl Fn(YomitanProgress), ) -> Result { let mut writer = YomitanWriter::new(self.db.dir.clone())?; - writer.create_schema(&progress_callback)?; + writer.create_schema(self.tokenizer.clone(), &progress_callback)?; Ok(writer) } @@ -187,7 +187,7 @@ impl AppService { { let mut writer = YomitanWriter::new(self.db.dir.clone())?; - writer.create_schema(&import_callback)?; + writer.create_schema(self.tokenizer.clone(), &import_callback)?; } { diff --git a/src-core/src/tokenizer.rs b/src-core/src/tokenizer.rs index 59fdc2e..d675639 100644 --- a/src-core/src/tokenizer.rs +++ b/src-core/src/tokenizer.rs @@ -1,6 +1,7 @@ -use std::{fs::File, path::Path}; +use std::path::Path; use serde::Serialize; +use vibrato_rkyv::CacheStrategy; use crate::CJDicError; @@ -20,16 +21,17 @@ pub struct TokenizeSegment { pub details: Vec, } +#[derive(Clone)] pub struct Tokenizer { - ja_tokenizer: vibrato::Tokenizer, + ja_tokenizer: vibrato_rkyv::Tokenizer, } impl Tokenizer { pub fn new(vibrato_dict: impl AsRef) -> Result { - let reader = zstd::Decoder::new(File::open(vibrato_dict)?)?; - let dictionary = vibrato::Dictionary::read(reader)?; + let dictionary = + vibrato_rkyv::Dictionary::from_zstd(vibrato_dict, CacheStrategy::GlobalCache)?; Ok(Self { - ja_tokenizer: vibrato::Tokenizer::new(dictionary), + ja_tokenizer: vibrato_rkyv::Tokenizer::new(dictionary), }) } diff --git a/src-core/src/util.rs b/src-core/src/util.rs index e596615..00fe005 100644 --- a/src-core/src/util.rs +++ b/src-core/src/util.rs @@ -1,13 +1,10 @@ -#[cfg(debug_assertions)] use std::time::Instant; -#[cfg(debug_assertions)] pub struct Timer { start: Instant, label: String, } -#[cfg(debug_assertions)] impl Timer { pub fn new(label: String) -> Self { Self { @@ -17,19 +14,8 @@ impl Timer { } } -#[cfg(debug_assertions)] impl Drop for Timer { fn drop(&mut self) { println!("[{}] {:.2?}", self.label, self.start.elapsed()) } } - -#[cfg(not(debug_assertions))] -pub struct Timer; - -#[cfg(not(debug_assertions))] -impl Timer { - pub fn new(_label: String) -> Self { - Self - } -} diff --git a/src-tauri/resources/.gitignore b/src-tauri/resources/.gitignore index b5cfdbf..f6f9f3f 100644 --- a/src-tauri/resources/.gitignore +++ b/src-tauri/resources/.gitignore @@ -4,5 +4,5 @@ jmdict.db manifest.json -# vibrato dicts from https://github.com/daac-tools/vibrato/releases +# vibrato-rkyv dicts from https://github.com/stellanomia/vibrato-rkyv/releases/tag/v0.6.2 system.dic.zst diff --git a/src-tauri/resources/ipadic-mecab/NOTICE b/src-tauri/resources/ipadic-mecab/NOTICE deleted file mode 100644 index 9bf64bd..0000000 --- a/src-tauri/resources/ipadic-mecab/NOTICE +++ /dev/null @@ -1,7 +0,0 @@ -This software includes a binary version of data from - - http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz, - -where the connection ids are remapped using CORE data in BCCWJ (except the PN category) - - https://clrd.ninjal.ac.jp/bccwj/. diff --git a/src-tauri/resources/ipadic-mecab/COPYING b/src-tauri/resources/mecab-ipadic/COPYING similarity index 97% rename from src-tauri/resources/ipadic-mecab/COPYING rename to src-tauri/resources/mecab-ipadic/COPYING index 85a0c63..dc7db62 100644 --- a/src-tauri/resources/ipadic-mecab/COPYING +++ b/src-tauri/resources/mecab-ipadic/COPYING @@ -1,7 +1,5 @@ Copyright 2000, 2001, 2002, 2003 Nara Institute of Science -and Technology. -Copyright 2023, LegalOn Technologies, Inc. -All Rights Reserved. +and Technology. All Rights Reserved. Use, reproduction, and distribution of this software is permitted. Any copy of this software, whether in its original form or modified, @@ -70,4 +68,4 @@ grants independently of ICOT any specific warranty to the user in writing, such person, organization or entity, will also be exempted from and not be held liable to the user for any such damages as noted above as far as the program is concerned. -÷÷ + diff --git a/src-tauri/resources/naist-jdic-mecab/COPYING b/src-tauri/resources/naist-jdic-mecab/COPYING deleted file mode 100644 index b2e68f4..0000000 --- a/src-tauri/resources/naist-jdic-mecab/COPYING +++ /dev/null @@ -1,32 +0,0 @@ -Copyright (c) 2009, Nara Institute of Science and Technology, Japan. -Copyright 2023, LegalOn Technologies, Inc. - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. -Neither the name of the Nara Institute of Science and Technology -(NAIST) nor the names of its contributors may be used to endorse or -promote products derived from this software without specific prior -written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - diff --git a/src-tauri/resources/naist-jdic-mecab/NOTICE b/src-tauri/resources/naist-jdic-mecab/NOTICE deleted file mode 100644 index d54f261..0000000 --- a/src-tauri/resources/naist-jdic-mecab/NOTICE +++ /dev/null @@ -1,7 +0,0 @@ -This software includes a binary version of data from - - https://ja.osdn.net/projects/naist-jdic/downloads/53500/mecab-naist-jdic-0.6.3b-20111013.tar.gz - -where the connection ids are remapped using CORE data in BCCWJ (except the PN category) - - https://clrd.ninjal.ac.jp/bccwj/. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 290b9e9..31e51f6 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -21,7 +21,7 @@ pub fn run() { let dict_path = { // Resolve the resource path (returns asset://localhost/ URI on Android) let resource_path = app.path().resolve( - "resources/naist-jdic-mecab/system.dic.zst", + "resources/mecab-ipadic/system.dic.zst", tauri::path::BaseDirectory::Resource, )?; diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 57f5bce..4f7bb18 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -24,7 +24,7 @@ }, "bundle": { "resources": [ - "resources/naist-jdic-mecab/system.dic.zst" + "resources/mecab-ipadic/system.dic.zst" ], "active": true, "targets": "all",