polv/cjdic2
1
0 Fork 0
mirror of https://github.com/patarapolw/cjdic2.git synced 2026-11-09 17:39:35 +00:00
Code Releases 2 Activity

make starting from zero work on Android + Windows

Browse Source
This commit is contained in:
polv committed 2026-03-10 01:11:36 +07:00
1 parent 653eacf8d4
commit a00efbf505
10 files changed
+166 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src-core/src/tokenizer.rs
+2 -3
View File
@@ -26,9 +26,8 @@ pub struct Tokenizer {
}
impl Tokenizer {
pub fn new(vibrato_dict: impl AsRef<Path>) -> Result<Self, CJDicError> {
let file = BufReader::new(File::open(vibrato_dict)?);
let reader = zstd::Decoder::new(file)?;
pub fn new(vibrato_dic: impl AsRef<Path>) -> Result<Self, CJDicError> {
let reader = BufReader::new(File::open(vibrato_dic)?);
let dictionary = vibrato_rkyv::Dictionary::read(reader)?;
Ok(Self {
ja_tokenizer: vibrato_rkyv::Tokenizer::new(dictionary),