diff --git a/Cargo.lock b/Cargo.lock index d5233e6..171c1f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -236,6 +236,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "base64" version = "0.21.7" @@ -511,6 +533,8 @@ version = "0.1.0" dependencies = [ "anyhow", "cjdic2-core", + "reqwest 0.13.2", + "serde", "serde_json", "tauri", "tauri-build", @@ -533,6 +557,15 @@ dependencies = [ "zip", ] +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "combine" version = "4.6.7" @@ -1150,6 +1183,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futf" version = "0.1.5" @@ -2639,6 +2678,12 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" @@ -3099,6 +3144,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3379,21 +3425,30 @@ checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -3478,6 +3533,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3486,6 +3542,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -3496,12 +3564,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3528,6 +3624,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3585,6 +3690,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.24.0" @@ -5178,6 +5306,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -6036,9 +6173,9 @@ dependencies = [ [[package]] name = "zip" -version = "8.1.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e499faf5c6b97a0d086f4a8733de6d47aee2252b8127962439d8d4311a73f72" +checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004" dependencies = [ "aes", "bzip2", diff --git a/cjdic2.code-workspace b/cjdic2.code-workspace index f397e10..1c426c3 100644 --- a/cjdic2.code-workspace +++ b/cjdic2.code-workspace @@ -17,7 +17,7 @@ }, ], "settings": { - "importSorter.generalConfiguration.sortOnBeforeSave": true, + // "importSorter.generalConfiguration.sortOnBeforeSave": true, "importSorter.importStringConfiguration.quoteMark": "double", "json.schemaDownload.enable": false, }, diff --git a/src-core/examples/remove_one_yomitan_dict.rs b/src-core/examples/remove_one_yomitan_dict.rs index 698d154..7a9dcfc 100644 --- a/src-core/examples/remove_one_yomitan_dict.rs +++ b/src-core/examples/remove_one_yomitan_dict.rs @@ -10,7 +10,7 @@ fn main() -> Result<(), anyhow::Error> { { let _timer = Timer::new("remove one".to_string()); let mut writer = service.get_yomitan_writer()?; - AppService::remove_yomitan_dictionary(&mut writer, "JMnedict")?; + AppService::remove_yomitan_dictionary(&mut writer, "JMnedict.zip", "ja")?; } Ok(()) diff --git a/src-core/src/db/yomitan_writer.rs b/src-core/src/db/yomitan_writer.rs index c3713af..6f7a66a 100644 --- a/src-core/src/db/yomitan_writer.rs +++ b/src-core/src/db/yomitan_writer.rs @@ -339,7 +339,7 @@ impl YomitanWriter { if let Some(row) = rows.next()? { let is_outdated: bool = row.get(0)?; if is_outdated { - self.drop_dictionary(title)?; + self.drop_dictionary(bundle_name, lang)?; } else { return Ok(YomitanZipImportResult { exists: true, @@ -608,9 +608,11 @@ impl YomitanWriter { }) } - pub fn drop_dictionary(&self, title: &str) -> Result<()> { - self.conn - .execute("DELETE FROM dictionaries WHERE title = ?1", [title])?; + pub fn drop_dictionary(&self, bundle_name: &str, lang: &str) -> Result<()> { + self.conn.execute( + "DELETE FROM dictionaries WHERE bundle_name = ?1 AND lang = ?2", + [bundle_name, lang], + )?; Ok(()) } diff --git a/src-core/src/error.rs b/src-core/src/error.rs index fffccbc..f744f0c 100644 --- a/src-core/src/error.rs +++ b/src-core/src/error.rs @@ -25,6 +25,9 @@ pub enum CJDicError { #[error("Not found")] NotFound, + + #[error("Error: {0}")] + Error(String), } impl From for CJDicError { diff --git a/src-core/src/service.rs b/src-core/src/service.rs index 2464172..961e7de 100644 --- a/src-core/src/service.rs +++ b/src-core/src/service.rs @@ -228,8 +228,9 @@ impl AppService { pub fn remove_yomitan_dictionary( writer: &mut YomitanWriter, - title: &str, + bundle_name: &str, + lang: &str, ) -> Result<(), CJDicError> { - Ok(writer.drop_dictionary(title)?) + Ok(writer.drop_dictionary(bundle_name, lang)?) } } diff --git a/src-react/src/components/LoadingDialog.tsx b/src-react/src/components/LoadingDialog.tsx index ca24191..8fac9c8 100644 --- a/src-react/src/components/LoadingDialog.tsx +++ b/src-react/src/components/LoadingDialog.tsx @@ -3,12 +3,8 @@ import { useEffect, useRef, useState } from "react"; import { Dialog, Portal } from "@chakra-ui/react"; import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; -import { - BaseDirectory, - exists, - readDir, - writeFile, -} from "@tauri-apps/plugin-fs"; +import { BaseDirectory, exists } from "@tauri-apps/plugin-fs"; +import { fetch } from "@tauri-apps/plugin-http"; import { supabase } from "../lib/supabaseClient"; @@ -24,6 +20,13 @@ interface YomitanZipImportProgress { steps: number; } +interface DownloadProgress { + url: string; + filepath: string; + content_length: number; + downloaded: number; +} + interface Progress { message: string; current?: number; @@ -39,28 +42,102 @@ function LoadingDialog() { if (isInit.current) return; isInit.current = true; - invoke("init_yomitan").then(async () => { - const { data, error } = await supabase.storage.from("yomitan").list("ja"); - if (error) throw error; + downloadAssets(); + async function downloadAssets(lang = "ja") { + const new_dicts = new Set(); + // const to_be_removed_dicts = new Set(); - for (const d of data || []) { - const filepath = `yomitan/ja/${d.name}`; - if ( - await exists(filepath, { - baseDir: BaseDirectory.AppData, - }) - ) - continue; + if (lang === "ja") { + await getPixiv().catch((e) => { + console.error(e); + }); + async function getPixiv() { + const filename = "Pixiv.zip"; + new_dicts.add(filename); - const { - data: { publicUrl }, - } = supabase.storage.from("yomitan").getPublicUrl(`ja/${d.name}`); + const filepath = `yomitan/${lang}/${filename}`; + if ( + !(await exists(filepath, { + baseDir: BaseDirectory.AppData, + })) + ) { + const publicUrl = await ghLatestReleaseURL( + "MarvNC/pixiv-yomitan", + /^Pixiv_.+\.zip$/, + ); - // await downloadURL(publicUrl, filepath, (received, total) => {}); + if (publicUrl) { + await invoke("download_url", { url: publicUrl, filepath }); + } + } + } + } + + let isSupabase = true; + await getSupabase().catch((e) => { + console.error(e); + isSupabase = false; + }); + async function getSupabase() { + const { data, error } = await supabase.storage + .from("yomitan") + .list(lang); + if (error) throw error; + + for (const d of data || []) { + new_dicts.add(d.name); + const filepath = `yomitan/${lang}/${d.name}`; + if ( + !(await exists(filepath, { + baseDir: BaseDirectory.AppData, + })) + ) { + const { + data: { publicUrl }, + } = supabase.storage + .from("yomitan") + .getPublicUrl(`${lang}/${d.name}`); + + await invoke("download_url", { url: publicUrl, filepath }); + } + } + } + + if (isSupabase) { + // console.log([...new_dicts], [...to_be_removed_dicts]); + // for (const f of await readDir(`yomitan/${lang}`, { + // baseDir: BaseDirectory.AppData, + // })) { + // if (f.isFile && f.name.endsWith(".zip")) { + // if (new_dicts.has(f.name)) { + // new_dicts.delete(f.name); + // } else { + // to_be_removed_dicts.add(f.name); + // } + // } + // } + // for (const bundleName of Array.from(new_dicts).sort()) { + // updateProgress({ + // message: `Importing ${bundleName}`, + // }); + // await invoke("import_yomitan_dict", { + // bundleName, + // lang, + // }); + // } + // for (const bundleName of Array.from(to_be_removed_dicts).sort()) { + // updateProgress({ + // message: `Removing old ${bundleName}`, + // }); + // await invoke("remove_yomitan_dict", { + // bundleName, + // lang, + // }); + // } } setMessages([]); - }); + } listen("load-yomitan-dir", ({ payload }) => { setMessages((messages) => { @@ -87,20 +164,32 @@ function LoadingDialog() { listen( "yomitan-import-progress", ({ payload }) => { - setMessages((messages) => { - const prev = messages[messages.length - 1]; - if (prev?.message === payload.message) { - messages = messages.slice(0, messages.length - 1); - } - - messages = [...messages, payload]; - - return messages; - }); + updateProgress(payload); }, ); + + listen("download-url-progress", ({ payload }) => { + updateProgress({ + message: `Downloading ${payload.filepath} (MiB)`, + current: payload.downloaded >> 20, + total: payload.content_length >> 20, + }); + }); }, []); + function updateProgress(payload: Progress) { + setMessages((messages) => { + const prev = messages[messages.length - 1]; + if (prev?.message === payload.message) { + messages = messages.slice(0, messages.length - 1); + } + + messages = [...messages, payload]; + + return messages; + }); + } + return ( 0}> @@ -133,48 +222,21 @@ function LoadingDialog() { export default LoadingDialog; -async function downloadURL( - url: string, - filepath: string, - progressCallback: (received: number, total: number) => void, -) { - const r = await fetch(url); - if (!r.ok) throw r; - if (!r.body) throw r; - - const contentLength = r.headers.get("Content-Length"); - const total = contentLength ? parseInt(contentLength) : 0; - - let receivedBytes = 0; - - const reader = r.body.getReader(); - const stream = new ReadableStream({ - start(controller) { - function push() { - reader - .read() - .then(({ done, value }) => { - if (done) { - controller.close(); - return; - } - receivedBytes += value.length; - - // Calculate and log progress - if (total) progressCallback(receivedBytes, total); - - controller.enqueue(value); - push(); // Read the next chunk - }) - .catch((error) => { - console.error("Stream reading error:", error); - controller.error(error); - }); - } - - push(); // Start reading the data +/** + * @see https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release + */ +async function ghLatestReleaseURL(user_repo: string, assetName: RegExp) { + const releaseData: { + assets: { name: string; browser_download_url: string }[]; + } = await fetch(`https://api.github.com/repos/${user_repo}/releases/latest`, { + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "cjdic2", }, - }); + }).then((r) => r.json()); - await writeFile(filepath, stream, { baseDir: BaseDirectory.AppData }); + const a = releaseData.assets.find((a) => assetName.test(a.name)); + if (!a) return; + return a.browser_download_url; } diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index f2b94ae..a5786b9 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -25,3 +25,5 @@ tauri-plugin-http = "2" anyhow = "1.0.102" tauri-plugin-fs = "2" serde_json = "1.0.149" +reqwest = { version = "0.13.2", features = ["stream"] } +serde = "1.0.228" diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index fb651f8..5ce2f18 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -17,12 +17,13 @@ ] }, "fs:allow-exists", - "fs:allow-appdata-write", - "fs:allow-appdata-meta", + "fs:allow-read-dir", + "fs:allow-appdata-read", { "identifier": "fs:scope", "allow": [ - "$APPDATA/**/*.zip" + "$APPDATA/**/*.zip", + "$APPDATA/yomitan/*" ], "deny": [ "$APPDATA/*.db" diff --git a/src-tauri/src/command.rs b/src-tauri/src/command.rs index d87f3e1..eb56231 100644 --- a/src-tauri/src/command.rs +++ b/src-tauri/src/command.rs @@ -1,6 +1,10 @@ -use std::{borrow::Cow, path::PathBuf}; +use std::{borrow::Cow, fs::File, io::Write, path::PathBuf}; -use cjdic2_core::{AppService, CJDicError, SqlParam, YomitanRow, ZipSource}; +use cjdic2_core::{ + AppService, CJDicError, SqlParam, YomitanRow, YomitanZipImportResult, ZipSource, +}; +use reqwest::Client; +use serde::Serialize; use tauri::{AppHandle, Emitter, Manager, Runtime, path::BaseDirectory}; use tauri_plugin_fs::FsExt; @@ -52,7 +56,7 @@ pub async fn init_yomitan( let zip_dir = app .path() .resolve(folder_name, tauri::path::BaseDirectory::Resource) - .map_err(|e| CJDicError::AnyhowError(e.to_string()))?; + .map_err(|e| CJDicError::Error(e.to_string()))?; // Manifest is needed, as Android bundled fs can't read_dir let files: Vec = @@ -62,7 +66,7 @@ pub async fn init_yomitan( .iter() .map(|f| BundledZip::new(app.clone(), &format!("{folder_name}/{f}"))) .collect::, _>>() - .map_err(|e| CJDicError::AnyhowError(e.to_string()))?; + .map_err(|e| CJDicError::Error(e.to_string()))?; if zip_files.is_empty() { continue; @@ -94,6 +98,37 @@ pub async fn init_yomitan( Ok(()) } +#[tauri::command] +pub async fn import_yomitan_dict( + app: AppHandle, + bundle_name: &str, + lang: &str, + state: tauri::State<'_, AppService>, +) -> Result { + let mut data_dir = app + .path() + .app_data_dir() + .map_err(|e| CJDicError::Error(e.to_string()))?; + data_dir.push("yomitan"); + data_dir.push(lang); + data_dir.push(bundle_name); + + let mut writer = state.get_yomitan_writer()?; + AppService::import_yomitan_zip_file(&mut writer, &data_dir, lang, |r| { + app.emit("yomitan-import-progress", r).unwrap(); + }) +} + +#[tauri::command] +pub async fn remove_yomitan_dict( + bundle_name: &str, + lang: &str, + state: tauri::State<'_, AppService>, +) -> Result<(), CJDicError> { + let mut writer = state.get_yomitan_writer()?; + AppService::remove_yomitan_dictionary(&mut writer, bundle_name, lang) +} + #[tauri::command] pub async fn search_yomitan( q_term: &str, @@ -113,3 +148,65 @@ pub async fn execute_sql( ) -> Result { state.execute_sql(sql, params) } + +#[derive(Serialize, Clone)] +struct DownloadProgress { + url: String, + filepath: String, + content_length: u64, + downloaded: u64, +} + +#[tauri::command] +pub async fn download_url( + app: AppHandle, + url: String, + filepath: String, +) -> Result { + let client = Client::new(); + let data_dir = app + .path() + .app_data_dir() + .map_err(|e| CJDicError::Error(e.to_string()))?; + let file_pf = data_dir.join(filepath.as_str()); + + // Making the asynchronous GET request + let mut response = client + .get(url.as_str()) + .send() + .await + .map_err(|e| CJDicError::Error(e.to_string()))?; + + // Check if the request was successful + if response.status().is_success() { + let content_length = response.content_length().unwrap_or(0); + let mut file = File::create(&file_pf).map_err(|e| CJDicError::Error(e.to_string()))?; + + // Downloading the file in chunks + let mut downloaded: u64 = 0; + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| CJDicError::Error(e.to_string()))? + { + file.write_all(&chunk)?; + downloaded += chunk.len() as u64; + + app.emit( + "download-url-progress", + DownloadProgress { + url: url.clone(), + filepath: filepath.clone(), + content_length, + downloaded, + }, + ) + .unwrap(); + } + println!("File downloaded successfully to {:?}", &file_pf); + } else { + eprintln!("Failed to download file: {:?}", response.status()); + } + + Ok(true) +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index e00db29..426e296 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -15,20 +15,24 @@ pub fn run() { .setup(|app| { let app_dir = app.path().app_config_dir()?; create_dir_all(&app_dir)?; - // println!("{:?}", app_dir); // db_path is app_dir; let db_dir = app_dir.as_path(); - // let db_dir = Path::new(r"D:\Projects\cjdic2\tmp\save-db"); let service = AppService::new(&db_dir)?; app.manage(service); + let app_data_dir = app.path().app_data_dir()?; + create_dir_all(&app_data_dir.join("yomitan/ja"))?; + Ok(()) }) .invoke_handler(tauri::generate_handler![ init_yomitan, + import_yomitan_dict, + remove_yomitan_dict, search_yomitan, - execute_sql + execute_sql, + download_url, ]) .build(tauri::generate_context!()) .expect("error while running tauri application")