fix fts prefix ~

package.json

@@ -1,7 +1,6 @@
 {
   "name": "cjdic2",
   "private": true,
-  "version": "0.1.1",
   "type": "module",
   "scripts": {
     "dev:desktop": "cross-env RUST_BACKTRACE=1 tauri dev",

src-core/src/db.rs

@@ -18,13 +18,14 @@ pub use yomitan::YomitanRow;
 mod yomitan_writer;
 pub use yomitan_writer::{YomitanProgress, YomitanWriter, YomitanZipImportResult, ZipSource};
 
-use crate::CJDicError;
+use crate::{CJDicError, tokenizer::Tokenizer};
 
 #[derive(Clone)]
 pub(crate) struct Database {
     pub(crate) conn: Arc<Mutex<Connection>>,
     pub(crate) dir: PathBuf,
     is_db_attached: EnumMap<DbChild, Arc<AtomicBool>>,
+    tokenizer: Tokenizer,
 }
 
 #[derive(Debug, Enum, Clone, Copy)]
@@ -52,7 +53,7 @@ pub(crate) static DBSCHEMA: LazyLock<EnumMap<DbChild, &str>> = LazyLock::new(||
 });
 
 impl Database {
-    pub fn new(db_dir: impl AsRef<Path>) -> Result<Self, CJDicError> {
+    pub fn new(db_dir: impl AsRef<Path>, tokenizer: Tokenizer) -> Result<Self, CJDicError> {
         let dir = db_dir.as_ref().to_path_buf();
         let conn = Connection::open(dir.join(USER_DBFILE))?;
 
@@ -73,6 +74,7 @@ impl Database {
                 DbChild::YomitanGlossary => Arc::new(AtomicBool::new(false)),
                 DbChild::Search => Arc::new(AtomicBool::new(false)),
             },
+            tokenizer,
         })
     }
 
@@ -80,7 +82,10 @@ impl Database {
         self.ensure_db_attached(DbChild::Yomitan)?;
         self.ensure_db_attached(DbChild::YomitanGlossary)?;
         self.ensure_db_attached(DbChild::Search)?;
-        Ok(yomitan::YomitanDatabase::new(self.clone()))
+        Ok(yomitan::YomitanDatabase::new(
+            self.clone(),
+            self.tokenizer.clone(),
+        ))
     }
 
     fn ensure_db_attached(&self, db_child: DbChild) -> Result<(), CJDicError> {

src-core/src/db/search.rs

@@ -1,5 +1,3 @@
-use std::collections::HashSet;
-
 use rusqlite::{Connection, Transaction, params};
 use wana_kana::utils::katakana_to_hiragana;
 
@@ -114,8 +112,6 @@ impl<'a> SearchDatabase<'a> {
                 DBSCHEMA[DbChild::Search]
             ))?;
 
-            let exclude_pos =
-                HashSet::from(["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()));
             let mut stmt_write_ft = tx.prepare(&format!(
                 "INSERT INTO {}.terms_ft (rowid, term) VALUES (?1, ?2)",
                 DBSCHEMA[DbChild::Search]
@@ -130,28 +126,7 @@ impl<'a> SearchDatabase<'a> {
                     max_score
                 ])?;
 
-                let mut segmented: Vec<String> = vec![];
-                for t in tokenizer.tokenize(term) {
-                    let mut base = t.surface;
-                    let mut pos = None;
-
-                    for (i, dt) in t.details.iter().enumerate() {
-                        match i {
-                            0 => base = dt.to_string(),
-                            6 => pos = Some(dt),
-                            _ => {}
-                        }
-                    }
-
-                    if let Some(p) = pos {
-                        if exclude_pos.contains(p.as_str()) {
-                            continue;
-                        }
-                    }
-                    segmented.push(base);
-                }
-
-                stmt_write_ft.execute(params![id, segmented.join(" ")])?;
+                stmt_write_ft.execute(params![id, tokenizer.ja_normalize(term).join(" ")])?;
 
                 if i % 10_000 == 0 {
                     progress_callback(YomitanProgress {

src-core/src/db/yomitan.rs

@@ -5,6 +5,7 @@ use zstd::bulk::Decompressor;
 use crate::{
     CJDicError, Timer,
     db::{Database, yomitan_writer::normalize_term},
+    tokenizer::Tokenizer,
 };
 
 #[derive(Serialize, Debug, Clone)]
@@ -22,11 +23,12 @@ pub struct YomitanRow {
 
 pub struct YomitanDatabase {
     db: Database,
+    tokenizer: Tokenizer,
 }
 
 impl YomitanDatabase {
-    pub(crate) fn new(db: Database) -> Self {
-        Self { db }
+    pub(crate) fn new(db: Database, tokenizer: Tokenizer) -> Self {
+        Self { db, tokenizer }
     }
 
     fn decompressor(&self) -> Result<Decompressor<'_>, CJDicError> {
@@ -65,8 +67,10 @@ impl YomitanDatabase {
         // max decompressed: 90KB  (compression ratio on largest: ~2.5x)
         const MAX_DECOMPRESSED_SIZE: usize = 512 * 1024; // 512KB, ~5x headroom
 
-        let and_or = if q_term == q_reading { "OR" } else { "AND" };
-
+        let is_fts = q_term.starts_with('~');
+        let q_term = if is_fts {
+            normalize_term(q_term)
+        } else {
             let mut new_t = String::new();
             let mut new_seg = String::new();
 
@@ -82,21 +86,58 @@ impl YomitanDatabase {
             }
 
             new_t.push_str(&normalize_term(&new_seg));
-        let q_term_norm = new_t;
+            new_t
+        };
 
-        let eq1 = if q_term_norm.len() > 0 { "GLOB" } else { "=" };
-        let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+        let and_or = if q_term == q_reading { "OR" } else { "AND" };
 
         let conn = self.db.conn.lock()?;
 
-        let mut stmt = conn.prepare(&format!(
+        conn.execute_batch(
             "
-            WITH s AS (
-                SELECT * FROM search.terms
+            CREATE TEMP TABLE IF NOT EXISTS search_ids (term_id INTEGER);
+            DELETE FROM search_ids;
+            ",
+        )?;
+
+        if is_fts {
+            let q = self.tokenizer.ja_normalize(q_term.to_string()).join(" ");
+            println!("fts: {q_term} => {q}");
+
+            if q.len() > 0 {
+                conn.execute(
+                    &format!(
+                        "
+                    INSERT INTO search_ids (term_id)
+                    SELECT rowid FROM search.terms_ft
+                    WHERE term MATCH ?1
+                    ORDER BY rank
+                    LIMIT ?2 OFFSET ?3
+                "
+                    ),
+                    params![q, limit, offset],
+                )?;
+            }
+        } else {
+            let eq1 = if q_term.len() > 0 { "GLOB" } else { "=" };
+            let eq2 = if q_reading.len() > 0 { "GLOB" } else { "=" };
+
+            conn.execute(
+                &format!(
+                    "
+                INSERT INTO search_ids (term_id)
+                SELECT id FROM search.terms
                 WHERE term {eq1} ?1 {and_or} reading {eq2} ?2
                 ORDER BY max_score DESC
                 LIMIT ?3 OFFSET ?4
-            )
+            "
+                ),
+                params![q_term, q_reading, limit, offset],
+            )?;
+        }
+
+        let mut stmt = conn.prepare(&format!(
+            "
             SELECT
                 t.term,
                 t.reading,
@@ -107,8 +148,8 @@ impl YomitanDatabase {
                 t.sequence,
                 COALESCE(tt.tags,  '')  AS term_tags,
                 d.title                 AS dict_title
-            FROM s
-            JOIN yomitan.terms              t  ON t.id  = s.id
+            FROM search_ids                 s
+            JOIN yomitan.terms              t  ON t.id  = s.term_id
             LEFT JOIN yomitan.dictionaries  d  ON d.id  = t.dict_id
             LEFT JOIN yomitan.def_tag_sets  dt ON dt.id = t.def_tags_id
             LEFT JOIN yomitan.rule_sets     r  ON r.id  = t.rules_id
@@ -116,7 +157,7 @@ impl YomitanDatabase {
             LEFT JOIN glossary.glossaries   g  ON g.id  = t.glossary_id
         "
         ))?;
-        let rows = stmt.query_map(params![q_term_norm, q_reading, limit, offset], |r| {
+        let rows = stmt.query_map([], |r| {
             Ok((
                 YomitanRow {
                     term: r.get(0)?,

src-core/src/lib.rs

@@ -9,7 +9,7 @@ mod util;
 
 pub use db::{YomitanProgress, YomitanRow, YomitanZipImportResult, ZipSource};
 pub use error::CJDicError;
-pub use service::{AppService, SqlParam};
+pub use service::{AppService, SqlParam, YomitanDictEntry};
 pub use tokenizer::TokenizeSegment;
 pub use util::Timer;
 

src-core/src/service.rs

@@ -16,6 +16,14 @@ use crate::{
     tokenizer::{TokenizeSegment, Tokenizer},
 };
 
+#[derive(Serialize, Debug)]
+pub struct YomitanDictEntry {
+    pub title: String,
+    pub bundle_name: String,
+    pub revision: String,
+    pub lang: String,
+}
+
 #[derive(Serialize, Debug, Clone)]
 pub struct LoadYomitanZipDirResult {
     pub new_dicts: Vec<String>,
@@ -42,8 +50,9 @@ impl AppService {
         db_dir: impl AsRef<Path>,
         vibrato_dic: impl AsRef<Path>,
     ) -> Result<Self, CJDicError> {
-        let db = Database::new(db_dir)?;
         let tokenizer = Tokenizer::new(vibrato_dic)?;
+        let db = Database::new(db_dir, tokenizer.clone())?;
+
         Ok(Self { db, tokenizer })
     }
 
@@ -125,6 +134,35 @@ impl AppService {
         Ok(writer)
     }
 
+    pub fn list_yomitan_dict(&self) -> Result<Vec<YomitanDictEntry>, CJDicError> {
+        let db_path = self.db.dir.join(DBFILE[DbChild::Yomitan]);
+        let mut out = vec![];
+
+        if db_path.exists() {
+            let conn = Connection::open(&db_path)?;
+            let mut stmt = conn.prepare(
+                "
+            SELECT title, bundle_name, revision, lang
+            FROM dictionaries
+            WHERE bundle_name IS NOT NULL",
+            )?;
+            let rows = stmt.query_map([], |r| {
+                Ok(YomitanDictEntry {
+                    title: r.get(0)?,
+                    bundle_name: r.get(1)?,
+                    revision: r.get(2)?,
+                    lang: r.get(3)?,
+                })
+            })?;
+
+            for r in rows {
+                out.push(r?);
+            }
+        }
+
+        Ok(out)
+    }
+
     pub fn load_yomitan_zip_dir<Z, LoadCallback, ImportCallback>(
         &self,
         zip_dir: Vec<Z>,

src-core/src/tokenizer.rs

@@ -1,4 +1,4 @@
-use std::{fs::File, io::BufReader, path::Path};
+use std::{collections::HashSet, fs::File, io::BufReader, path::Path};
 
 use serde::Serialize;
 
@@ -25,6 +25,7 @@ pub struct TokenizeSegment {
 #[derive(Clone)]
 pub struct Tokenizer {
     ja_tokenizer: vibrato_rkyv::Tokenizer,
+    ja_exclude_pos: HashSet<String>,
 }
 
 impl Tokenizer {
@@ -33,6 +34,9 @@ impl Tokenizer {
         let dictionary = vibrato_rkyv::Dictionary::read(reader)?;
         Ok(Self {
             ja_tokenizer: vibrato_rkyv::Tokenizer::new(dictionary),
+            ja_exclude_pos: HashSet::from(
+                ["助詞", "助動詞", "記号", "接続詞"].map(|p| p.to_string()),
+            ),
         })
     }
 
@@ -56,4 +60,34 @@ impl Tokenizer {
 
         output
     }
+
+    pub fn ja_normalize(&self, text: String) -> Vec<String> {
+        let mut segmented: Vec<String> = vec![];
+        for t in self.ja_tokenize(text) {
+            let mut surface = t.surface;
+            let mut pos = None;
+
+            for (i, dt) in t.details.iter().enumerate() {
+                match i {
+                    0 => pos = Some(dt),
+                    6 => {
+                        let base = dt.to_string();
+                        if base != "*" {
+                            surface = base
+                        }
+                    }
+                    _ => {}
+                }
+            }
+
+            if let Some(p) = pos {
+                if self.ja_exclude_pos.contains(p.as_str()) {
+                    continue;
+                }
+            }
+            segmented.push(surface);
+        }
+
+        segmented
+    }
 }

src-react/public/vite.svg

@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="512" height="512" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

src-react/src/components/LoadingDialog.tsx

@@ -7,6 +7,13 @@ import { listen } from "@tauri-apps/api/event";
 import { supabase } from "../lib/supabaseClient";
 
 interface YomitanDictEntry {
+  title: string;
+  bundle_name: string;
+  revision: string;
+  lang: string;
+}
+
+interface YomitanDownloadDictEntry {
   url: string;
   filepath: string;
 }
@@ -66,13 +73,17 @@ function LoadingDialog() {
       }
     });
     async function downloadAssets(lang = "ja") {
-      const dicts: YomitanDictEntry[] = [];
+      const existing_dicts =
+        await invoke<YomitanDictEntry[]>("list_yomitan_dict");
+
+      const dicts: YomitanDownloadDictEntry[] = [];
 
       if (lang === "ja") {
-        {
         const filename = "Pixiv.zip";
         const filepath = `yomitan/${lang}/${filename}`;
-
+        const existing = existing_dicts.find((d) => d.bundle_name === filename);
+        if (!existing) {
+          // Maybe check revision update
           dicts.push({
             filepath,
             url:
@@ -84,6 +95,11 @@ function LoadingDialog() {
                 return "";
               })) || "",
           });
+        } else {
+          dicts.push({
+            filepath,
+            url: "",
+          });
         }
 
         {

src-react/src/pages/SearchPage.tsx

@@ -53,6 +53,7 @@ function SearchPage() {
   const [searchTimeout, set_searchTimeout] = useState(0);
   const [furigana, set_furigana] = useState("");
   const [isAutoKana, set_isAutoKana] = useState(true);
+  const [isScrollEnd, set_isScrollEnd] = useState(false);
 
   const nSearch = useRef(0);
   const searchboxRef = useRef<HTMLInputElement | null>(null);
@@ -114,6 +115,9 @@ function SearchPage() {
     if (isAutoKana) {
       norm_q = norm_q.replace(/n$/i, "ん").replace(/[a-z]$/i, "");
     }
+
+    norm_q = norm_q.replace(/＊/g, "*").replace(/？/g, "?").replace(/^〜/, "~");
+
     if (!norm_q) return;
 
     const ender =
@@ -149,9 +153,12 @@ function SearchPage() {
     });
 
     if (isNew) {
-      if (!result.length && !isFromSplit && /^[^ ]{3,} $/.test(q)) {
+      const [s1, prefix = "", s2 = ""] = q.split(/^([\*\?\[\]\~]*)/);
+      const text = s1 || s2;
+
+      if (!result.length && !isFromSplit && /^[^ ]{3,} $/.test(text)) {
         const segs = await invoke<{ surface: string }[]>("tokenize", {
-          text: q,
+          text,
         });
         if (segs.length > 1) {
           const newQ =
@@ -160,13 +167,14 @@ function SearchPage() {
             segs.map((s) => s.surface).join("") +
             " ";
 
-          searchParams_setQ(newQ);
+          searchParams_setQ(prefix + newQ);
           return;
         }
       }
 
       if (resultScrollRef.current) {
         resultScrollRef.current.scrollTop = 0;
+        set_isScrollEnd(false);
       }
 
       set_entries(result);
@@ -181,18 +189,19 @@ function SearchPage() {
     if (!(target instanceof HTMLElement)) return;
     const { scrollHeight, scrollTop } = target;
 
-    if (scrollHeight - 20 > scrollTop + target.getBoundingClientRect().height)
+    if (
+      isScrollEnd ||
+      scrollHeight - window.innerHeight / 2 >
+        scrollTop + target.getBoundingClientRect().height
+    )
       return;
 
     const results = await runSearch(false);
     if (!results?.length) {
       target.style.paddingBottom = "50vh";
+      set_isScrollEnd(true);
       return;
     }
-
-    setTimeout(() => {
-      target.scrollTop = scrollTop + 100;
-    }, 100);
   };
 
   function onSearchboxChange(q: string) {
@@ -368,7 +377,11 @@ function SearchPage() {
                   <Card.Header display={"block"}>
                     <a
                       onClick={() =>
-                        onTermClicked(reading ? `<${term}>{${reading}}` : term)
+                        onTermClicked(
+                          reading && reading !== term
+                            ? `<${term}>{${reading}}`
+                            : term,
+                        )
                       }
                     >
                       {term}
@@ -380,6 +393,9 @@ function SearchPage() {
                         {"）"}
                       </>
                     ) : null}
+                    <button onClick={() => onTermClicked("~" + term)}>
+                      🔗
+                    </button>
                     <span>【{dict_title}】</span>
                     <span>{JSONdumpClean(it)}</span>
                   </Card.Header>

src-tauri/gen/android/app/.gitignore

@@ -9,3 +9,4 @@
 /src/main/assets/**/*.zst
 /src/main/assets/**/*.db
 /src/main/assets/**/manifest.json
+!/src/debug

src-tauri/gen/android/app/src/debug/res/mipmap-anydpi-v26/ic_launcher.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+  <foreground android:drawable="@mipmap/ic_launcher_foreground"/>
+  <background android:drawable="@color/ic_launcher_background"/>
+</adaptive-icon>

src-tauri/gen/android/app/src/debug/res/values/ic_launcher_background.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+  <color name="ic_launcher_background">#fff</color>
+</resources>

src-tauri/gen/android/app/src/debug/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name">cjdic2 (debug)</string>
+    <string name="main_activity_title">cjdic2 (debug)</string>
+</resources>

src-tauri/src/command.rs

@@ -5,7 +5,8 @@ use std::{
 };
 
 use cjdic2_core::{
-    AppService, CJDicError, SqlParam, TokenizeSegment, YomitanRow, YomitanZipImportResult,
+    AppService, CJDicError, SqlParam, TokenizeSegment, YomitanDictEntry, YomitanRow,
+    YomitanZipImportResult,
 };
 use serde::{Deserialize, Serialize};
 use tauri::{AppHandle, Emitter, Manager};
@@ -13,8 +14,15 @@ use tauri_plugin_http::reqwest::Client;
 use tokio::io::AsyncWriteExt;
 use zip::ZipArchive;
 
+#[tauri::command]
+pub async fn list_yomitan_dict(
+    state: tauri::State<'_, AppService>,
+) -> Result<Vec<YomitanDictEntry>, CJDicError> {
+    state.list_yomitan_dict()
+}
+
 #[derive(Deserialize)]
-pub struct YomitanDictEntry {
+pub struct YomitanDownloadDictEntry {
     url: String,
     filepath: String,
 }
@@ -22,7 +30,7 @@ pub struct YomitanDictEntry {
 #[tauri::command]
 pub async fn init_yomitan(
     app: AppHandle,
-    dicts: Vec<YomitanDictEntry>,
+    dicts: Vec<YomitanDownloadDictEntry>,
     lang: String,
     state: tauri::State<'_, AppService>,
 ) -> Result<(), CJDicError> {

src-tauri/src/lib.rs

@@ -47,6 +47,7 @@ pub fn run() {
             Ok(())
         })
         .invoke_handler(tauri::generate_handler![
+            list_yomitan_dict,
             init_yomitan,
             import_yomitan_dict,
             remove_yomitan_dict,

src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://schema.tauri.app/config/2",
   "productName": "cjdic2",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "identifier": "cc.polv.cjdic2",
   "build": {
     "beforeDevCommand": "pnpm dev",