diff --git a/.github/workflows/android-apk.yml b/.github/workflows/android-apk.yml new file mode 100644 index 00000000..a4156250 --- /dev/null +++ b/.github/workflows/android-apk.yml @@ -0,0 +1,241 @@ +name: Android APK (debug) + +# Triggers: +# - push v*-tauri tag → build debug APK, upload artifact + attach to GitHub Release +# - workflow_dispatch → build debug APK, upload artifact only (no release) +# +# Scope: full overlay/accessibility APK for ADB testing (v1 RECORD_AUDIO + overlay manifest merge). + +on: + push: + tags: + - 'v*-tauri' + workflow_dispatch: + +jobs: + build-android-apk: + permissions: + contents: write + runs-on: ubuntu-latest + env: + CI: true + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Java 17 (Zulu) + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: '17' + + - name: Setup Android SDK + uses: android-actions/setup-android@v3 + with: + packages: platform-tools + + - name: Install NDK and accept SDK licenses + shell: bash + run: | + set -euo pipefail + sdkmanager "ndk;26.1.10909125" + ndk_dir="$ANDROID_HOME/ndk/26.1.10909125" + if [ ! -d "$ndk_dir" ]; then + echo "::error::NDK not found at $ndk_dir" + exit 1 + fi + echo "ANDROID_NDK_HOME=$ndk_dir" >> "$GITHUB_ENV" + echo "NDK_HOME=$ndk_dir" >> "$GITHUB_ENV" + set +o pipefail + yes | sdkmanager --licenses + sdkmanager_exit=${PIPESTATUS[1]} + set -o pipefail + if [ "$sdkmanager_exit" -ne 0 ]; then + echo "::error::sdkmanager --licenses failed with exit code $sdkmanager_exit" + exit "$sdkmanager_exit" + fi + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: openless-all/app/package-lock.json + + - uses: dtolnay/rust-toolchain@stable + with: + targets: >- + aarch64-linux-android, + armv7-linux-androideabi, + i686-linux-android, + x86_64-linux-android + + - name: Cache Cargo + uses: swatinem/rust-cache@v2 + with: + workspaces: openless-all/app/src-tauri -> target + + - uses: gradle/actions/setup-gradle@v4 + + - name: Install npm deps + working-directory: openless-all/app + run: npm ci + + - name: Build frontend + working-directory: openless-all/app + run: npm run build + + - name: Initialize Android project + working-directory: openless-all/app + run: npm run tauri -- android init --ci + + - name: Copy Android scaffolding (Kotlin + XML) + working-directory: openless-all/app + run: node scripts/copy-android-scaffolding.mjs + + - name: Merge APK v1 manifest permissions + working-directory: openless-all/app + run: node scripts/merge-android-v1-manifest.mjs + + - name: Merge overlay / accessibility manifest + working-directory: openless-all/app + run: node scripts/merge-android-overlay-manifest.mjs + + - name: Prime Gradle wrapper + working-directory: openless-all/app + shell: bash + run: | + set -euo pipefail + for attempt in 1 2 3; do + if src-tauri/gen/android/gradlew --project-dir src-tauri/gen/android --version --no-daemon; then + exit 0 + fi + if [ "$attempt" -lt 3 ]; then + sleep $([ "$attempt" -eq 1 ] && echo 15 || echo 30) + fi + done + exit 1 + + - name: Build Android debug APK + working-directory: openless-all/app + run: npm run tauri:android:build + + - name: Free disk before artifact upload + shell: bash + working-directory: openless-all/app + run: | + set -euo pipefail + rm -rf src-tauri/target + rm -rf ~/.cargo/registry ~/.cargo/git ~/.gradle/caches + df -h + + - name: Collect split debug APKs + id: apk + shell: bash + working-directory: openless-all/app + run: | + set -euo pipefail + if [[ "${{ github.ref }}" == refs/tags/v* ]] && [[ "${{ github.ref_name }}" == *-tauri ]]; then + label="${{ github.ref_name }}" + else + label="run-${{ github.run_number }}" + fi + export OPENLESS_APK_LABEL="$label" + python - <<'PY' + import os + import shutil + import sys + import zipfile + from pathlib import Path + + expected = { + "arm64-v8a": "arm64_v8a", + "armeabi-v7a": "armeabi_v7a", + "x86": "x86", + "x86_64": "x86_64", + } + root = Path("src-tauri/gen/android") + label = os.environ["OPENLESS_APK_LABEL"] + out_dir = Path(os.environ["RUNNER_TEMP"]) / "openless-android-debug-split" + out_dir.mkdir(parents=True, exist_ok=True) + found = {} + candidates = [ + apk for apk in sorted(root.rglob("*.apk")) + if "outputs" in apk.parts + ] + if not candidates: + print("::error::No APK found under src-tauri/gen/android/**/outputs/") + for apk in sorted(root.rglob("*.apk")): + print(apk) + sys.exit(1) + for apk in candidates: + with zipfile.ZipFile(apk) as archive: + abis = sorted({ + name.split("/")[1] + for name in archive.namelist() + if name.startswith("lib/") and len(name.split("/")) >= 3 + }) + if len(abis) != 1: + print(f"::error::{apk} contains ABI directories {abis}; expected exactly one ABI per APK") + sys.exit(1) + abi = abis[0] + if abi not in expected: + print(f"::error::{apk} contains unexpected ABI {abi}") + sys.exit(1) + if abi in found: + print(f"::error::Duplicate APKs for ABI {abi}: {found[abi]} and {apk}") + sys.exit(1) + dest = out_dir / f"OpenLess-android-debug-{abi}-{label}.apk" + shutil.copy2(apk, dest) + found[abi] = dest + print(f"Collected {abi}: {apk} -> {dest}") + missing = sorted(set(expected) - set(found)) + if missing: + print(f"::error::Missing split APKs for ABI(s): {', '.join(missing)}") + sys.exit(1) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + for abi, key in expected.items(): + output.write(f"{key}_path={found[abi]}\n") + output.write("release_files<_x64-setup.exe` — run the installer. +- **Android debug APK**: choose exactly one split APK for your device: + - `OpenLess-android-debug-arm64-v8a-*.apk` for most modern Android phones. + - `OpenLess-android-debug-armeabi-v7a-*.apk` for older 32-bit ARM phones. + - `OpenLess-android-debug-x86_64-*.apk` for most 64-bit Android emulator images. + - `OpenLess-android-debug-x86-*.apk` for older 32-bit x86 emulator images. + - If unsure, run `adb shell getprop ro.product.cpu.abi` and download the APK matching that ABI. Do not install the x86 builds on a normal ARM phone. - **macOS (Homebrew)**: ```bash brew tap appergb/openless https://github.com/appergb/openless diff --git a/docs/android-mobile-apk-overlay-plan.md b/docs/android-mobile-apk-overlay-plan.md new file mode 100644 index 00000000..28fcfd9b --- /dev/null +++ b/docs/android-mobile-apk-overlay-plan.md @@ -0,0 +1,315 @@ +# OpenLess Android APK 与悬浮窗实施计划 + +> 状态:实施中 / 后端分层已落地 +> 日期:2026-06-07 +> 范围:Android APK v1(应用内录音)→ IME v2(跨 App 输入)→ 悬浮窗 v3;不改桌面语义 + +目标是把现有 OpenLess 桌面应用扩展为 Android APK,并在手机端提供可用的语音输入体验。当前项目是 React/Vite + Tauri v2 + Rust 后端,Tauri 官方支持 Android 构建;现仓库已增加 Android 分层与脚手架,桌面核心能力(全局热键、托盘、桌面浮窗、TSF IME 等)通过 `#[cfg(not(mobile))]` 收口。 + +核心策略:先产出可安装 APK,再做 Android 输入法服务,最后做跨 App 悬浮窗。桌面功能保持不受影响。 + +参考依据: +- Tauri Android 依赖:Android Studio、SDK/NDK、`ANDROID_HOME`、`NDK_HOME`、Android Rust targets。 +- Tauri Android 构建命令:`tauri android init`,`tauri android build --apk`。 +- Android 悬浮窗:Android 8.0+ 使用 `SYSTEM_ALERT_WINDOW` + `TYPE_APPLICATION_OVERLAY`。 + +--- + +## 1. 目标与非目标 + +### 目标 + +- **APK v1**:应用内主窗口、设置、历史、云端 ASR/LLM、基础录音、复制结果 +- **IME v2**:`OpenLessImeService` 作为跨 App 输入主路径 +- **悬浮窗 v3**:前台服务 + `TYPE_APPLICATION_OVERLAY`,仅作录音控制入口 +- **平台能力查询**:前端通过 `get_platform_capabilities` 隐藏桌面专属设置项 +- **桌面零破坏**:所有适配经 `#[cfg(not(mobile))]` / `#[cfg(mobile)]` 分层 + +### 非目标 + +- APK 首版不纳入本地 ASR(Foundry、Sherpa、Qwen 桌面假设) +- 首版不承诺直接写入其他 App(无 IME 时走复制兜底) +- Accessibility 跨 App 输入不作为默认路径 +- 悬浮窗不承担最终文本插入职责 +- 不改桌面现有功能语义 + +### 明确边界 + +- **不动** macOS / Windows / Linux 热键、托盘、胶囊、QA 浮窗逻辑 +- **Android 命令名与桌面一致**;不支持的命令返回明确 unavailable 状态 +- **Coordinator 听写主链路复用**;`end_session` 在 Android 增加 IME commit 分支 + +--- + +## 2. 架构定位 + +``` +openless-all/app/ + package.json # tauri:android:* scripts + vite.config.ts # TAURI_ENV_PLATFORM → 0.0.0.0 + HMR + src-tauri/ + tauri.conf.json # bundle.android.minSdkVersion ≥ 26 + tauri.android.conf.json # 单 main 窗口(无 capsule/qa/tray) + capabilities/ + default.json # 桌面 + mobile.json # Android 主窗口 + android-scaffolding/ # Kotlin 模板(init 后复制到 gen/android) + src/ + lib.rs # desktop/mobile run() 分层 + android_ime.rs # IME 状态 + commit(JNI 桩) + android_overlay.rs # 悬浮窗权限/状态(JNI 桩) + permissions.rs # Android 麦克风 runtime permission 分支 + persistence.rs # Android 凭据加密文件桩 + types.rs # PlatformCapabilities + commands.rs # get_platform_capabilities + 桌面命令 stub + coordinator/dictation.rs # end_session Android IME 分支 +``` + +平台分层示意: + +``` +┌─────────────────────────────────────────────────────────┐ +│ React UI(能力查询 → 隐藏桌面专属设置) │ +├─────────────────────────────────────────────────────────┤ +│ Tauri commands(同名 IPC;mobile 返回 unavailable stub) │ +├──────────────┬──────────────────────────────────────────┤ +│ desktop │ mobile (Android) │ +│ hotkey/tray │ in-app dictation + cloud ASR │ +│ capsule/qa │ android_ime (v2) / android_overlay(v3) │ +│ TSF/AX/粘贴 │ IME commit (v1); clipboard TBD │ +└──────────────┴──────────────────────────────────────────┘ +``` + +> **v1 剪贴板**:APK v1 不使用 Android 剪贴板兜底(未接 arboard);跨 App 文本输入依赖后续 IME/JNI 接线。 + +--- + +## 3. 模块设计 + +### 3.1 `PlatformCapabilities`(`types.rs`) + +```rust +pub struct PlatformCapabilities { + pub platform: String, + pub supports_ime_input: bool, + pub supports_overlay: bool, + pub supports_desktop_hotkey: bool, + pub supports_tray: bool, + pub supports_local_asr: bool, + pub supports_capsule_overlay: bool, +} +``` + +- Android:`supportsImeInput=true`(v2 起)、`supportsOverlay=true`(v3 起)、`supportsDesktopHotkey=false`、`supportsTray=false` +- 桌面:按 OS 填真实能力 + +### 3.2 `android_ime.rs` + +- `get_android_ime_status()` → 是否已启用 OpenLess 输入法 +- `commit_text(text)` → 通过 JNI 提交到 `InputConnection`(桩 → 日志 + 返回未连接) +- Kotlin:`OpenLessImeService` 继承 `InputMethodService` + +### 3.3 `android_overlay.rs` + +- `get_android_overlay_status()` → `SYSTEM_ALERT_WINDOW` 授权状态 +- `request_android_overlay_permission()` → 跳转 `OverlayPermissionActivity` +- Kotlin:`OpenLessOverlayService`(前台服务 + overlay window) + +### 3.4 `permissions.rs` / `persistence.rs` + +- 麦克风:Android runtime permission 分支(JNI 桩;未接线时 `NotDetermined`) +- 凭据:Android 加密 JSON 文件桩(`credentials.enc.json`),不沿用桌面 keyring + +### 3.5 `coordinator/dictation.rs` + +`end_session` 插入阶段: + +``` +#[cfg(target_os = "android")] + android_ime::commit_text → Inserted + 失败 → copy_fallback → CopiedFallback +#[cfg(not(mobile))] + 现有 Windows TSF / AX / paste 路径 +``` + +--- + +## 4. 原生层接口 + +| 组件 | 职责 | +|---|---| +| `MainActivity` | Tauri WebView + 权限状态桥接 | +| `OpenLessImeService` | 接收识别结果,`commitText` 到当前输入框 | +| `OpenLessOverlayService` | 悬浮窗开始/停止录音、显示状态 | +| `OverlayPermissionActivity` | 引导用户授权 `SYSTEM_ALERT_WINDOW` | + +Rust ↔ Kotlin 通信:Tauri mobile plugin / `jni`(脚手架阶段为桩,init 后接线)。 + +--- + +## 5. 实施里程碑 + +### M0 环境与文档(本期) + +- 扩展本计划文档(元数据、架构、文件表、风险) +- `package.json`:`tauri:android:init|dev|build` +- `vite.config.ts`:mobile dev host `0.0.0.0` + HMR +- `tauri.conf.json`:`bundle.android.minSdkVersion: 26` +- `tauri.android.conf.json` + `capabilities/mobile.json` + +### M1 Rust 分层 + APK 骨架(本期) + +- `lib.rs` desktop/mobile `run()` 分层 +- `Cargo.toml` gate 桌面专属依赖 +- `PlatformCapabilities` + 命令 stub +- `permissions.rs` / `persistence.rs` Android 分支 +- `cargo check` 桌面通过;Android target 尽力验证 + +### M2 IME v2 + +- 接线 `OpenLessImeService` JNI +- 设置页显示输入法启用状态 +- 跨 App 提交验收 + +### M3 悬浮窗 v3 + +- 接线 `OpenLessOverlayService` +- 授权引导 + 前台服务稳定性 + +--- + +## 6. 文件触达表 + +| 文件 | 变更 | +|---|---| +| `docs/android-mobile-apk-overlay-plan.md` | 扩展为完整实施规划(本文档) | +| `openless-all/app/package.json` | `tauri:android:*` scripts | +| `.github/workflows/android-apk.yml` | Android debug APK CI | +| `openless-all/app/scripts/merge-android-v1-manifest.mjs` | v1 manifest merge (RECORD_AUDIO) | +| `openless-all/app/vite.config.ts` | mobile dev server / HMR | +| `openless-all/app/src-tauri/tauri.conf.json` | `bundle.android` | +| `openless-all/app/src-tauri/tauri.android.conf.json` | 单 main 窗口 | +| `openless-all/app/src-tauri/capabilities/mobile.json` | Android 权限集 | +| `openless-all/app/src-tauri/Cargo.toml` | gate desktop deps | +| `openless-all/app/src-tauri/src/lib.rs` | mobile/desktop 分层 | +| `openless-all/app/src-tauri/src/types.rs` | `PlatformCapabilities` 等 | +| `openless-all/app/src-tauri/src/commands.rs` | 能力查询 + mobile stub | +| `openless-all/app/src-tauri/src/permissions.rs` | Android 麦克风 | +| `openless-all/app/src-tauri/src/persistence.rs` | Android 凭据 | +| `openless-all/app/src-tauri/src/android_ime.rs` | 新增 | +| `openless-all/app/src-tauri/src/android_overlay.rs` | 新增 | +| `openless-all/app/src-tauri/src/coordinator/dictation.rs` | `end_session` Android 分支 | +| `openless-all/app/src-tauri/android-scaffolding/*.kt` | Kotlin 模板 | + +--- + +## 7. 风险与对策 + +| 风险 | 对策 | +|---|---| +| 本机无 Android SDK / NDK | 文档记录手动脚手架;`android-scaffolding/` 供 init 后复制 | +| `global-hotkey` / `enigo` / `arboard` 无法编 Android | `Cargo.toml` `cfg(not(mobile))` gate | +| `keyring` 在 Android 不可用 | 加密文件桩 + 后续 Keystore 接线 | +| 桌面构建被 mobile 分层破坏 | 桌面 `cargo check` 为 CI 门禁;mobile 代码 `#[cfg(mobile)]` 隔离 | +| JNI 未接线时 IME/overlay 假成功 | 状态查询返回 `enabled=false`;commit 走 copy fallback | +| 多窗口配置污染移动端 | `tauri.android.conf.json` 仅声明 `main` | +| 本地 ASR 拖慢 Android 首版 | 明确排除;`supportsLocalAsr=false` | +| `SYSTEM_ALERT_WINDOW` 用户拒绝 | 设置页显示状态;悬浮窗功能降级为应用内入口 | + +--- + +## Android APK CI Workflow + +GitHub Actions workflow: [`.github/workflows/android-apk.yml`](../.github/workflows/android-apk.yml) + +### Capability platform isolation + +Desktop permissions live in `capabilities/default.json` with `"platforms": ["macOS", "windows", "linux"]`, so updater, autostart, and multi-window permissions do not apply on Android. Android uses `capabilities/mobile.json` with `"platforms": ["android"]` for the main-window permission set only (no updater/autostart). + +### Triggers + +| Trigger | Behavior | +|---|---| +| `workflow_dispatch` | Build debug APK → upload Actions artifact only | +| Push tag `v*-tauri` | Build debug APK → upload artifact **and** attach APK to the existing GitHub Release for that tag | + +Tag-triggered runs share the same `v*-tauri` convention as [`.github/workflows/release-tauri.yml`](../.github/workflows/release-tauri.yml). The Android job is independent and does not modify the desktop release workflow. + +### Debug APK policy + +- CI builds **debug** APKs (`tauri android build --apk --debug`) for faster iteration and to avoid release-signing requirements in v1. +- Actions artifact name: `openless-android-debug`. +- On-disk APK filename: + - Tag runs (`v*-tauri`): `OpenLess-android-debug-.apk` (e.g. `OpenLess-android-debug-v1.0.0-tauri.apk`) + - Manual dispatch: `OpenLess-android-debug-run-.apk` (not branch name) + +### Command chain (CI) + +```bash +cd openless-all/app +npm ci && npm run build +CI=true npm run tauri -- android init --ci +node scripts/merge-android-v1-manifest.mjs +CI=true npm run tauri:android:build +``` + +Local equivalent (after Android SDK/NDK setup): + +```bash +cd openless-all/app +npm run build +npm run tauri:android:init +npm run merge:android-v1-manifest +npm run tauri:android:build +``` + +### Manifest merge (v1 only) + +`scripts/merge-android-v1-manifest.mjs` merges **only** `RECORD_AUDIO` from `android-scaffolding/AndroidManifest.v1.snippet.xml` into the generated `src-tauri/gen/android/app/src/main/AndroidManifest.xml`. The script is idempotent (skips if permission already present). v2 (IME) and v3 (overlay) manifest snippets are **not** merged in this workflow. + +--- + +## 8. 验收标准 + +### 构建验证 + +```bash +cd openless-all/app +npm run build +cargo check --manifest-path src-tauri/Cargo.toml +# 需 Android SDK / NDK(与 CI 一致:debug APK): +npm run tauri:android:init +npm run merge:android-v1-manifest +npm run tauri:android:build +# 等价于 CI 的 debug 构建: +# CI=true npm run tauri -- android init --ci && node scripts/merge-android-v1-manifest.mjs && CI=true npm run tauri:android:build +``` + +### APK v1 + +- 首次启动进入主界面 +- 麦克风授权流程可触发 +- 应用内录音 → 云端转写 → 历史 + 复制 +- 桌面专属命令不导致前端白屏(返回 unavailable) +- 桌面 `cargo check` 仍通过 + +### IME v2 / 悬浮窗 v3 + +见原 Summary 中 Test Plan 章节(启用输入法后跨 App 提交;悬浮窗授权与前台服务稳定性)。 + +--- + +## Compatibility fixes(2026-06-07) + +- **`app_invoke_handler_mobile`**:仅保留 dictation / settings / credentials / history / cloud ASR / platform capabilities / Android IME·overlay / permissions / marketplace / style packs / mic devices;已移除 `get_hotkey_*`、`set_shortcut_recording_active` 及全部 desktop-only 命令(热键 setter、updater、local ASR、coding agent、tray 等)。前端 `ipc.ts` 在 `supportsDesktopHotkey === false` 时本地返回 stub,不再 invoke 这些命令。 +- **`mobile_stubs`**:`unicode_keystroke` 补齐 `typed_chars()` / `Partial`(与 coordinator 流式插入一致);`shortcut_binding::binding_from_legacy_trigger` 与桌面实现对齐。 +- **`Cargo.toml`**:`enigo` / `global-hotkey` / updater / single-instance / autostart 仅在 `cfg(not(mobile))`;Android 侧 `jni` + `ndk-context` 已声明。 + +--- + +## 9. 相关参考 + +- Tauri Android:https://v2.tauri.app/develop/mobile/ +- 桌面 Windows ASR 规划风格:`docs/windows-sherpa-onnx-asr-plan.md` +- 主听写链路:`openless-all/app/src-tauri/src/coordinator/dictation.rs` +- Windows IME unavailable 模式:`openless-all/app/src-tauri/src/windows_ime_profile.rs` diff --git a/openless-all/app/android/README.md b/openless-all/app/android/README.md new file mode 100644 index 00000000..30b04abc --- /dev/null +++ b/openless-all/app/android/README.md @@ -0,0 +1,79 @@ +# OpenLess Android 平台代码 + +Android 相关 Rust、Kotlin 与前端代码的统一入口。桌面端通过 `#[cfg(not(mobile))]` 分层,不受影响。 + +## 目录结构 + +```text +android/ +├── kotlin/ # Kotlin 模板(CI 复制到 gen/android/) +├── manifests/ # AndroidManifest snippet + res/xml +└── frontend/ # React 模块(Vite 别名 @android) + +src-tauri/src/android/ # Rust 运行时模块(crate::android) +``` + +## Rust(`src-tauri/src/android/`) + +| 模块 | 职责 | +|------|------| +| `jni.rs` | JNI 工具(clipboard、overlay service、accessibility) | +| `native_bridge.rs` | Kotlin ↔ Coordinator JNI 入口 | +| `overlay.rs` | 悬浮窗权限与 show/hide | +| `accessibility.rs` | 无障碍服务状态与 paste | +| `insert.rs` | 跨 App 文本插入策略 | +| `types.rs` | Android 偏好与状态类型 | + +主 crate 通过 `mod android;` 引入,常用 API 经 `crate::android::` 扁平 re-export。 + +## Kotlin(`android/kotlin/`) + +`tauri android init` 后由 [`scripts/copy-android-scaffolding.mjs`](../scripts/copy-android-scaffolding.mjs) 复制到 `src-tauri/gen/android/app/src/main/java/com/openless/app/`。 + +Manifest 合并脚本: + +- [`scripts/merge-android-v1-manifest.mjs`](../scripts/merge-android-v1-manifest.mjs) — 麦克风权限(`android/manifests/AndroidManifest.v1.snippet.xml`) +- [`scripts/merge-android-overlay-manifest.mjs`](../scripts/merge-android-overlay-manifest.mjs) — 悬浮窗 / 无障碍 + +## 前端(`android/frontend/`,别名 `@android`) + +| 路径 | 职责 | +|------|------| +| `lib/androidTypes.ts` | Android 偏好与状态 TS 类型 | +| `lib/androidIpc.ts` | overlay / accessibility Tauri invoke | +| `lib/androidMicrophonePermission.ts` | WebView 麦克风权限辅助 | +| `components/AndroidPermissionsPanel.tsx` | 设置页 Android 权限与 overlay 配置 | + +`src/lib/types.ts` 与 `src/lib/ipc.ts` 保留 re-export,现有 import 路径仍可用。 + +## 构建与 CI + +**CI(overlay / 无障碍 ADB 测试 APK)** — 合并 v1 麦克风权限 + overlay / 无障碍 manifest,用于真机 ADB 测试完整悬浮窗与无障碍能力(非仅应用内听写): + +```bash +cd openless-all/app +npm ci && npm run build +CI=true npm run tauri -- android init --ci +node scripts/copy-android-scaffolding.mjs +node scripts/merge-android-v1-manifest.mjs +node scripts/merge-android-overlay-manifest.mjs +CI=true npm run tauri:android:build +``` + +Workflow: [`.github/workflows/android-apk.yml`](../../.github/workflows/android-apk.yml) + +**本地 overlay / 无障碍开发(v3)** — 与 CI 相同的 manifest 合并链,使用本地 init / copy 脚本: + +```bash +cd openless-all/app +npm run tauri:android:init +npm run copy:android-scaffolding +node scripts/merge-android-v1-manifest.mjs +node scripts/merge-android-overlay-manifest.mjs +npm run tauri:android:build +``` + +## 相关文档 + +- [AGENTS.md](../../AGENTS.md) — 真机闪退排查 +- [docs/android-mobile-apk-overlay-plan.md](../../docs/android-mobile-apk-overlay-plan.md) — 分阶段产品计划 diff --git a/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx b/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx new file mode 100644 index 00000000..959a5565 --- /dev/null +++ b/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx @@ -0,0 +1,386 @@ +import { useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Icon } from '../../../src/components/Icon'; +import { getSettings, setSettings } from '../../../src/lib/ipc'; +import type { UserPreferences } from '../../../src/lib/types'; +import { Btn, Pill } from '../../../src/pages/_atoms'; +import { SettingRow } from '../../../src/pages/settings/shared'; +import { + getAndroidAccessibilityStatus, + getAndroidOverlayStatus, + requestAndroidAccessibilityPermission, + requestAndroidOverlayPermission, +} from '../lib/androidIpc'; +import type { + AndroidAccessibilityStatus, + AndroidInsertStrategy, + AndroidOverlayActivationMode, + AndroidOverlayCancelSwipeDirection, + AndroidOverlayLeftSwipeAction, + AndroidOverlayStatus, + AndroidOverlayTrigger, + AndroidPreferenceKey, +} from '../lib/androidTypes'; +import { + clampAndroidOverlaySize, + normalizeAndroidOverlayTrigger, +} from '../lib/androidTypes'; + +type AndroidPrefsSlice = Pick; + +function pickAndroidPrefs(settings: UserPreferences): AndroidPrefsSlice { + return { + androidInsertStrategy: settings.androidInsertStrategy, + androidOverlayTrigger: settings.androidOverlayTrigger, + androidOverlayActivationMode: settings.androidOverlayActivationMode, + androidOverlayLeftSwipeAction: settings.androidOverlayLeftSwipeAction, + androidOverlayCancelSwipeDirection: settings.androidOverlayCancelSwipeDirection, + androidOverlaySizeDp: settings.androidOverlaySizeDp, + }; +} + +let androidOverlaySaveQueue = Promise.resolve(); + +function enqueueAndroidOverlaySave(task: () => Promise): Promise { + const run = androidOverlaySaveQueue.then(task); + androidOverlaySaveQueue = run.then(() => undefined, () => undefined); + return run; +} + +function persistAndroidOverlayPrefs(patch: Partial): Promise { + return enqueueAndroidOverlaySave(async () => { + const settings = await getSettings(); + const next = { + ...settings, + ...patch, + }; + await setSettings(next); + return pickAndroidPrefs(next); + }); +} + +type AndroidPermissionsPanelMode = 'all' | 'accessibility' | 'overlayPermission' | 'overlayConfig'; + +interface AndroidPermissionsPanelProps { + mode?: AndroidPermissionsPanelMode; +} + +export function AndroidPermissionsPanel({ mode = 'all' }: AndroidPermissionsPanelProps) { + const { t } = useTranslation(); + const [androidOverlay, setAndroidOverlay] = useState(null); + const [androidAccessibility, setAndroidAccessibility] = useState(null); + const [androidPrefs, setAndroidPrefs] = useState | null>(null); + const [sizeDraft, setSizeDraft] = useState(null); + const sizeDebounceRef = useRef(null); + const sizePendingRef = useRef(false); + + const refreshAndroid = async () => { + const [overlay, accessibility, settings] = await Promise.all([ + getAndroidOverlayStatus(), + getAndroidAccessibilityStatus(), + getSettings(), + ]); + let migratedSettings = settings; + if (settings.androidOverlayTrigger === 'keyboard') { + const migratedPrefs = await persistAndroidOverlayPrefs({ + androidOverlayTrigger: normalizeAndroidOverlayTrigger(settings.androidOverlayTrigger), + }); + migratedSettings = { ...settings, ...migratedPrefs }; + } + setAndroidOverlay(overlay); + setAndroidAccessibility(accessibility); + setAndroidPrefs(pickAndroidPrefs(migratedSettings)); + }; + + useEffect(() => { + void refreshAndroid(); + const androidId = window.setInterval(refreshAndroid, 3000); + const onFocus = () => { void refreshAndroid(); }; + window.addEventListener('focus', onFocus); + return () => { + window.clearInterval(androidId); + window.removeEventListener('focus', onFocus); + }; + }, []); + + useEffect(() => { + if (sizePendingRef.current) return; + if (androidPrefs?.androidOverlaySizeDp != null) { + setSizeDraft(androidPrefs.androidOverlaySizeDp); + } + }, [androidPrefs?.androidOverlaySizeDp]); + + useEffect(() => { + return () => { + if (sizeDebounceRef.current) clearTimeout(sizeDebounceRef.current); + }; + }, []); + + const saveAndroidOverlaySize = async (value: number) => { + const clamped = clampAndroidOverlaySize(value); + try { + const nextPrefs = await persistAndroidOverlayPrefs({ androidOverlaySizeDp: clamped }); + setAndroidPrefs((prev) => (prev ? { ...prev, ...nextPrefs } : nextPrefs)); + setSizeDraft(clamped); + } catch (error) { + console.error('[android] failed to save overlay size', error); + const settings = await getSettings(); + const rolledBack = settings.androidOverlaySizeDp; + const safeDraft = rolledBack != null ? clampAndroidOverlaySize(rolledBack) : null; + setAndroidPrefs((prev) => (prev ? { ...prev, androidOverlaySizeDp: rolledBack } : null)); + setSizeDraft(safeDraft); + } finally { + sizePendingRef.current = false; + } + }; + + const scheduleAndroidOverlaySizeSave = (value: number) => { + if (sizeDebounceRef.current) clearTimeout(sizeDebounceRef.current); + sizeDebounceRef.current = window.setTimeout(() => { + sizeDebounceRef.current = null; + void saveAndroidOverlaySize(value); + }, 200); + }; + + const flushAndroidOverlaySizeSave = (value: number) => { + const hasDebounce = sizeDebounceRef.current != null; + const hasPending = sizePendingRef.current; + if (!hasDebounce && !hasPending) return; + + if (sizeDebounceRef.current) { + clearTimeout(sizeDebounceRef.current); + sizeDebounceRef.current = null; + } + + const clamped = clampAndroidOverlaySize(value); + const committed = androidPrefs?.androidOverlaySizeDp; + if (committed != null && clamped === committed) { + sizePendingRef.current = false; + return; + } + + void saveAndroidOverlaySize(clamped); + }; + + const handleAndroidOverlaySizeChange = (value: number) => { + const clamped = clampAndroidOverlaySize(value); + sizePendingRef.current = true; + setSizeDraft(clamped); + scheduleAndroidOverlaySizeSave(clamped); + }; + + const updateAndroidPref = async (key: K, value: UserPreferences[K]) => { + if (key !== 'androidOverlaySizeDp' && sizeDebounceRef.current) { + clearTimeout(sizeDebounceRef.current); + sizeDebounceRef.current = null; + } + + const patch: Partial = { + [key]: key === 'androidOverlayTrigger' + ? normalizeAndroidOverlayTrigger(value as AndroidOverlayTrigger) + : value, + }; + + if (key !== 'androidOverlaySizeDp') { + const committedSize = androidPrefs?.androidOverlaySizeDp; + const draftSize = sizeDraft; + if (sizePendingRef.current || (draftSize != null && draftSize !== committedSize)) { + patch.androidOverlaySizeDp = clampAndroidOverlaySize(draftSize ?? committedSize ?? 72); + } + sizePendingRef.current = false; + } + + try { + const nextPrefs = await persistAndroidOverlayPrefs(patch); + setAndroidPrefs(nextPrefs); + if (patch.androidOverlaySizeDp != null) { + setSizeDraft(nextPrefs.androidOverlaySizeDp); + } + await refreshAndroid(); + } catch (error) { + console.error('[android] failed to save overlay pref', error); + await refreshAndroid(); + } + }; + + const showOverlayPermission = mode === 'all' || mode === 'overlayPermission'; + const showAccessibility = mode === 'all' || mode === 'accessibility'; + const showOverlayConfig = mode === 'all' || mode === 'overlayConfig'; + + return ( + <> + {showOverlayPermission && ( + +
+ {androidOverlay?.message && ( + + {androidOverlay.message} + + )} + + {androidOverlay?.permission !== 'granted' && ( + { void requestAndroidOverlayPermission().then(refreshAndroid); }}> + {t('settings.permissions.grant')} + + )} +
+
+ )} + {showAccessibility && ( + +
+
+ {androidAccessibility?.message && ( + + {androidAccessibility.message} + + )} + + {!androidAccessibility?.enabled && ( + { void requestAndroidAccessibilityPermission().then(refreshAndroid); }}> + {t('settings.permissions.openSystem')} + + )} +
+ + {t('settings.permissions.androidAccessibilityImpact')} + +
+
+ )} + {showOverlayConfig && ( + <> + +
+ + + {t(`settings.permissions.androidInsertStrategyHint.${androidPrefs?.androidInsertStrategy ?? 'accessibility'}`)} + +
+
+ +
+ + + {t(`settings.permissions.androidOverlayTriggerHint.${androidPrefs?.androidOverlayTrigger ?? 'background'}`)} + + + {t('settings.permissions.androidOverlayTriggerDisabled.keyboard')} + +
+
+ +
+ + + {t(`settings.permissions.androidOverlayActivationModeHint.${androidPrefs?.androidOverlayActivationMode ?? 'tap'}`)} + +
+
+ +
+ + + {t(`settings.permissions.androidOverlayLeftSwipeActionHint.${androidPrefs?.androidOverlayLeftSwipeAction ?? 'translation'}`)} + +
+
+ +
+ + + {t(`settings.permissions.androidOverlayCancelSwipeDirectionHint.${androidPrefs?.androidOverlayCancelSwipeDirection ?? 'up'}`)} + +
+
+ +
+
+ { + handleAndroidOverlaySizeChange(Number(event.target.value)); + }} + onPointerUp={(event) => { + flushAndroidOverlaySizeSave(Number(event.currentTarget.value)); + }} + onTouchEnd={(event) => { + flushAndroidOverlaySizeSave(Number(event.currentTarget.value)); + }} + onBlur={(event) => { + flushAndroidOverlaySizeSave(Number(event.currentTarget.value)); + }} + style={{ width: 132 }} + /> + + {sizeDraft ?? androidPrefs?.androidOverlaySizeDp ?? 72} dp + +
+ + {t('settings.permissions.androidOverlaySizeHint')} + +
+
+ + )} + + ); +} + +function AndroidOverlayStatusPill({ status }: { status: AndroidOverlayStatus | null }) { + const { t } = useTranslation(); + if (!status) return {t('settings.permissions.checking')}; + if (status.permission === 'granted') { + return {t('settings.permissions.granted')}; + } + return {t('settings.permissions.denied')}; +} + +function AndroidAccessibilityStatusPill({ status }: { status: AndroidAccessibilityStatus | null }) { + const { t } = useTranslation(); + if (!status) return {t('settings.permissions.checking')}; + if (status.enabled) { + return {t('settings.permissions.granted')}; + } + return {t('settings.permissions.denied')}; +} diff --git a/openless-all/app/android/frontend/lib/androidIpc.ts b/openless-all/app/android/frontend/lib/androidIpc.ts new file mode 100644 index 00000000..062407ab --- /dev/null +++ b/openless-all/app/android/frontend/lib/androidIpc.ts @@ -0,0 +1,43 @@ +import { invokeOrMock } from '../../../src/lib/ipc'; +import type { + AndroidAccessibilityStatus, + AndroidOverlayStatus, +} from './androidTypes'; + +export function getAndroidOverlayStatus(): Promise { + return invokeOrMock('get_android_overlay_status', undefined, () => ({ + permission: 'notAndroid', + overlayVisible: false, + message: 'Android overlay is only available on Android', + })); +} + +export function requestAndroidOverlayPermission(): Promise<{ launched: boolean; message: string }> { + return invokeOrMock('request_android_overlay_permission', undefined, () => ({ + launched: false, + message: 'Mock: overlay permission unavailable in browser preview', + })); +} + +export function showAndroidOverlay(): Promise { + return invokeOrMock('show_android_overlay', undefined, () => undefined); +} + +export function hideAndroidOverlay(): Promise { + return invokeOrMock('hide_android_overlay', undefined, () => undefined); +} + +export function getAndroidAccessibilityStatus(): Promise { + return invokeOrMock('get_android_accessibility_status', undefined, () => ({ + state: 'notAndroid', + enabled: false, + message: 'Android accessibility is only available on Android', + })); +} + +export function requestAndroidAccessibilityPermission(): Promise<{ launched: boolean; message: string }> { + return invokeOrMock('request_android_accessibility_permission', undefined, () => ({ + launched: false, + message: 'Mock: accessibility settings unavailable in browser preview', + })); +} diff --git a/openless-all/app/android/frontend/lib/androidMicrophonePermission.ts b/openless-all/app/android/frontend/lib/androidMicrophonePermission.ts new file mode 100644 index 00000000..d5fee524 --- /dev/null +++ b/openless-all/app/android/frontend/lib/androidMicrophonePermission.ts @@ -0,0 +1,104 @@ +import type { PermissionStatus as AppPermissionStatus } from '../../../src/lib/types'; +import { checkMicrophonePermission, requestMicrophonePermission } from '../../../src/lib/ipc'; + +const ANDROID_MIC_GRANTED_KEY = 'openless.androidMicrophoneGranted'; +const ANDROID_MIC_REQUESTED_KEY = 'openless.androidMicrophoneRequested'; + +export async function checkAndroidMicrophoneAccess(): Promise { + try { + const nativeStatus = await checkMicrophonePermission(); + if (nativeStatus === 'granted' || nativeStatus === 'notApplicable') { + localStorage.setItem(ANDROID_MIC_GRANTED_KEY, '1'); + return 'granted'; + } + if (nativeStatus === 'denied' || nativeStatus === 'restricted') { + localStorage.removeItem(ANDROID_MIC_GRANTED_KEY); + return localStorage.getItem(ANDROID_MIC_REQUESTED_KEY) === '1' ? 'denied' : 'notDetermined'; + } + } catch { + // Fall through to WebView-local checks below. + } + + if (localStorage.getItem(ANDROID_MIC_GRANTED_KEY) === '1') { + return 'granted'; + } + + try { + const permissions = navigator.permissions; + if (permissions?.query) { + const status = await permissions.query({ name: 'microphone' as PermissionName }); + if (status.state === 'granted') return 'granted'; + if (status.state === 'denied') { + localStorage.removeItem(ANDROID_MIC_GRANTED_KEY); + return 'denied'; + } + } + } catch { + // Android WebView versions differ on navigator.permissions support. + } + + return 'notDetermined'; +} + +export async function requestAndroidMicrophoneAccess(): Promise { + try { + localStorage.setItem(ANDROID_MIC_REQUESTED_KEY, '1'); + const nativeStatus = await requestMicrophonePermission(); + if (nativeStatus === 'granted' || nativeStatus === 'notApplicable') { + localStorage.setItem(ANDROID_MIC_GRANTED_KEY, '1'); + return 'granted'; + } + if (nativeStatus === 'denied' || nativeStatus === 'restricted') { + localStorage.removeItem(ANDROID_MIC_GRANTED_KEY); + } + if (nativeStatus === 'notDetermined') { + return 'notDetermined'; + } + } catch { + // Fall through to WebView-local checks below. + } + + if (localStorage.getItem(ANDROID_MIC_GRANTED_KEY) === '1') { + return 'granted'; + } + + try { + const permissions = navigator.permissions; + if (permissions?.query) { + const status = await permissions.query({ name: 'microphone' as PermissionName }); + if (status.state === 'granted') { + localStorage.setItem(ANDROID_MIC_GRANTED_KEY, '1'); + return 'granted'; + } + if (status.state === 'denied') { + localStorage.removeItem(ANDROID_MIC_GRANTED_KEY); + localStorage.setItem(ANDROID_MIC_REQUESTED_KEY, '1'); + return 'denied'; + } + } + } catch { + // Android WebView versions differ on navigator.permissions support. + } + + const mediaDevices = navigator.mediaDevices; + if (!mediaDevices?.getUserMedia) { + return 'notDetermined'; + } + + let stream: MediaStream | null = null; + try { + localStorage.setItem(ANDROID_MIC_REQUESTED_KEY, '1'); + stream = await mediaDevices.getUserMedia({ audio: true }); + localStorage.setItem(ANDROID_MIC_GRANTED_KEY, '1'); + return 'granted'; + } catch (error) { + console.warn('[android-mic] WebView microphone permission request failed', error); + localStorage.removeItem(ANDROID_MIC_GRANTED_KEY); + if (error instanceof DOMException && error.name === 'NotAllowedError') { + return 'denied'; + } + return 'notDetermined'; + } finally { + stream?.getTracks().forEach(track => track.stop()); + } +} diff --git a/openless-all/app/android/frontend/lib/androidTypes.ts b/openless-all/app/android/frontend/lib/androidTypes.ts new file mode 100644 index 00000000..387d5419 --- /dev/null +++ b/openless-all/app/android/frontend/lib/androidTypes.ts @@ -0,0 +1,38 @@ +/** Android-specific preference and status types (mirrors Rust IPC payloads). */ + +export type AndroidInsertStrategy = 'accessibility' | 'clipboard'; +export type AndroidOverlayTrigger = 'background' | 'keyboard' | 'always'; +export type AndroidOverlayActivationMode = 'tap' | 'long_press'; +export type AndroidOverlayLeftSwipeAction = 'translation' | 'style_pack'; +export type AndroidOverlayCancelSwipeDirection = 'up' | 'down'; + +export interface AndroidOverlayStatus { + permission: 'granted' | 'notGranted' | 'notAndroid'; + overlayVisible: boolean; + message: string; +} + +export interface AndroidAccessibilityStatus { + state: 'enabled' | 'notEnabled' | 'notAndroid'; + enabled: boolean; + message: string; +} + +export type AndroidPreferenceKey = + | 'androidInsertStrategy' + | 'androidOverlayTrigger' + | 'androidOverlayActivationMode' + | 'androidOverlayLeftSwipeAction' + | 'androidOverlayCancelSwipeDirection' + | 'androidOverlaySizeDp'; + +export function normalizeAndroidOverlayTrigger( + trigger: AndroidOverlayTrigger, +): AndroidOverlayTrigger { + return trigger === 'keyboard' ? 'background' : trigger; +} + +export function clampAndroidOverlaySize(size: number): number { + if (!Number.isFinite(size)) return 72; + return Math.min(120, Math.max(48, Math.round(size / 4) * 4)); +} diff --git a/openless-all/app/android/kotlin/MicrophonePermissionActivity.kt b/openless-all/app/android/kotlin/MicrophonePermissionActivity.kt new file mode 100644 index 00000000..746941f2 --- /dev/null +++ b/openless-all/app/android/kotlin/MicrophonePermissionActivity.kt @@ -0,0 +1,49 @@ +package com.openless.app + +import android.Manifest +import android.app.Activity +import android.content.pm.PackageManager +import android.os.Bundle +import android.util.Log + +class MicrophonePermissionActivity : Activity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { + OpenLessPermissionBridge.resolveRecordAudioPermission(true) + finish() + return + } + requestPermissions(arrayOf(Manifest.permission.RECORD_AUDIO), REQUEST_RECORD_AUDIO) + } + + override fun onRequestPermissionsResult( + requestCode: Int, + permissions: Array, + grantResults: IntArray, + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + if (requestCode != REQUEST_RECORD_AUDIO) { + return + } + val granted = grantResults.isNotEmpty() && + grantResults[0] == PackageManager.PERMISSION_GRANTED + Log.i(TAG, "RECORD_AUDIO permission result granted=$granted") + OpenLessPermissionBridge.resolveRecordAudioPermission(granted) + finish() + } + + override fun onDestroy() { + if (isFinishing) { + OpenLessPermissionBridge.resolveRecordAudioPermission( + checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED, + ) + } + super.onDestroy() + } + + companion object { + private const val TAG = "OpenLessMicPermission" + private const val REQUEST_RECORD_AUDIO = 9101 + } +} diff --git a/openless-all/app/android/kotlin/OpenLessAccessibilityCommandReceiver.kt b/openless-all/app/android/kotlin/OpenLessAccessibilityCommandReceiver.kt new file mode 100644 index 00000000..a3d69ad4 --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessAccessibilityCommandReceiver.kt @@ -0,0 +1,36 @@ +package com.openless.app + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.os.ResultReceiver +import android.util.Log + +class OpenLessAccessibilityCommandReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent?) { + if (intent?.action != ACTION_PASTE) return + val pasted = OpenLessAccessibilityService.performPasteFromCommand() + resultReceiver(intent)?.send( + if (pasted) RESULT_PASTE_SUCCESS else RESULT_PASTE_FAILED, + Bundle().apply { putBoolean(EXTRA_PASTE_RESULT, pasted) }, + ) + if (!pasted) { + Log.w(TAG, "paste command did not find an editable focused field") + } + } + + @Suppress("DEPRECATION") + private fun resultReceiver(intent: Intent): ResultReceiver? { + return intent.getParcelableExtra(EXTRA_RESULT_RECEIVER) as? ResultReceiver + } + + companion object { + const val ACTION_PASTE = "com.openless.app.accessibility.PASTE" + const val EXTRA_RESULT_RECEIVER = "result_receiver" + const val EXTRA_PASTE_RESULT = "paste_result" + const val RESULT_PASTE_FAILED = 0 + const val RESULT_PASTE_SUCCESS = 1 + private const val TAG = "OpenLessA11yCommand" + } +} diff --git a/openless-all/app/android/kotlin/OpenLessAccessibilityService.kt b/openless-all/app/android/kotlin/OpenLessAccessibilityService.kt new file mode 100644 index 00000000..8a843049 --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessAccessibilityService.kt @@ -0,0 +1,373 @@ +package com.openless.app + +import android.accessibilityservice.AccessibilityService +import android.content.ClipboardManager +import android.content.Context +import android.content.Intent +import android.graphics.Rect +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.ResultReceiver +import android.provider.Settings +import android.util.Log +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Detects IME windows for overlay keyboard trigger mode and performs paste insertion. + */ +class OpenLessAccessibilityService : AccessibilityService() { + private val mainHandler = Handler(Looper.getMainLooper()) + private val heartbeatRunnable = object : Runnable { + override fun run() { + markServiceAlive() + mainHandler.postDelayed(this, HEARTBEAT_INTERVAL_MS) + } + } + private val keyboardRefreshRunnable = Runnable { updateKeyboardOverlayState() } + private var lastEditableFocus: AccessibilityNodeInfo? = null + + override fun onServiceConnected() { + super.onServiceConnected() + instance = this + startHeartbeat() + updateKeyboardOverlayState() + scheduleKeyboardOverlayRefresh() + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + if (event == null) return + markServiceAlive() + when (event.eventType) { + AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED, + AccessibilityEvent.TYPE_WINDOWS_CHANGED, + AccessibilityEvent.TYPE_VIEW_FOCUSED -> { + rememberFocusedEditable(event) + updateKeyboardOverlayState() + scheduleKeyboardOverlayRefresh() + } + } + } + + override fun onInterrupt() = Unit + + override fun onDestroy() { + mainHandler.removeCallbacks(heartbeatRunnable) + mainHandler.removeCallbacks(keyboardRefreshRunnable) + lastEditableFocus?.recycle() + lastEditableFocus = null + if (instance === this) { + instance = null + } + super.onDestroy() + } + + private fun scheduleKeyboardOverlayRefresh() { + mainHandler.removeCallbacks(keyboardRefreshRunnable) + for (delayMs in KEYBOARD_REFRESH_DELAYS_MS) { + mainHandler.postDelayed(keyboardRefreshRunnable, delayMs) + } + } + + private fun startHeartbeat() { + mainHandler.removeCallbacks(heartbeatRunnable) + heartbeatRunnable.run() + } + + private fun updateKeyboardOverlayState() { + if (!shouldTrackKeyboard()) { + return + } + if (!canDrawOverlays()) { + return + } + val imeBounds = findInputMethodBounds() + val intent = Intent(this, OpenLessOverlayService::class.java).apply { + action = OpenLessOverlayService.ACTION_KEYBOARD_CHANGED + putExtra(OpenLessOverlayService.EXTRA_KEYBOARD_VISIBLE, imeBounds != null) + imeBounds?.let { + putExtra(OpenLessOverlayService.EXTRA_KEYBOARD_TOP, it.top) + putExtra(OpenLessOverlayService.EXTRA_KEYBOARD_BOTTOM, it.bottom) + } + } + try { + Log.i(TAG, "keyboard overlay event visible=${imeBounds != null} bounds=$imeBounds") + startService(intent) + } catch (error: Throwable) { + Log.w(TAG, "send keyboard overlay event failed", error) + } + } + + private fun findInputMethodBounds(): Rect? { + for (window in windows) { + if (window.type != AccessibilityWindowInfo.TYPE_INPUT_METHOD) { + continue + } + val bounds = Rect() + window.getBoundsInScreen(bounds) + if (!bounds.isEmpty) { + return bounds + } + } + return null + } + + private fun shouldTrackKeyboard(): Boolean { + return OpenLessAndroidPreferences.isKeyboardOverlayTrigger(this) + } + + private fun canDrawOverlays(): Boolean { + return if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) { + Settings.canDrawOverlays(this) + } else { + true + } + } + + private fun performPasteToFocusedField(): Boolean { + val target = findEditableTarget() ?: return false + return try { + target.performAction(AccessibilityNodeInfo.ACTION_FOCUS) + pasteWithRetryOrSetText(target) + } finally { + target.recycle() + } + } + + private fun rememberFocusedEditable(event: AccessibilityEvent) { + val source = event.source ?: return + try { + if (!source.isEditable) return + lastEditableFocus?.recycle() + lastEditableFocus = AccessibilityNodeInfo.obtain(source) + } finally { + source.recycle() + } + } + + private fun findEditableTarget(): AccessibilityNodeInfo? { + lastEditableFocus?.let { cached -> + if (cached.refresh() && cached.isEditable) { + return AccessibilityNodeInfo.obtain(cached) + } + } + val root = rootInActiveWindow ?: return null + editableFocusedNode(root, AccessibilityNodeInfo.FOCUS_INPUT)?.let { return it } + editableFocusedNode(root, AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)?.let { return it } + return findEditableInTree(root, 0) + } + + private fun editableFocusedNode(root: AccessibilityNodeInfo, focusType: Int): AccessibilityNodeInfo? { + val focused = root.findFocus(focusType) ?: return null + if (focused.isEditable) { + return focused + } + focused.recycle() + return null + } + + private fun findEditableInTree(node: AccessibilityNodeInfo, depth: Int): AccessibilityNodeInfo? { + if (depth > MAX_EDITABLE_SEARCH_DEPTH) return null + var firstEditable: AccessibilityNodeInfo? = null + if (node.isEditable) { + if (node.isFocused) { + return AccessibilityNodeInfo.obtain(node) + } + firstEditable = AccessibilityNodeInfo.obtain(node) + } + for (index in 0 until node.childCount) { + val child = node.getChild(index) ?: continue + try { + findEditableInTree(child, depth + 1)?.let { found -> + firstEditable?.recycle() + return found + } + } finally { + child.recycle() + } + } + return firstEditable + } + + private fun pasteWithRetryOrSetText(target: AccessibilityNodeInfo): Boolean { + sleepQuietly(PASTE_INITIAL_DELAY_MS) + repeat(PASTE_RETRY_COUNT) { attempt -> + if (target.performAction(AccessibilityNodeInfo.ACTION_PASTE)) { + Log.i(TAG, "paste=true attempt=${attempt + 1} package=${target.packageName}") + return true + } + sleepQuietly(PASTE_RETRY_DELAY_MS) + } + val setText = appendClipboardTextWithSetText(target) + Log.i(TAG, "paste=false setText=$setText package=${target.packageName}") + return setText + } + + private fun appendClipboardTextWithSetText(target: AccessibilityNodeInfo): Boolean { + if (target.isPassword) return false + val clipboardText = clipboardText().takeIf { it.isNotEmpty() } ?: return false + val existingText = target.text?.toString().orEmpty() + val args = Bundle().apply { + putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + existingText + clipboardText, + ) + } + return target.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) + } + + private fun clipboardText(): String { + val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as? ClipboardManager ?: return "" + val clip = clipboard.primaryClip ?: return "" + if (clip.itemCount <= 0) return "" + return clip.getItemAt(0)?.coerceToText(this)?.toString().orEmpty() + } + + private fun sleepQuietly(delayMs: Long) { + try { + Thread.sleep(delayMs) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } + + private fun captureSelectedTextFromFocusedNode(): String { + val root = rootInActiveWindow ?: return "" + val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) + ?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) + focused?.let { + return try { + selectedTextFromNode(it) + } finally { + it.recycle() + } + } + return selectedTextFromTree(root) + } + + private fun selectedTextFromTree(node: AccessibilityNodeInfo?): String { + if (node == null) return "" + selectedTextFromNode(node).takeIf { it.isNotBlank() }?.let { return it } + for (index in 0 until node.childCount) { + val child = node.getChild(index) ?: continue + try { + selectedTextFromTree(child).takeIf { it.isNotBlank() }?.let { return it } + } finally { + child.recycle() + } + } + return "" + } + + private fun selectedTextFromNode(node: AccessibilityNodeInfo): String { + val text = node.text?.toString() ?: return "" + val start = node.textSelectionStart + val end = node.textSelectionEnd + if (start < 0 || end < 0 || start == end) return "" + val from = minOf(start, end).coerceIn(0, text.length) + val to = maxOf(start, end).coerceIn(0, text.length) + if (from >= to) return "" + return text.substring(from, to) + } + + private fun markServiceAlive() { + getSharedPreferences(PREFS_NAME, prefsMode()) + .edit() + .putLong(PREF_KEY_LAST_HEARTBEAT, System.currentTimeMillis()) + .apply() + } + + companion object { + @Volatile + var instance: OpenLessAccessibilityService? = null + private set + + @JvmStatic + fun pasteToFocusedField(): Boolean { + instance?.let { return it.performPasteToFocusedField() } + return sendPasteRequestToAccessibilityProcess() + } + + @JvmStatic + fun captureSelectedText(): String { + return instance?.captureSelectedTextFromFocusedNode().orEmpty() + } + + @JvmStatic + fun isEnabled(context: Context): Boolean { + val enabled = Settings.Secure.getInt( + context.contentResolver, + Settings.Secure.ACCESSIBILITY_ENABLED, + 0, + ) == 1 + if (!enabled) return false + val services = Settings.Secure.getString( + context.contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, + ) ?: return false + return services.contains("${context.packageName}/${OpenLessAccessibilityService::class.java.name}") + } + + @JvmStatic + fun isOperational(context: Context): Boolean { + if (!isEnabled(context)) return false + val lastHeartbeat = context + .getSharedPreferences(PREFS_NAME, prefsMode()) + .getLong(PREF_KEY_LAST_HEARTBEAT, 0L) + if (lastHeartbeat <= 0L) return false + return System.currentTimeMillis() - lastHeartbeat <= HEARTBEAT_STALE_MS + } + + internal fun performPasteFromCommand(): Boolean { + return instance?.performPasteToFocusedField() == true + } + + private fun sendPasteRequestToAccessibilityProcess(): Boolean { + val context = OpenLessAppContext.context ?: return false + if (!isOperational(context)) return false + val latch = CountDownLatch(1) + val success = AtomicBoolean(false) + val receiver = object : ResultReceiver(null) { + override fun onReceiveResult(resultCode: Int, resultData: Bundle?) { + success.set(resultCode == OpenLessAccessibilityCommandReceiver.RESULT_PASTE_SUCCESS) + latch.countDown() + } + } + return try { + val intent = Intent(context, OpenLessAccessibilityCommandReceiver::class.java).apply { + action = OpenLessAccessibilityCommandReceiver.ACTION_PASTE + putExtra(OpenLessAccessibilityCommandReceiver.EXTRA_RESULT_RECEIVER, receiver) + } + context.sendBroadcast(intent) + if (!latch.await(PASTE_COMMAND_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + Log.w(TAG, "accessibility paste result timed out") + return false + } + success.get() + } catch (error: Throwable) { + Log.w(TAG, "send accessibility paste request failed", error) + false + } + } + + @Suppress("DEPRECATION") + private fun prefsMode(): Int = Context.MODE_PRIVATE or Context.MODE_MULTI_PROCESS + + private val KEYBOARD_REFRESH_DELAYS_MS = longArrayOf(120L, 360L, 900L, 1600L) + private const val MAX_EDITABLE_SEARCH_DEPTH = 4 + private const val PASTE_INITIAL_DELAY_MS = 50L + private const val PASTE_RETRY_COUNT = 3 + private const val PASTE_RETRY_DELAY_MS = 80L + private const val PASTE_COMMAND_TIMEOUT_MS = 800L + private const val TAG = "OpenLessAccessibility" + private const val PREFS_NAME = "openless_accessibility" + private const val PREF_KEY_LAST_HEARTBEAT = "last_heartbeat" + private const val HEARTBEAT_INTERVAL_MS = 5_000L + private const val HEARTBEAT_STALE_MS = 15_000L + } +} diff --git a/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt b/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt new file mode 100644 index 00000000..1b06e9c8 --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt @@ -0,0 +1,112 @@ +package com.openless.app + +import android.content.Context +import android.util.Log +import java.io.File +import org.json.JSONObject + +/** + * Reads Android-visible preferences without depending on the Rust coordinator. + */ +object OpenLessAndroidPreferences { + private const val TAG = "OpenLessAndroidPrefs" + private const val APP_DIR = "OpenLess" + private const val PREFERENCES_FILE = "preferences.json" + private const val KEY_OVERLAY_TRIGGER = "androidOverlayTrigger" + private const val KEY_OVERLAY_ACTIVATION_MODE = "androidOverlayActivationMode" + private const val KEY_OVERLAY_LEFT_SWIPE_ACTION = "androidOverlayLeftSwipeAction" + private const val KEY_OVERLAY_CANCEL_SWIPE_DIRECTION = "androidOverlayCancelSwipeDirection" + private const val KEY_OVERLAY_SIZE_DP = "androidOverlaySizeDp" + private const val DEFAULT_OVERLAY_SIZE_DP = 72 + private const val MIN_OVERLAY_SIZE_DP = 48 + private const val MAX_OVERLAY_SIZE_DP = 120 + private val VALID_OVERLAY_TRIGGERS = setOf("background", "always") + private val VALID_OVERLAY_ACTIVATION_MODES = setOf("tap", "long_press") + private val VALID_OVERLAY_LEFT_SWIPE_ACTIONS = setOf("translation", "style_pack") + private val VALID_OVERLAY_CANCEL_SWIPE_DIRECTIONS = setOf("up", "down") + + fun overlayTriggerMode(context: Context): String? { + val value = readPreferenceString(context, KEY_OVERLAY_TRIGGER) ?: return null + if (value == "keyboard") { + return "background" + } + return value.takeIf { it in VALID_OVERLAY_TRIGGERS } + } + + /** True when preferences still store legacy `"keyboard"` (before migration). */ + fun isKeyboardOverlayTrigger(context: Context): Boolean { + return readPreferenceString(context, KEY_OVERLAY_TRIGGER) == "keyboard" + } + + fun overlayActivationMode(context: Context): String { + return readPreferenceString(context, KEY_OVERLAY_ACTIVATION_MODE) + ?.takeIf { it in VALID_OVERLAY_ACTIVATION_MODES } + ?: "tap" + } + + fun overlayLeftSwipeAction(context: Context): String { + return readPreferenceString(context, KEY_OVERLAY_LEFT_SWIPE_ACTION) + ?.takeIf { it in VALID_OVERLAY_LEFT_SWIPE_ACTIONS } + ?: "translation" + } + + fun overlayCancelSwipeDirection(context: Context): String { + return readPreferenceString(context, KEY_OVERLAY_CANCEL_SWIPE_DIRECTION) + ?.takeIf { it in VALID_OVERLAY_CANCEL_SWIPE_DIRECTIONS } + ?: "up" + } + + fun overlaySizeDp(context: Context): Int { + return readPreferenceInt(context, KEY_OVERLAY_SIZE_DP) + ?.coerceIn(MIN_OVERLAY_SIZE_DP, MAX_OVERLAY_SIZE_DP) + ?: DEFAULT_OVERLAY_SIZE_DP + } + + private fun readPreferenceString(context: Context, key: String): String? { + for (file in preferenceFiles(context).distinctBy { it.absolutePath }) { + if (!file.isFile) { + continue + } + val value = try { + JSONObject(file.readText()).optString(key, "") + } catch (error: Throwable) { + Log.w(TAG, "read ${file.absolutePath} failed", error) + "" + } + if (value.isNotBlank()) { + return value + } + } + return null + } + + private fun readPreferenceInt(context: Context, key: String): Int? { + for (file in preferenceFiles(context).distinctBy { it.absolutePath }) { + if (!file.isFile) { + continue + } + val value = try { + val json = JSONObject(file.readText()) + if (json.has(key)) json.optInt(key) else null + } catch (error: Throwable) { + Log.w(TAG, "read ${file.absolutePath} failed", error) + null + } + if (value != null) { + return value + } + } + return null + } + + private fun preferenceFiles(context: Context): List { + val files = mutableListOf() + val envDir = System.getenv("TAURI_ANDROID_APP_DATA_DIR") + if (!envDir.isNullOrBlank()) { + files += File(File(envDir), APP_DIR).resolve(PREFERENCES_FILE) + } + files += File(File(context.cacheDir, APP_DIR), PREFERENCES_FILE) + files += File(File(context.filesDir, APP_DIR), PREFERENCES_FILE) + return files + } +} diff --git a/openless-all/app/android/kotlin/OpenLessAppContext.kt b/openless-all/app/android/kotlin/OpenLessAppContext.kt new file mode 100644 index 00000000..e1c627ae --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessAppContext.kt @@ -0,0 +1,13 @@ +package com.openless.app + +import android.content.Context + +object OpenLessAppContext { + @Volatile + var context: Context? = null + private set + + fun initialize(context: Context) { + this.context = context.applicationContext + } +} diff --git a/openless-all/app/android/kotlin/OpenLessApplication.kt b/openless-all/app/android/kotlin/OpenLessApplication.kt new file mode 100644 index 00000000..8e25fd93 --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessApplication.kt @@ -0,0 +1,84 @@ +package com.openless.app + +import android.app.Activity +import android.app.Application +import android.content.Intent +import android.os.Bundle +import android.provider.Settings +import android.util.Log + +/** + * Registers activity lifecycle hooks for overlay background trigger mode. + */ +class OpenLessApplication : Application() { + override fun onCreate() { + super.onCreate() + OpenLessAppContext.initialize(this) + registerActivityLifecycleCallbacks(object : ActivityLifecycleCallbacks { + override fun onActivityCreated(activity: Activity, savedInstanceState: Bundle?) = Unit + override fun onActivityStarted(activity: Activity) { + if (activity.javaClass.name.endsWith("MainActivity")) { + maybeHideOverlayOnForeground() + } + } + override fun onActivityResumed(activity: Activity) = Unit + override fun onActivityPaused(activity: Activity) = Unit + override fun onActivityStopped(activity: Activity) { + if (activity.javaClass.name.endsWith("MainActivity")) { + maybeShowOverlayOnBackground() + } + } + override fun onActivitySaveInstanceState(activity: Activity, outState: Bundle) = Unit + override fun onActivityDestroyed(activity: Activity) = Unit + }) + } + + private fun maybeShowOverlayOnBackground() { + val configured = configuredOverlayTriggerMode() + val shouldShow = configured == "background" || + configured == "always" + if (!shouldShow) { + return + } + if (!canDrawOverlays()) { + return + } + sendOverlayAction(OpenLessOverlayService.ACTION_SHOW) + } + + private fun maybeHideOverlayOnForeground() { + if (configuredOverlayTriggerMode() == "always") { + if (canDrawOverlays()) { + sendOverlayAction(OpenLessOverlayService.ACTION_SHOW) + } + return + } + sendOverlayAction(OpenLessOverlayService.ACTION_HIDE) + } + + private fun configuredOverlayTriggerMode(): String { + return OpenLessAndroidPreferences.overlayTriggerMode(this) ?: "background" + } + + private fun canDrawOverlays(): Boolean { + return if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) { + Settings.canDrawOverlays(this) + } else { + true + } + } + + private fun sendOverlayAction(action: String) { + try { + startService(Intent(this, OpenLessOverlayService::class.java).apply { + this.action = action + }) + } catch (error: Throwable) { + Log.w(TAG, "overlay action failed: $action", error) + } + } + + companion object { + private const val TAG = "OpenLessApplication" + } +} diff --git a/openless-all/app/android/kotlin/OpenLessNative.kt b/openless-all/app/android/kotlin/OpenLessNative.kt new file mode 100644 index 00000000..3c6f297a --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessNative.kt @@ -0,0 +1,42 @@ +package com.openless.app + +/** + * JNI bridge from Kotlin overlay / lifecycle code into Rust Coordinator. + */ +object OpenLessNative { + init { + try { + System.loadLibrary("openless_lib") + } catch (error: UnsatisfiedLinkError) { + android.util.Log.e("OpenLessNative", "failed to load openless_lib", error) + } + } + + @JvmStatic external fun nativeStartDictation() + + @JvmStatic external fun nativeStartDictationWithTranslation(translation: Boolean) + + @JvmStatic external fun nativeStopDictation() + + @JvmStatic external fun nativeStopDictationWithTranslation(translation: Boolean) + + @JvmStatic external fun nativeCancelDictation() + + @JvmStatic external fun nativeSwitchStylePack() + + @JvmStatic external fun nativeOpenQaFromOverlay() + + @JvmStatic external fun nativeFinalizeQaFromOverlay() + + @JvmStatic external fun nativeGetOverlayTriggerMode(): String + + @JvmStatic external fun nativeCanDrawOverlays(context: android.content.Context): Boolean + + @JvmStatic external fun nativeShowOverlay(context: android.content.Context) + + @JvmStatic external fun nativeHideOverlay(context: android.content.Context) + + @JvmStatic external fun nativeIsOverlayVisible(): Boolean + + @JvmStatic external fun nativeNotifyOverlayPermissionChanged(context: android.content.Context) +} diff --git a/openless-all/app/android/kotlin/OpenLessOverlayBridge.kt b/openless-all/app/android/kotlin/OpenLessOverlayBridge.kt new file mode 100644 index 00000000..92406730 --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessOverlayBridge.kt @@ -0,0 +1,33 @@ +package com.openless.app + +import android.os.Handler +import android.os.Looper + +/** + * Rust calls back into this object to refresh overlay UI state. + */ +object OpenLessOverlayBridge { + private val mainHandler = Handler(Looper.getMainLooper()) + + @Volatile + var listener: OverlayStateListener? = null + + interface OverlayStateListener { + fun onCapsuleStateChanged(state: String, message: String?) + } + + @JvmStatic + fun onCapsuleStateChanged(state: String, message: String?) { + mainHandler.post { + listener?.onCapsuleStateChanged(state, message) + } + } + + @JvmStatic + fun showToast(message: String) { + mainHandler.post { + val service = OpenLessOverlayService.instance ?: return@post + android.widget.Toast.makeText(service.applicationContext, message, android.widget.Toast.LENGTH_SHORT).show() + } + } +} diff --git a/openless-all/app/android/kotlin/OpenLessOverlayService.kt b/openless-all/app/android/kotlin/OpenLessOverlayService.kt new file mode 100644 index 00000000..658d22eb --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessOverlayService.kt @@ -0,0 +1,831 @@ +package com.openless.app + +import android.Manifest +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.Service +import android.content.Intent +import android.content.pm.PackageManager +import android.content.pm.ServiceInfo +import android.graphics.Color +import android.graphics.PixelFormat +import android.graphics.drawable.GradientDrawable +import android.os.Build +import android.os.IBinder +import android.provider.Settings +import android.util.Log +import android.view.Gravity +import android.view.MotionEvent +import android.view.View +import android.view.WindowManager +import android.widget.FrameLayout +import android.widget.ImageView +import android.widget.Toast +import kotlin.math.abs + +/** + * Foreground service + TYPE_APPLICATION_OVERLAY floating dictation control. + */ +class OpenLessOverlayService : Service(), OpenLessOverlayBridge.OverlayStateListener { + + private var windowManager: WindowManager? = null + private var rootView: FrameLayout? = null + private var layoutParams: WindowManager.LayoutParams? = null + private var recording = false + private var processing = false + private var keyboardVisible = false + private var armed = false + private var dragStartX = 0 + private var dragStartY = 0 + private var paramStartX = 0 + private var paramStartY = 0 + private var dragging = false + private var longPressRecording = false + private var pendingSwipe: SwipeDirection? = null + private var swipeConsumed = false + + private lateinit var iconContainer: FrameLayout + private lateinit var iconButton: ImageView + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onCreate() { + super.onCreate() + instance = this + OpenLessOverlayBridge.listener = this + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + Log.i( + TAG, + "onStartCommand action=${intent?.action} startId=$startId rootAttached=${rootView?.isAttachedToWindow}", + ) + when (intent?.action) { + ACTION_SHOW -> showOverlay() + ACTION_START_RECORDING -> { + showOverlay() + startRecordingFromOverlay() + } + ACTION_HIDE -> { + hideOverlay() + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + stopForeground(STOP_FOREGROUND_REMOVE) + } else { + @Suppress("DEPRECATION") + stopForeground(true) + } + stopSelf(startId) + } + ACTION_REPLACE_OVERLAY -> replaceOverlay() + ACTION_TOGGLE_EXPAND -> handleIconClick() + ACTION_KEYBOARD_CHANGED -> handleKeyboardChanged(intent) + ACTION_REFRESH_LAYOUT -> refreshOverlayLayout() + } + return START_STICKY + } + + override fun onDestroy() { + if (OpenLessOverlayBridge.listener === this) { + OpenLessOverlayBridge.listener = null + } + hideOverlay() + if (instance === this) { + instance = null + } + super.onDestroy() + } + + override fun onCapsuleStateChanged(state: String, message: String?) { + when (state) { + "recording" -> { + recording = true + processing = false + if (!tryPromoteRecordingForeground()) { + try { + OpenLessNative.nativeCancelDictation() + } catch (error: Throwable) { + Log.w(TAG, "cancel dictation bridge unavailable", error) + } + return + } + applyVisualState(OverlayVisualState.Recording) + } + "transcribing", "polishing" -> { + recording = false + processing = true + applyVisualState(OverlayVisualState.Processing) + } + "done" -> { + recording = false + processing = false + setArmed(false) + } + "error" -> { + recording = false + processing = false + setArmed(false) + applyVisualState(OverlayVisualState.Error) + message?.takeIf { it.isNotBlank() }?.let { showToast(it) } + } + "cancelled", "idle" -> { + recording = false + processing = false + setArmed(false) + } + } + } + + private fun showOverlay() = withOverlayLock { + windowManager = getSystemService(WINDOW_SERVICE) as WindowManager + reconcileOverlayRoots() + overlayRoots.lastOrNull()?.let { existing -> + val params = existing.layoutParams as? WindowManager.LayoutParams + if (params != null) { + refreshExistingOverlayLayout(existing, params) + Log.i(TAG, "overlay already shown roots=${overlayRoots.size}") + return@withOverlayLock + } + removeOverlayRoot(existing) + synchronized(overlayRoots) { + overlayRoots.remove(existing) + } + if (rootView === existing) { + rootView = null + layoutParams = null + } + } + attachNewOverlayRoot() + } + + private fun replaceOverlay() = withOverlayLock { + windowManager = windowManager ?: getSystemService(WINDOW_SERVICE) as WindowManager + reconcileOverlayRoots() + val removed = clearAllOverlayRoots() + if (removed > 0) { + Log.i(TAG, "overlay replace cleared removed=$removed") + } + if (!canDrawOverlays()) { + Log.i(TAG, "overlay replace skipped no overlay permission") + return@withOverlayLock + } + if (!attachNewOverlayRoot()) { + return@withOverlayLock + } + if (recording) { + tryPromoteRecordingForeground() + } + Log.i(TAG, "overlay replaced roots=1") + } + + private fun hideOverlay() = withOverlayLock { + val removed = clearAllOverlayRoots() + if (removed > 0) { + Log.i(TAG, "overlay hidden removed=$removed") + } + } + + private fun clearAllOverlayRoots(): Int { + windowManager = windowManager ?: getSystemService(WINDOW_SERVICE) as WindowManager + val views = synchronized(overlayRoots) { + (overlayRoots + listOfNotNull(rootView)).distinct().also { + overlayRoots.clear() + } + } + views.forEach { view -> + removeOverlayRoot(view) + } + rootView = null + layoutParams = null + return views.size + } + + private fun attachNewOverlayRoot(): Boolean { + val savedPosition = loadSavedPosition() + val params = WindowManager.LayoutParams( + WindowManager.LayoutParams.WRAP_CONTENT, + WindowManager.LayoutParams.WRAP_CONTENT, + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY + } else { + @Suppress("DEPRECATION") + WindowManager.LayoutParams.TYPE_PHONE + }, + WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE or + WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL or + WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN, + PixelFormat.TRANSLUCENT, + ).apply { + gravity = Gravity.TOP or Gravity.START + x = savedPosition.first + y = savedPosition.second + } + layoutParams = params + + val root = FrameLayout(this).apply { + contentDescription = "OpenLess" + isClickable = true + isFocusable = false + setOnClickListener { handleIconClick() } + } + iconContainer = root + iconButton = buildIconButton() + root.addView( + iconButton, + FrameLayout.LayoutParams(1, 1, Gravity.CENTER), + ) + applyOverlaySize(root) + attachDragHandler(root, params) + return try { + windowManager?.addView(root, params) + rootView = root + synchronized(overlayRoots) { + overlayRoots.add(root) + } + Log.i(TAG, "overlay shown x=${params.x} y=${params.y} roots=${overlayRoots.size}") + applyVisualState( + when { + recording -> OverlayVisualState.Recording + processing -> OverlayVisualState.Processing + armed -> OverlayVisualState.Armed + else -> OverlayVisualState.Idle + }, + ) + true + } catch (error: Throwable) { + Log.w(TAG, "show overlay failed", error) + layoutParams = null + false + } + } + + private fun canDrawOverlays(): Boolean { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + Settings.canDrawOverlays(this) + } else { + true + } + } + + private fun refreshExistingOverlayLayout( + root: FrameLayout, + params: WindowManager.LayoutParams, + ) { + rootView = root + layoutParams = params + iconContainer = root + (root.getChildAt(0) as? ImageView)?.let { iconButton = it } + applyOverlaySize(root) + attachDragHandler(root, params) + clampToScreen(params) + windowManager?.updateViewLayout(root, params) + } + + private fun refreshOverlayLayout() = withOverlayLock { + windowManager = windowManager ?: getSystemService(WINDOW_SERVICE) as WindowManager + reconcileOverlayRoots() + val root = overlayRoots.lastOrNull() ?: rootView + if (root == null || !root.isAttachedToWindow) { + Log.i(TAG, "overlay refresh skipped roots=0") + return@withOverlayLock + } + val params = root.layoutParams as? WindowManager.LayoutParams + if (params == null) { + Log.i(TAG, "overlay refresh skipped roots=0") + return@withOverlayLock + } + refreshExistingOverlayLayout(root, params) + val sizeDp = OpenLessAndroidPreferences.overlaySizeDp(this) + Log.i(TAG, "overlay layout refreshed sizeDp=$sizeDp roots=1") + } + + private fun reconcileOverlayRoots() { + val roots = synchronized(overlayRoots) { + (overlayRoots + listOfNotNull(rootView)).distinct().filter { it.isAttachedToWindow }.also { + overlayRoots.clear() + overlayRoots.addAll(it) + } + } + if (roots.isEmpty()) { + rootView = null + layoutParams = null + return + } + roots.dropLast(1).forEach { staleRoot -> + removeOverlayRoot(staleRoot) + synchronized(overlayRoots) { + overlayRoots.remove(staleRoot) + } + } + val activeRoot = roots.last() + synchronized(overlayRoots) { + overlayRoots.clear() + overlayRoots.add(activeRoot) + } + rootView = activeRoot + layoutParams = activeRoot.layoutParams as? WindowManager.LayoutParams + Log.i(TAG, "reconciled overlay roots kept=1 removed=${roots.size - 1}") + } + + private fun removeOverlayRoot(view: FrameLayout) { + try { + if (view.isAttachedToWindow) { + windowManager?.removeViewImmediate(view) + } + } catch (error: Throwable) { + Log.w(TAG, "remove overlay root failed", error) + } + } + + private fun buildIconButton(): ImageView { + return ImageView(this).apply { + setImageResource(R.drawable.ic_overlay_logo) + scaleType = ImageView.ScaleType.CENTER_INSIDE + setPadding(0, 0, 0, 0) + contentDescription = "OpenLess" + isClickable = false + isFocusable = false + } + } + + private fun applyOverlaySize(container: FrameLayout) { + if (!::iconButton.isInitialized) return + val sizeDp = OpenLessAndroidPreferences.overlaySizeDp(this) + val paddingDp = overlayPaddingDp(sizeDp) + val imageSizePx = dp((sizeDp - paddingDp * 2).coerceAtLeast(MIN_ICON_IMAGE_SIZE_DP)) + val paddingPx = dp(paddingDp) + container.setPadding(paddingPx, paddingPx, paddingPx, paddingPx) + (iconButton.layoutParams as? FrameLayout.LayoutParams)?.let { childParams -> + childParams.width = imageSizePx + childParams.height = imageSizePx + childParams.gravity = Gravity.CENTER + iconButton.layoutParams = childParams + } + container.requestLayout() + Log.i(TAG, "overlay size applied sizeDp=$sizeDp imagePx=$imageSizePx") + } + + private fun overlayPaddingDp(sizeDp: Int): Int { + return (sizeDp * ICON_PADDING_DP / DEFAULT_ICON_SIZE_DP).coerceIn(8, 20) + } + + private fun handleIconClick() { + if (processing) return + if (recording) { + stopRecordingFromOverlay() + return + } + if (!isTapActivationMode()) { + return + } + startRecordingFromOverlay() + } + + private fun handleKeyboardChanged(intent: Intent) { + val visible = intent.getBooleanExtra(EXTRA_KEYBOARD_VISIBLE, false) + keyboardVisible = visible + Log.i(TAG, "keyboard changed visible=$visible") + if (visible) { + showOverlay() + return + } + if (!recording && !processing) { + hideOverlay() + } + } + + private fun attachDragHandler(view: View, params: WindowManager.LayoutParams) { + view.setOnTouchListener { touchedView, event -> + when (event.actionMasked) { + MotionEvent.ACTION_DOWN -> { + dragging = false + swipeConsumed = false + longPressRecording = false + pendingSwipe = null + if (processing) { + return@setOnTouchListener true + } + dragStartX = event.rawX.toInt() + dragStartY = event.rawY.toInt() + paramStartX = params.x + paramStartY = params.y + if (!isTapActivationMode() && !recording && !processing) { + longPressRecording = true + startRecordingFromOverlay() + } + true + } + MotionEvent.ACTION_MOVE -> { + val dx = event.rawX.toInt() - dragStartX + val dy = event.rawY.toInt() - dragStartY + val verticalSwipe = detectVerticalSwipe(dx, dy) + if (recording && verticalSwipe != null && matchesConfiguredCancelSwipe(verticalSwipe) && !swipeConsumed) { + pendingSwipe = verticalSwipe + swipeConsumed = true + applySwipePreview(verticalSwipe) + return@setOnTouchListener true + } + val swipe = detectHorizontalSwipe(dx, dy) + if ((recording || armed || longPressRecording) && swipe != null && !swipeConsumed) { + pendingSwipe = swipe + swipeConsumed = true + applySwipePreview(swipe) + return@setOnTouchListener true + } + if (!processing && !armed && !recording && !longPressRecording && (abs(dx) > DRAG_SLOP_PX || abs(dy) > DRAG_SLOP_PX)) { + dragging = true + params.x = paramStartX + dx + params.y = paramStartY + dy + clampToScreen(params) + rootView?.let { windowManager?.updateViewLayout(it, params) } + } + true + } + MotionEvent.ACTION_UP -> { + if (!dragging) { + val swipe = pendingSwipe + if (swipe != null) { + commitSwipe(swipe) + } else if (longPressRecording || (!isTapActivationMode() && recording)) { + stopRecordingFromOverlay() + } else if (!isTapActivationMode()) { + setArmed(false) + } else if (!swipeConsumed) { + touchedView.performClick() + } + } else { + savePosition(params.x, params.y) + } + longPressRecording = false + pendingSwipe = null + swipeConsumed = false + true + } + MotionEvent.ACTION_CANCEL -> { + if (longPressRecording || (!isTapActivationMode() && recording)) { + stopRecordingFromOverlay() + } + longPressRecording = false + pendingSwipe = null + swipeConsumed = false + true + } + else -> false + } + } + } + + private fun applyVisualState(state: OverlayVisualState) { + if (!::iconContainer.isInitialized || !::iconButton.isInitialized) return + val (alpha, fill, stroke, strokeWidth, enabled) = when (state) { + OverlayVisualState.Idle -> VisualStyle( + alpha = 0.58f, + fill = Color.parseColor("#66202A36"), + stroke = Color.parseColor("#66FFFFFF"), + strokeWidth = 1, + enabled = true, + ) + OverlayVisualState.Armed -> VisualStyle( + alpha = 1f, + fill = Color.parseColor("#E6111827"), + stroke = Color.parseColor("#38BDF8"), + strokeWidth = 3, + enabled = true, + ) + OverlayVisualState.Recording -> VisualStyle( + alpha = 1f, + fill = Color.parseColor("#E6111827"), + stroke = Color.parseColor("#F43F5E"), + strokeWidth = 3, + enabled = true, + ) + OverlayVisualState.Processing -> VisualStyle( + alpha = 0.86f, + fill = Color.parseColor("#D1111827"), + stroke = Color.parseColor("#38BDF8"), + strokeWidth = 2, + enabled = true, + ) + OverlayVisualState.Error -> VisualStyle( + alpha = 0.95f, + fill = Color.parseColor("#E67F1D1D"), + stroke = Color.parseColor("#EF4444"), + strokeWidth = 2, + enabled = true, + ) + } + iconContainer.alpha = alpha + iconContainer.isEnabled = enabled + iconContainer.background = circleDrawable(fill, stroke, dp(strokeWidth)) + iconButton.isEnabled = enabled + } + + private fun setArmed(value: Boolean) { + armed = value + if (!recording && !processing) { + applyVisualState(if (value) OverlayVisualState.Armed else OverlayVisualState.Idle) + } + } + + private fun detectHorizontalSwipe(dx: Int, dy: Int): SwipeDirection? { + if (abs(dx) < dp(SWIPE_THRESHOLD_DP)) return null + if (abs(dy) > abs(dx) * SWIPE_VERTICAL_RATIO) return null + return if (dx < 0) SwipeDirection.Left else SwipeDirection.Right + } + + private fun detectVerticalSwipe(dx: Int, dy: Int): SwipeDirection? { + if (abs(dy) < dp(SWIPE_THRESHOLD_DP)) return null + if (abs(dx) > abs(dy) * SWIPE_VERTICAL_RATIO) return null + return if (dy < 0) SwipeDirection.Up else SwipeDirection.Down + } + + private fun matchesConfiguredCancelSwipe(direction: SwipeDirection): Boolean { + val configured = OpenLessAndroidPreferences.overlayCancelSwipeDirection(this) + return (direction == SwipeDirection.Up && configured == "up") || + (direction == SwipeDirection.Down && configured == "down") + } + + private fun applySwipePreview(direction: SwipeDirection) { + when (direction) { + SwipeDirection.Left -> applyVisualState(OverlayVisualState.Armed) + SwipeDirection.Right -> applyVisualState(OverlayVisualState.Processing) + SwipeDirection.Up, + SwipeDirection.Down -> applyVisualState(OverlayVisualState.Error) + } + } + + private fun commitSwipe(direction: SwipeDirection) { + Log.i(TAG, "commit swipe direction=$direction recording=$recording processing=$processing") + when (direction) { + SwipeDirection.Left -> handleLeftSwipe() + SwipeDirection.Right -> finalizeQaFromOverlay() + SwipeDirection.Up, + SwipeDirection.Down -> cancelRecordingFromOverlay(direction) + } + } + + private fun cancelRecordingFromOverlay(direction: SwipeDirection) { + if (!recording || !matchesConfiguredCancelSwipe(direction)) { + return + } + try { + OpenLessNative.nativeCancelDictation() + recording = false + processing = false + longPressRecording = false + setArmed(false) + applyVisualState(OverlayVisualState.Idle) + Log.i(TAG, "recording cancelled from overlay direction=$direction") + } catch (error: Throwable) { + Log.w(TAG, "cancel dictation bridge unavailable", error) + applyVisualState(OverlayVisualState.Error) + showToast("语音服务未就绪,请打开 OpenLess 后重试") + } + } + + private fun handleLeftSwipe() { + when (OpenLessAndroidPreferences.overlayLeftSwipeAction(this)) { + "style_pack" -> { + switchStylePackFromOverlay() + if (recording) { + stopRecordingFromOverlay() + } + } + else -> stopRecordingFromOverlay(translation = true) + } + } + + private fun switchStylePackFromOverlay() { + try { + OpenLessNative.nativeSwitchStylePack() + setArmed(false) + } catch (error: Throwable) { + Log.w(TAG, "switch style pack bridge unavailable", error) + applyVisualState(OverlayVisualState.Error) + showToast("语音服务未就绪,请打开 OpenLess 后重试") + } + } + + private fun openQaFromOverlay() { + try { + Log.i(TAG, "open QA from overlay") + OpenLessNative.nativeOpenQaFromOverlay() + setArmed(false) + } catch (error: Throwable) { + Log.w(TAG, "open QA bridge unavailable", error) + applyVisualState(OverlayVisualState.Error) + showToast("问答服务未就绪,请打开 OpenLess 后重试") + } + } + + private fun finalizeQaFromOverlay() { + try { + Log.i(TAG, "finalize QA from overlay") + OpenLessNative.nativeFinalizeQaFromOverlay() + recording = false + processing = true + setArmed(false) + applyVisualState(OverlayVisualState.Processing) + } catch (error: Throwable) { + Log.w(TAG, "finalize QA bridge unavailable", error) + processing = false + applyVisualState(OverlayVisualState.Error) + showToast("问答服务未就绪,请打开 OpenLess 后重试") + } + } + + private fun startRecordingFromOverlay(translation: Boolean = false) { + showOverlay() + if (tryPromoteRecordingForeground()) { + try { + if (translation) { + OpenLessNative.nativeStartDictationWithTranslation(true) + } else { + OpenLessNative.nativeStartDictation() + } + recording = true + processing = false + setArmed(false) + applyVisualState(OverlayVisualState.Recording) + } catch (error: Throwable) { + Log.w(TAG, "start dictation bridge unavailable", error) + recording = false + processing = false + applyVisualState(OverlayVisualState.Error) + showToast("语音服务未就绪,请打开 OpenLess 后重试") + } + return + } + applyVisualState(OverlayVisualState.Error) + } + + private fun stopRecordingFromOverlay(translation: Boolean = false) { + try { + recording = false + processing = true + applyVisualState(OverlayVisualState.Processing) + if (translation) { + OpenLessNative.nativeStopDictationWithTranslation(true) + } else { + OpenLessNative.nativeStopDictation() + } + } catch (error: Throwable) { + Log.w(TAG, "stop dictation bridge unavailable", error) + recording = false + processing = false + applyVisualState(OverlayVisualState.Error) + showToast("语音服务未就绪,请打开 OpenLess 后重试") + } + } + + private fun tryPromoteRecordingForeground(): Boolean { + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + showToast("请先授予麦克风权限") + return false + } + val notification = buildNotification("录音中") + return try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground( + NOTIFICATION_ID, + notification, + ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE, + ) + } else { + startForeground(NOTIFICATION_ID, notification) + } + true + } catch (error: SecurityException) { + Log.w(TAG, "microphone foreground service not allowed from current state", error) + showToast("系统限制后台录音,请在 OpenLess 内开始") + false + } + } + + private fun buildNotification(contentText: String): Notification { + val channelId = "openless_overlay" + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val nm = getSystemService(NotificationManager::class.java) + nm.createNotificationChannel( + NotificationChannel(channelId, "OpenLess Overlay", NotificationManager.IMPORTANCE_LOW), + ) + } + return Notification.Builder(this, channelId) + .setContentTitle("OpenLess") + .setContentText(contentText) + .setSmallIcon(R.mipmap.ic_launcher) + .build() + } + + private fun circleDrawable(color: Int, strokeColor: Int, strokeWidth: Int): GradientDrawable { + return GradientDrawable().apply { + shape = GradientDrawable.OVAL + setColor(color) + setStroke(strokeWidth, strokeColor) + } + } + + private fun overlaySize(): Int { + val root = rootView + val measured = maxOf(root?.width ?: 0, root?.height ?: 0) + return measured.takeIf { it > 0 } ?: dp(OpenLessAndroidPreferences.overlaySizeDp(this)) + } + + private fun clampToScreen(params: WindowManager.LayoutParams) { + val iconSize = overlaySize() + val margin = dp(8) + val maxX = (resources.displayMetrics.widthPixels - iconSize - margin).coerceAtLeast(margin) + val maxY = (resources.displayMetrics.heightPixels - iconSize - margin).coerceAtLeast(margin) + params.x = params.x.coerceIn(margin, maxX) + params.y = params.y.coerceIn(margin, maxY) + } + + private fun loadSavedPosition(): Pair { + val prefs = getSharedPreferences(PREFS_NAME, MODE_PRIVATE) + val defaultX = dp(24) + val defaultY = dp(120) + val x = prefs.getInt(PREF_KEY_X, defaultX) + val y = prefs.getInt(PREF_KEY_Y, defaultY) + return x to y + } + + private fun savePosition(x: Int, y: Int) { + getSharedPreferences(PREFS_NAME, MODE_PRIVATE) + .edit() + .putInt(PREF_KEY_X, x) + .putInt(PREF_KEY_Y, y) + .apply() + } + + private fun isTapActivationMode(): Boolean { + return OpenLessAndroidPreferences.overlayActivationMode(this) == "tap" + } + + private fun showToast(message: String) { + Toast.makeText(this, message, Toast.LENGTH_SHORT).show() + } + + private fun dp(value: Int): Int { + return (value * resources.displayMetrics.density).toInt() + } + + private inline fun withOverlayLock(block: () -> T): T { + return synchronized(overlayLock) { + block() + } + } + + private data class VisualStyle( + val alpha: Float, + val fill: Int, + val stroke: Int, + val strokeWidth: Int, + val enabled: Boolean, + ) + + private enum class OverlayVisualState { + Idle, + Armed, + Recording, + Processing, + Error, + } + + private enum class SwipeDirection { + Left, + Right, + Up, + Down, + } + + companion object { + const val ACTION_SHOW = "com.openless.app.overlay.SHOW" + const val ACTION_HIDE = "com.openless.app.overlay.HIDE" + const val ACTION_REPLACE_OVERLAY = "com.openless.app.overlay.REPLACE_OVERLAY" + const val ACTION_REFRESH_LAYOUT = "com.openless.app.overlay.REFRESH_LAYOUT" + const val ACTION_TOGGLE_EXPAND = "com.openless.app.overlay.TOGGLE_EXPAND" + const val ACTION_START_RECORDING = "com.openless.app.overlay.START_RECORDING" + const val ACTION_KEYBOARD_CHANGED = "com.openless.app.overlay.KEYBOARD_CHANGED" + const val EXTRA_KEYBOARD_VISIBLE = "keyboard_visible" + const val EXTRA_KEYBOARD_TOP = "keyboard_top" + const val EXTRA_KEYBOARD_BOTTOM = "keyboard_bottom" + private const val DEFAULT_ICON_SIZE_DP = 72 + private const val MIN_ICON_IMAGE_SIZE_DP = 32 + private const val ICON_PADDING_DP = 12 + private const val DRAG_SLOP_PX = 8 + private const val SWIPE_THRESHOLD_DP = 56 + private const val SWIPE_VERTICAL_RATIO = 0.6f + private const val PREFS_NAME = "openless_overlay" + private const val PREF_KEY_X = "overlay_x" + private const val PREF_KEY_Y = "overlay_y" + private const val NOTIFICATION_ID = 42001 + private const val TAG = "OpenLessOverlayService" + + private val overlayLock = Any() + private val overlayRoots = mutableListOf() + + @Volatile + var instance: OpenLessOverlayService? = null + private set + } +} diff --git a/openless-all/app/android/kotlin/OpenLessPermissionBridge.kt b/openless-all/app/android/kotlin/OpenLessPermissionBridge.kt new file mode 100644 index 00000000..9fb437fd --- /dev/null +++ b/openless-all/app/android/kotlin/OpenLessPermissionBridge.kt @@ -0,0 +1,46 @@ +package com.openless.app + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import android.util.Log +import java.util.concurrent.atomic.AtomicBoolean + +object OpenLessPermissionBridge { + private const val TAG = "OpenLessPermissionBridge" + + private val requestInFlight = AtomicBoolean(false) + + @JvmStatic + fun requestRecordAudioPermission(context: Context): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + return true + } + if (context.checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { + return true + } + if (!requestInFlight.compareAndSet(false, true)) { + Log.i(TAG, "RECORD_AUDIO permission request already in flight") + return false + } + return try { + val intent = Intent(context, MicrophonePermissionActivity::class.java).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + context.startActivity(intent) + false + } catch (error: Throwable) { + requestInFlight.set(false) + Log.w(TAG, "failed to launch RECORD_AUDIO permission activity", error) + context.checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + } + } + + @JvmStatic + fun resolveRecordAudioPermission(granted: Boolean) { + Log.i(TAG, "RECORD_AUDIO permission completed granted=$granted") + requestInFlight.set(false) + } +} diff --git a/openless-all/app/android/kotlin/OverlayPermissionActivity.kt b/openless-all/app/android/kotlin/OverlayPermissionActivity.kt new file mode 100644 index 00000000..d3a05f8f --- /dev/null +++ b/openless-all/app/android/kotlin/OverlayPermissionActivity.kt @@ -0,0 +1,27 @@ +package com.openless.app + +import android.app.Activity +import android.content.Intent +import android.net.Uri +import android.os.Build +import android.os.Bundle +import android.provider.Settings + +/** + * 引导用户授权 SYSTEM_ALERT_WINDOW。 + * Rust 命令 request_android_overlay_permission 通过 Intent 启动本 Activity。 + */ +class OverlayPermissionActivity : Activity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && !Settings.canDrawOverlays(this)) { + val intent = Intent( + Settings.ACTION_MANAGE_OVERLAY_PERMISSION, + Uri.parse("package:$packageName"), + ) + startActivity(intent) + } + finish() + } +} diff --git a/openless-all/app/android/kotlin/README.md b/openless-all/app/android/kotlin/README.md new file mode 100644 index 00000000..465107de --- /dev/null +++ b/openless-all/app/android/kotlin/README.md @@ -0,0 +1,27 @@ +# Android Kotlin scaffolding + +Copy these files into `src-tauri/gen/android/` after running: + +```bash +cd openless-all/app +npm run tauri:android:init +``` + +## Copy / merge paths + +| Source (this folder) | Destination (after init) | +| --- | --- | +| `OpenLessOverlayService.kt` | `gen/android/app/src/main/java/com/openless/app/OpenLessOverlayService.kt` | +| `OverlayPermissionActivity.kt` | `gen/android/app/src/main/java/com/openless/app/OverlayPermissionActivity.kt` | +| `AndroidManifest.v1.snippet.xml` | 在 `../manifests/`,merge 进 `gen/android/.../AndroidManifest.xml` | +| `AndroidManifest.v3.snippet.xml` | 在 `../manifests/`,**future / not complete** — overlay v3 only | + +Tauri `android init` generates the base manifest under `gen/android/app/src/main/AndroidManifest.xml`. +Merge the v1 snippet permissions into that file before building APK v1. + +## Manifest snippets + +- **v1** (`AndroidManifest.v1.snippet.xml`): `RECORD_AUDIO` and `MODIFY_AUDIO_SETTINGS` for in-app dictation — required for APK v1. +- **v3** (`AndroidManifest.v3.snippet.xml`): overlay + foreground service — **not complete / future**. + +Do not treat v3 snippets as shipped; they document planned permissions and service entries only. diff --git a/openless-all/app/android/manifests/AndroidManifest.v1.snippet.xml b/openless-all/app/android/manifests/AndroidManifest.v1.snippet.xml new file mode 100644 index 00000000..7f012e3c --- /dev/null +++ b/openless-all/app/android/manifests/AndroidManifest.v1.snippet.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/openless-all/app/android/manifests/AndroidManifest.v3.snippet.xml b/openless-all/app/android/manifests/AndroidManifest.v3.snippet.xml new file mode 100644 index 00000000..859f99b6 --- /dev/null +++ b/openless-all/app/android/manifests/AndroidManifest.v3.snippet.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + diff --git a/openless-all/app/android/manifests/res/xml/openless_accessibility_config.xml b/openless-all/app/android/manifests/res/xml/openless_accessibility_config.xml new file mode 100644 index 00000000..281b8c75 --- /dev/null +++ b/openless-all/app/android/manifests/res/xml/openless_accessibility_config.xml @@ -0,0 +1,9 @@ + + diff --git a/openless-all/app/android/manifests/res/xml/openless_ime_method.xml b/openless-all/app/android/manifests/res/xml/openless_ime_method.xml new file mode 100644 index 00000000..5a2afd44 --- /dev/null +++ b/openless-all/app/android/manifests/res/xml/openless_ime_method.xml @@ -0,0 +1,9 @@ + + + + diff --git a/openless-all/app/package.json b/openless-all/app/package.json index a2b584c8..0569e026 100644 --- a/openless-all/app/package.json +++ b/openless-all/app/package.json @@ -8,6 +8,12 @@ "build": "tsc && vite build", "preview": "vite preview", "tauri": "tauri", + "tauri:android:init": "tauri android init", + "tauri:android:dev": "tauri android dev", + "tauri:android:build": "tauri android build --apk --debug --target aarch64 armv7 i686 x86_64 --split-per-abi", + "merge:android-v1-manifest": "node scripts/merge-android-v1-manifest.mjs", + "merge:android-overlay-manifest": "node scripts/merge-android-overlay-manifest.mjs", + "copy:android-scaffolding": "node scripts/copy-android-scaffolding.mjs", "check:aura-skin": "node scripts/aura-skin-contract.test.mjs", "check:macos-capsule-spaces": "node scripts/macos-capsule-spaces-contract.test.mjs", "check:hotkey-injection": "node scripts/check-hotkey-injection.mjs" diff --git a/openless-all/app/public/AppIcon.png b/openless-all/app/public/AppIcon.png index ee876eb5..3bdcb6b8 100755 Binary files a/openless-all/app/public/AppIcon.png and b/openless-all/app/public/AppIcon.png differ diff --git a/openless-all/app/scripts/copy-android-scaffolding.mjs b/openless-all/app/scripts/copy-android-scaffolding.mjs new file mode 100644 index 00000000..d785d2e1 --- /dev/null +++ b/openless-all/app/scripts/copy-android-scaffolding.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const appRoot = fileURLToPath(new URL('..', import.meta.url)); +const kotlinRoot = join(appRoot, 'android/kotlin'); +const manifestsRoot = join(appRoot, 'android/manifests'); +const androidIconRoot = join(appRoot, 'src-tauri/icons/android'); +const genRoot = join(appRoot, 'src-tauri/gen/android/app/src/main'); +const kotlinDest = join(genRoot, 'java/com/openless/app'); +const resDest = join(genRoot, 'res'); +const resXmlDest = join(genRoot, 'res/xml'); + +const KOTLIN_FILES = [ + 'OpenLessAppContext.kt', + 'OpenLessNative.kt', + 'OpenLessPermissionBridge.kt', + 'MicrophonePermissionActivity.kt', + 'OpenLessAndroidPreferences.kt', + 'OpenLessApplication.kt', + 'OpenLessOverlayService.kt', + 'OpenLessOverlayBridge.kt', + 'OpenLessAccessibilityService.kt', + 'OpenLessAccessibilityCommandReceiver.kt', + 'OverlayPermissionActivity.kt', +]; + +const XML_FILES = [ + ['res/xml/openless_accessibility_config.xml', 'openless_accessibility_config.xml'], +]; + +const GENERATED_ACCESSIBILITY_CONFIG = ` + +`; + +const GENERATED_STRINGS_SNIPPET = ` + OpenLess uses accessibility to detect the keyboard and paste dictation results without switching your current keyboard. +`; + +function printHelp() { + console.log(`Usage: node scripts/copy-android-scaffolding.mjs [options] + +Copy Kotlin scaffolding and XML resources into gen/android after \`tauri android init\`. + +Options: + --dry-run Print planned copies without writing + --help Show this help text +`); +} + +function parseArgs(argv) { + let dryRun = false; + for (const arg of argv) { + if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + if (arg === '--dry-run') { + dryRun = true; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + return { dryRun }; +} + +function ensureDir(path, dryRun) { + if (dryRun || existsSync(path)) { + return; + } + mkdirSync(path, { recursive: true }); +} + +function mergeStringsXml(dryRun) { + const stringsPath = join(genRoot, 'res/values/strings.xml'); + if (!existsSync(stringsPath)) { + const content = ` +${GENERATED_STRINGS_SNIPPET} + +`; + if (dryRun) { + console.log(`[dry-run] Would create ${stringsPath}`); + return; + } + ensureDir(dirname(stringsPath), dryRun); + writeFileSync(stringsPath, content, 'utf8'); + console.log(`Created ${stringsPath}`); + return; + } + + const existing = readFileSync(stringsPath, 'utf8'); + if (existing.includes('openless_accessibility_description')) { + console.log(`OpenLess strings already present in ${stringsPath}; skipping.`); + return; + } + + const updated = existing.replace('', `${GENERATED_STRINGS_SNIPPET}\n`); + if (dryRun) { + console.log(`[dry-run] Would merge OpenLess strings into ${stringsPath}`); + return; + } + writeFileSync(stringsPath, updated, 'utf8'); + console.log(`Merged OpenLess strings into ${stringsPath}`); +} + +function copyDirectoryContents(srcRoot, destRoot, dryRun) { + if (!existsSync(srcRoot)) { + throw new Error(`Missing Android icon resources: ${srcRoot}`); + } + + ensureDir(destRoot, dryRun); + for (const entry of readdirSync(srcRoot)) { + const src = join(srcRoot, entry); + const dest = join(destRoot, entry); + if (statSync(src).isDirectory()) { + copyDirectoryContents(src, dest, dryRun); + continue; + } + if (dryRun) { + console.log(`[dry-run] Would copy ${src} -> ${dest}`); + continue; + } + ensureDir(dirname(dest), dryRun); + copyFileSync(src, dest); + console.log(`Copied ${dest}`); + } +} + +function main() { + const { dryRun } = parseArgs(process.argv.slice(2)); + + if (!existsSync(join(appRoot, 'src-tauri/gen/android'))) { + throw new Error( + `Generated Android project not found under src-tauri/gen/android.\nRun "npm run tauri -- android init --ci" first.`, + ); + } + + ensureDir(kotlinDest, dryRun); + ensureDir(resXmlDest, dryRun); + copyDirectoryContents(androidIconRoot, resDest, dryRun); + + for (const file of KOTLIN_FILES) { + const src = join(kotlinRoot, file); + const dest = join(kotlinDest, file); + if (!existsSync(src)) { + throw new Error(`Missing scaffolding file: ${src}`); + } + if (dryRun) { + console.log(`[dry-run] Would copy ${src} -> ${dest}`); + continue; + } + copyFileSync(src, dest); + console.log(`Copied ${file}`); + } + + for (const [relSrc, destName] of XML_FILES) { + const src = join(manifestsRoot, relSrc); + const dest = join(resXmlDest, destName); + const content = existsSync(src) + ? readFileSync(src, 'utf8') + : GENERATED_ACCESSIBILITY_CONFIG; + if (dryRun) { + console.log(`[dry-run] Would write ${dest}`); + continue; + } + writeFileSync(dest, content, 'utf8'); + console.log(`Wrote ${destName}`); + } + + mergeStringsXml(dryRun); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} diff --git a/openless-all/app/scripts/merge-android-overlay-manifest.mjs b/openless-all/app/scripts/merge-android-overlay-manifest.mjs new file mode 100644 index 00000000..66810c2f --- /dev/null +++ b/openless-all/app/scripts/merge-android-overlay-manifest.mjs @@ -0,0 +1,174 @@ +#!/usr/bin/env node +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const targetPath = fileURLToPath( + new URL('../src-tauri/gen/android/app/src/main/AndroidManifest.xml', import.meta.url), +); + +const PERMISSIONS = [ + 'android.permission.SYSTEM_ALERT_WINDOW', + 'android.permission.FOREGROUND_SERVICE', + 'android.permission.FOREGROUND_SERVICE_MICROPHONE', +]; + +const APPLICATION_SNIPPET = ` + + +`; + +const SERVICE_SNIPPETS = [ + ``, + ` + + + + + `, + ``, + ``, + ``, +]; + +function printHelp() { + console.log(`Usage: node scripts/merge-android-overlay-manifest.mjs [options] + +Merge overlay / accessibility declarations into generated AndroidManifest.xml. + +Options: + --dry-run Print planned changes without writing + --help Show this help text +`); +} + +function parseArgs(argv) { + let dryRun = false; + for (const arg of argv) { + if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + if (arg === '--dry-run') { + dryRun = true; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + return { dryRun }; +} + +function permissionExists(manifestXml, permissionName) { + const escaped = permissionName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`]*android:name="${escaped}"[^>]*\\/?>`).test(manifestXml); +} + +function mergePermissions(manifestXml) { + let content = manifestXml; + let changed = false; + for (const name of PERMISSIONS) { + if (permissionExists(content, name)) { + continue; + } + const line = ` `; + const applicationIdx = content.indexOf(''); + } + content = `${content.slice(0, applicationIdx)}${line}\n${content.slice(applicationIdx)}`; + changed = true; + } + return { content, changed }; +} + +function ensureApplicationName(manifestXml) { + if (/android:name="\.OpenLessApplication"/.test(manifestXml)) { + return { content: manifestXml, changed: false }; + } + const updated = manifestXml.replace( + /)/, + ''); + if (closingIdx === -1) { + throw new Error('Target manifest is missing '); + } + + for (const snippet of SERVICE_SNIPPETS) { + const marker = snippet.match(/android:name="([^"]+)"/)?.[1]; + if (!marker || snippetExists(content, marker)) { + continue; + } + content = `${content.slice(0, closingIdx)} ${snippet}\n${content.slice(closingIdx)}`; + changed = true; + } + + return { content, changed }; +} + +function main() { + const { dryRun } = parseArgs(process.argv.slice(2)); + + if (!existsSync(targetPath)) { + throw new Error( + `Generated Android manifest not found: ${targetPath}\nRun "npm run tauri -- android init --ci" first.`, + ); + } + + let content = readFileSync(targetPath, 'utf8'); + let changed = false; + + for (const step of [mergePermissions, ensureApplicationName, mergeApplicationChildren]) { + const result = step(content); + content = result.content; + changed = changed || result.changed; + } + + if (!changed) { + console.log(`Overlay manifest entries already present in ${targetPath}; skipping merge.`); + return; + } + + if (dryRun) { + console.log(`[dry-run] Would merge overlay manifest entries into ${targetPath}`); + return; + } + + writeFileSync(targetPath, content, 'utf8'); + console.log(`Merged overlay / accessibility entries into ${targetPath}`); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} diff --git a/openless-all/app/scripts/merge-android-v1-manifest.mjs b/openless-all/app/scripts/merge-android-v1-manifest.mjs new file mode 100644 index 00000000..e5982d8d --- /dev/null +++ b/openless-all/app/scripts/merge-android-v1-manifest.mjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const targetPath = fileURLToPath( + new URL('../src-tauri/gen/android/app/src/main/AndroidManifest.xml', import.meta.url), +); +const sourcePath = fileURLToPath( + new URL('../android/manifests/AndroidManifest.v1.snippet.xml', import.meta.url), +); + +const PERMISSION_LINE_RE = + /]*android:name="([^"]+)"[^>]*\/?>/g; + +function printHelp() { + console.log(`Usage: node scripts/merge-android-v1-manifest.mjs [options] + +Merge APK v1 permissions from android/manifests into the generated +AndroidManifest.xml (post \`tauri android init\`). + +Options: + --dry-run Print planned changes without writing the manifest + --help Show this help text + +Target: ${targetPath} +Source: ${sourcePath} +`); +} + +function parseArgs(argv) { + let dryRun = false; + for (const arg of argv) { + if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + if (arg === '--dry-run') { + dryRun = true; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + return { dryRun }; +} + +function extractPermissionLines(snippetXml) { + const lines = []; + for (const match of snippetXml.matchAll(PERMISSION_LINE_RE)) { + lines.push({ name: match[1], line: match[0] }); + } + if (lines.length === 0) { + throw new Error( + `Source manifest snippet does not contain any uses-permission entries: ${sourcePath}`, + ); + } + return lines; +} + +function permissionExists(manifestXml, permissionName) { + const escaped = permissionName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`]*android:name="${escaped}"[^>]*\\/?>`).test(manifestXml); +} + +function mergePermissionLines(manifestXml, permissionLines) { + const missing = permissionLines.filter((permission) => !permissionExists(manifestXml, permission.name)); + if (missing.length === 0) { + return { changed: false, content: manifestXml }; + } + + const insertionBlock = (indent) => missing.map((permission) => `${indent}${permission.line}`).join('\n') + '\n'; + + const applicationIdx = manifestXml.indexOf(''); + if (closingManifestIdx === -1) { + throw new Error(`Target manifest is missing : ${targetPath}`); + } + + const indent = ' '; + return { + changed: true, + content: `${manifestXml.slice(0, closingManifestIdx)}${insertionBlock(indent)}${manifestXml.slice(closingManifestIdx)}`, + }; +} + +function main() { + const { dryRun } = parseArgs(process.argv.slice(2)); + + if (!existsSync(targetPath)) { + throw new Error( + `Generated Android manifest not found: ${targetPath}\nRun "npm run tauri -- android init --ci" first.`, + ); + } + if (!existsSync(sourcePath)) { + throw new Error(`Source manifest snippet not found: ${sourcePath}`); + } + + const permissionLines = extractPermissionLines(readFileSync(sourcePath, 'utf8')); + const manifestXml = readFileSync(targetPath, 'utf8'); + const { changed, content } = mergePermissionLines(manifestXml, permissionLines); + + if (!changed) { + console.log(`APK v1 permissions already present in ${targetPath}; skipping merge.`); + return; + } + + if (dryRun) { + console.log(`[dry-run] Would merge APK v1 permissions into ${targetPath}`); + for (const permission of permissionLines) { + console.log(`[dry-run] Permission line: ${permission.line}`); + } + return; + } + + writeFileSync(targetPath, content, 'utf8'); + console.log(`Merged APK v1 permissions into ${targetPath}`); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index 1fbadbdf..f2649478 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -155,7 +155,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "wl-clipboard-rs", "x11rb", ] @@ -1523,7 +1523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3718,9 +3718,11 @@ dependencies = [ "foundry-local-sdk", "futures-util", "global-hotkey", + "jni 0.21.1", "keyring", "libc", "log", + "ndk-context", "objc2 0.5.2", "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", @@ -3733,6 +3735,7 @@ dependencies = [ "sha2", "sherpa-onnx", "simplelog", + "tao", "tar", "tauri", "tauri-build", @@ -3818,7 +3821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -4309,7 +4312,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4685,7 +4688,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4743,7 +4746,7 @@ dependencies = [ "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5870,7 +5873,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6072,11 +6075,11 @@ dependencies = [ "futures-util", "log", "rustls", - "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", "tungstenite", + "webpki-roots 0.26.11", ] [[package]] diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index 3030f5d3..85734a84 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -18,11 +18,10 @@ cc = "1.1" [dependencies] # 锁 ~2.11 因为 npm @tauri-apps/api 与 plugin-dialog 都已升 2.11; # tauri build 跨 minor 一致性检查会拒绝 npm 2.11 + Rust 2.10 的组合。 -tauri = { version = "~2.11", features = ["macos-private-api", "tray-icon"] } +# macos-private-api must live in [dependencies] so tauri_build can match tauri.conf.json +# macOSPrivateApi; tray-icon stays desktop-only in the target table below. +tauri = { version = "~2.11", features = ["macos-private-api"] } tauri-plugin-shell = "2" -tauri-plugin-updater = "2" -tauri-plugin-single-instance = "2" -tauri-plugin-autostart = "2" tauri-plugin-dialog = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -32,9 +31,9 @@ sha2 = "0.10" bzip2 = "0.4" tar = "0.4" tokio = { version = "1", features = ["full"] } -tokio-tungstenite = { version = "0.24", features = ["rustls-tls-native-roots"] } +tokio-tungstenite = { version = "0.24", features = ["rustls-tls-webpki-roots"] } futures-util = "0.3" -reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "native-tls", "stream", "system-proxy"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "stream", "system-proxy"] } zip = "2" thiserror = "1" anyhow = "1" @@ -49,10 +48,17 @@ bytes = "1" url = "2" raw-window-handle = "0.6" ferrous-opencc = "0.4" +# Audio capture — shared across desktop and mobile. +cpal = "0.15" -# Hotkey + audio + insertion +# Desktop-only plugins, hotkey/insertion helpers, and tray-icon (not built for mobile). +[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] +reqwest = { version = "0.12", default-features = false, features = ["native-tls"] } +tauri = { version = "~2.11", features = ["macos-private-api", "tray-icon"] } +tauri-plugin-updater = "2" +tauri-plugin-single-instance = "2" +tauri-plugin-autostart = "2" global-hotkey = "0.6" -cpal = "0.15" enigo = "0.2" arboard = { version = "3", features = ["wayland-data-control"] } @@ -68,11 +74,16 @@ features = ["windows-native"] [target.'cfg(target_os = "linux")'.dependencies] dbus = "0.9" -[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies.keyring] +[target.'cfg(all(unix, not(target_os = "macos"), not(target_os = "android")))'.dependencies.keyring] version = "3.6.3" default-features = false features = ["linux-native-sync-persistent", "crypto-rust"] +[target.'cfg(target_os = "android")'.dependencies] +jni = "0.21" +ndk-context = "0.1" +tao = "0.35" + [target.'cfg(target_os = "macos")'.dependencies] block2 = "0.5" core-foundation = "0.10" diff --git a/openless-all/app/src-tauri/build.rs b/openless-all/app/src-tauri/build.rs index fef616ba..c8995eb7 100644 --- a/openless-all/app/src-tauri/build.rs +++ b/openless-all/app/src-tauri/build.rs @@ -5,9 +5,20 @@ fn main() { #[cfg(target_os = "macos")] build_qwen_asr_macos(); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("android") { + link_android_cpp_runtime(); + } + tauri_build::build(); } +/// cpal → oboe → oboe-sys 会编译 C++;最终 cdylib 需显式链接 NDK libc++。 +fn link_android_cpp_runtime() { + // oboe-ext 已部分静态链入 libc++;补链 c++abi 提供 __cxa_pure_virtual 等 ABI 符号。 + println!("cargo:rustc-link-lib=c++_static"); + println!("cargo:rustc-link-lib=c++abi"); +} + #[cfg(target_os = "windows")] fn link_windows_common_controls_v6_manifest_dependency() { let mut source_path = std::path::PathBuf::from( diff --git a/openless-all/app/src-tauri/capabilities/default.json b/openless-all/app/src-tauri/capabilities/default.json index 760d74ed..759ae4f2 100644 --- a/openless-all/app/src-tauri/capabilities/default.json +++ b/openless-all/app/src-tauri/capabilities/default.json @@ -2,6 +2,7 @@ "$schema": "../gen/schemas/desktop-schema.json", "identifier": "default", "description": "Default capabilities for OpenLess windows", + "platforms": ["macOS", "windows", "linux"], "windows": ["main", "capsule", "qa", "less-computer", "less-computer-glow"], "permissions": [ "core:default", diff --git a/openless-all/app/src-tauri/capabilities/mobile.json b/openless-all/app/src-tauri/capabilities/mobile.json new file mode 100644 index 00000000..911ad21d --- /dev/null +++ b/openless-all/app/src-tauri/capabilities/mobile.json @@ -0,0 +1,18 @@ +{ + "$schema": "../gen/schemas/mobile-schema.json", + "identifier": "mobile", + "description": "Capabilities for OpenLess Android main window", + "platforms": ["android"], + "windows": ["main", "qa"], + "permissions": [ + "core:default", + "core:window:default", + "core:window:allow-show", + "core:window:allow-hide", + "core:window:allow-set-focus", + "core:webview:default", + "core:event:default", + "shell:allow-open", + "dialog:default" + ] +} diff --git a/openless-all/app/src-tauri/icons/android/drawable/ic_overlay_logo.xml b/openless-all/app/src-tauri/icons/android/drawable/ic_overlay_logo.xml new file mode 100644 index 00000000..12c75565 --- /dev/null +++ b/openless-all/app/src-tauri/icons/android/drawable/ic_overlay_logo.xml @@ -0,0 +1,4 @@ + + diff --git a/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png b/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png index 776a8ca2..f8696157 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png and b/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png b/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png index 5d3b4d18..fff60b8a 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png and b/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png b/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png index 184e80b9..9a133041 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png and b/openless-all/app/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png b/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png index cda13e09..163c19bd 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png and b/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png b/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png index c300c13e..48132e3e 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png and b/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png b/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png index 6ea9db97..0ed6b5b3 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png and b/openless-all/app/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png b/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png index 42f5c671..eeb2a49b 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png and b/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png b/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png index 7e77d7df..bf5c0c5f 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png and b/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png b/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png index 83bc0d45..16125b39 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png and b/openless-all/app/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png b/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png index 7d53a159..c51aef19 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png and b/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png b/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png index bb34a590..4c063c8c 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png and b/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png b/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png index 6490428c..4ca17315 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png and b/openless-all/app/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png b/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png index cfe887bf..3964920c 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png and b/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png b/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png index 8baebd64..56d8948a 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png and b/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png differ diff --git a/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png b/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png index 404db72a..bd389617 100644 Binary files a/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png and b/openless-all/app/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/openless-all/app/src-tauri/src/android/accessibility.rs b/openless-all/app/src-tauri/src/android/accessibility.rs new file mode 100644 index 00000000..ec5ec976 --- /dev/null +++ b/openless-all/app/src-tauri/src/android/accessibility.rs @@ -0,0 +1,123 @@ +//! Android accessibility service integration for keyboard detection and paste insertion. + +use serde::Serialize; + +use crate::android::types::{AndroidAccessibilityState, AndroidAccessibilityStatus}; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AndroidAccessibilityPermissionResult { + pub launched: bool, + pub message: String, +} + +pub fn get_android_accessibility_status() -> AndroidAccessibilityStatus { + #[cfg(target_os = "android")] + { + android_impl::get_android_accessibility_status() + } + + #[cfg(not(target_os = "android"))] + { + AndroidAccessibilityStatus { + state: AndroidAccessibilityState::NotAndroid, + enabled: false, + message: "Android accessibility backend is only available on Android".to_string(), + } + } +} + +pub fn request_android_accessibility_permission() -> AndroidAccessibilityPermissionResult { + #[cfg(target_os = "android")] + { + android_impl::request_android_accessibility_permission() + } + + #[cfg(not(target_os = "android"))] + { + AndroidAccessibilityPermissionResult { + launched: false, + message: "Android accessibility settings are only available on Android".to_string(), + } + } +} + +pub fn paste_via_accessibility() -> bool { + #[cfg(target_os = "android")] + { + return android_impl::paste_via_accessibility(); + } + + #[cfg(not(target_os = "android"))] + false +} + +#[cfg(target_os = "android")] +mod android_impl { + use super::{AndroidAccessibilityPermissionResult, AndroidAccessibilityStatus}; + use crate::android::types::{AndroidAccessibilityState, AndroidAccessibilityStatus as Status}; + + pub fn get_android_accessibility_status() -> AndroidAccessibilityStatus { + let enabled = match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::accessibility_enabled(env, context) + }) { + Ok(enabled) => enabled, + Err(error) => { + return Status { + state: AndroidAccessibilityState::NotEnabled, + enabled: false, + message: error, + }; + } + }; + if !enabled { + return Status { + state: AndroidAccessibilityState::NotEnabled, + enabled: false, + message: "请在系统设置中启用 OpenLess 无障碍服务".to_string(), + }; + } + + match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::accessibility_operational(env, context) + }) { + Ok(true) => Status { + state: AndroidAccessibilityState::Enabled, + enabled: true, + message: "无障碍服务已启用".to_string(), + }, + Ok(false) => Status { + state: AndroidAccessibilityState::NotEnabled, + enabled: false, + message: "无障碍服务已开启,但当前未运行或已被系统标记为故障,请重新开启 OpenLess 无障碍服务".to_string(), + }, + Err(error) => Status { + state: AndroidAccessibilityState::NotEnabled, + enabled: false, + message: error, + }, + } + } + + pub fn request_android_accessibility_permission() -> AndroidAccessibilityPermissionResult { + match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::launch_accessibility_settings(env, context) + }) { + Ok(()) => AndroidAccessibilityPermissionResult { + launched: true, + message: "已打开无障碍设置".to_string(), + }, + Err(error) => AndroidAccessibilityPermissionResult { + launched: false, + message: error, + }, + } + } + + pub fn paste_via_accessibility() -> bool { + crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::accessibility_paste(env, context) + }) + .unwrap_or(false) + } +} diff --git a/openless-all/app/src-tauri/src/android/insert.rs b/openless-all/app/src-tauri/src/android/insert.rs new file mode 100644 index 00000000..13f4aedb --- /dev/null +++ b/openless-all/app/src-tauri/src/android/insert.rs @@ -0,0 +1,51 @@ +//! Android cross-app text insertion strategies. + +#![cfg(target_os = "android")] +use crate::android::types::AndroidInsertStrategy; +use crate::insertion::TextInserter; +use crate::types::InsertStatus; + +pub fn android_insert_with_strategy( + inserter: &TextInserter, + text: &str, + strategy: AndroidInsertStrategy, +) -> InsertStatus { + if text.is_empty() { + return InsertStatus::CopiedFallback; + } + + match strategy { + AndroidInsertStrategy::Clipboard => clipboard_fallback(inserter, text), + AndroidInsertStrategy::Accessibility + | AndroidInsertStrategy::Auto + | AndroidInsertStrategy::Ime => { + try_accessibility(inserter, text).unwrap_or_else(|| clipboard_fallback(inserter, text)) + } + } +} + +fn try_accessibility(inserter: &TextInserter, text: &str) -> Option { + if !crate::android::accessibility::get_android_accessibility_status().enabled { + log::info!("[android-insert] accessibility service not enabled"); + return None; + } + if !matches!(inserter.copy_fallback(text), InsertStatus::CopiedFallback) { + return None; + } + if crate::android::accessibility::paste_via_accessibility() { + Some(InsertStatus::Inserted) + } else { + log::warn!("[android-insert] accessibility paste failed; text remains on clipboard"); + Some(InsertStatus::CopiedFallback) + } +} + +fn clipboard_fallback(inserter: &TextInserter, text: &str) -> InsertStatus { + let status = inserter.copy_fallback(text); + if matches!(status, InsertStatus::CopiedFallback) { + let _ = crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::show_overlay_toast(env, context, "已复制到剪贴板") + }); + } + status +} diff --git a/openless-all/app/src-tauri/src/android/jni.rs b/openless-all/app/src-tauri/src/android/jni.rs new file mode 100644 index 00000000..dcaeae8a --- /dev/null +++ b/openless-all/app/src-tauri/src/android/jni.rs @@ -0,0 +1,510 @@ +//! Shared JNI helpers for Android Rust modules. + +#[cfg(target_os = "android")] +pub mod android { + use jni::objects::{JClass, JObject, JString, JValue}; + use jni::JNIEnv; + use jni::JavaVM; + + pub fn with_android_env( + f: impl for<'local> FnOnce(&mut JNIEnv<'local>, &JObject<'local>) -> Result, + ) -> Result { + let android_context = ndk_context::android_context(); + let vm = unsafe { + JavaVM::from_raw(android_context.vm().cast()) + .map_err(|error| format!("attach Android JVM: {error}"))? + }; + let mut env = vm + .attach_current_thread() + .map_err(|error| format!("attach Android thread: {error}"))?; + let context = unsafe { JObject::from_raw(android_context.context() as jni::sys::jobject) }; + f(&mut env, &context) + } + + pub fn call_static_void( + env: &mut JNIEnv, + class_name: &str, + method: &str, + sig: &str, + args: &[JValue], + ) -> Result<(), String> { + let class = env + .find_class(class_name) + .map_err(|error| format!("find class {class_name}: {error}"))?; + env.call_static_method(class, method, sig, args) + .map_err(|error| format!("call {class_name}.{method}: {error}"))?; + Ok(()) + } + + fn load_context_class<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + class_name: &str, + ) -> Result, String> { + let class_loader = env + .call_method(context, "getClassLoader", "()Ljava/lang/ClassLoader;", &[]) + .and_then(|value| value.l()) + .map_err(|error| format!("get Context class loader: {error}"))?; + let class_name_obj = jobject_str(env, class_name)?; + let class_obj = env + .call_method( + &class_loader, + "loadClass", + "(Ljava/lang/String;)Ljava/lang/Class;", + &[JValue::Object(&class_name_obj)], + ) + .and_then(|value| value.l()) + .map_err(|error| format!("load app class {class_name}: {error}"))?; + Ok(JClass::from(class_obj)) + } + + fn call_static_void_with_context_class<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + class_name: &str, + method: &str, + sig: &str, + args: &[JValue], + ) -> Result<(), String> { + let class = load_context_class(env, context, class_name)?; + env.call_static_method(class, method, sig, args) + .map_err(|error| format!("call {class_name}.{method}: {error}"))?; + Ok(()) + } + + fn call_static_bool_with_context_class<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + class_name: &str, + method: &str, + sig: &str, + args: &[JValue], + ) -> Result { + let class = load_context_class(env, context, class_name)?; + env.call_static_method(class, method, sig, args) + .and_then(|value| value.z()) + .map_err(|error| format!("call {class_name}.{method}: {error}")) + } + + pub fn jstring<'local>( + env: &mut JNIEnv<'local>, + value: &str, + ) -> Result, String> { + env.new_string(value) + .map_err(|error| format!("create jstring: {error}")) + } + + fn jobject_str<'local>( + env: &mut JNIEnv<'local>, + value: &str, + ) -> Result, String> { + Ok(jstring(env, value)?.into()) + } + + pub fn start_activity_class( + env: &mut JNIEnv, + context: &JObject, + class_name: &str, + ) -> Result<(), String> { + start_activity_class_with_flags(env, context, class_name, 0x10000000) + } + + pub fn start_activity_class_with_flags( + env: &mut JNIEnv, + context: &JObject, + class_name: &str, + flags: i32, + ) -> Result<(), String> { + let intent = env + .new_object("android/content/Intent", "()V", &[]) + .map_err(|error| format!("create activity intent: {error}"))?; + let class_name_obj = jobject_str(env, class_name)?; + let component = env + .new_object( + "android/content/ComponentName", + "(Landroid/content/Context;Ljava/lang/String;)V", + &[JValue::Object(context), JValue::Object(&class_name_obj)], + ) + .map_err(|error| format!("create component name: {error}"))?; + env.call_method( + &intent, + "setComponent", + "(Landroid/content/ComponentName;)Landroid/content/Intent;", + &[JValue::Object(&component)], + ) + .map_err(|error| format!("set activity component: {error}"))?; + env.call_method( + &intent, + "addFlags", + "(I)Landroid/content/Intent;", + &[JValue::Int(flags)], + ) + .map_err(|error| format!("set intent flags: {error}"))?; + env.call_method( + context, + "startActivity", + "(Landroid/content/Intent;)V", + &[JValue::Object(&intent)], + ) + .map_err(|error| format!("start activity: {error}"))?; + Ok(()) + } + + pub fn start_service_action( + env: &mut JNIEnv, + context: &JObject, + service_class: &str, + action: &str, + ) -> Result<(), String> { + let intent = env + .new_object("android/content/Intent", "()V", &[]) + .map_err(|error| format!("create service intent: {error}"))?; + let service_class_obj = jobject_str(env, service_class)?; + let component = env + .new_object( + "android/content/ComponentName", + "(Landroid/content/Context;Ljava/lang/String;)V", + &[JValue::Object(context), JValue::Object(&service_class_obj)], + ) + .map_err(|error| format!("create component name: {error}"))?; + env.call_method( + &intent, + "setComponent", + "(Landroid/content/ComponentName;)Landroid/content/Intent;", + &[JValue::Object(&component)], + ) + .map_err(|error| format!("set service component: {error}"))?; + let action_obj = jobject_str(env, action)?; + env.call_method( + &intent, + "setAction", + "(Ljava/lang/String;)Landroid/content/Intent;", + &[JValue::Object(&action_obj)], + ) + .map_err(|error| format!("set service action: {error}"))?; + let start_method = if action.ends_with(".HIDE") || action.ends_with(".SHOW") { + "startService" + } else if android_sdk_int(env)? >= 26 { + "startForegroundService" + } else { + "startService" + }; + env.call_method( + context, + start_method, + "(Landroid/content/Intent;)Landroid/content/ComponentName;", + &[JValue::Object(&intent)], + ) + .map_err(|error| format!("{start_method}: {error}"))?; + Ok(()) + } + + pub fn can_draw_overlays(env: &mut JNIEnv, context: &JObject) -> Result { + if android_sdk_int(env)? < 23 { + return Ok(true); + } + env.call_static_method( + "android/provider/Settings", + "canDrawOverlays", + "(Landroid/content/Context;)Z", + &[JValue::Object(context)], + ) + .and_then(|value| value.z()) + .map_err(|error| format!("Settings.canDrawOverlays: {error}")) + } + + pub fn check_self_permission( + env: &mut JNIEnv, + context: &JObject, + permission: &str, + ) -> Result { + if android_sdk_int(env)? < 23 { + return Ok(true); + } + let permission_obj = jobject_str(env, permission)?; + let result = env + .call_method( + context, + "checkSelfPermission", + "(Ljava/lang/String;)I", + &[JValue::Object(&permission_obj)], + ) + .and_then(|value| value.i()) + .map_err(|error| format!("Context.checkSelfPermission({permission}): {error}"))?; + Ok(result == 0) + } + + pub fn request_record_audio_permission<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + ) -> Result { + call_static_bool_with_context_class( + env, + context, + "com.openless.app.OpenLessPermissionBridge", + "requestRecordAudioPermission", + "(Landroid/content/Context;)Z", + &[JValue::Object(context)], + ) + } + + pub fn launch_app_details_settings(env: &mut JNIEnv, context: &JObject) -> Result<(), String> { + let action_obj = jobject_str(env, "android.settings.APPLICATION_DETAILS_SETTINGS")?; + let null_obj = JObject::null(); + let package_name = env + .call_method(context, "getPackageName", "()Ljava/lang/String;", &[]) + .and_then(|value| value.l()) + .map_err(|error| format!("Context.getPackageName: {error}"))?; + let package_prefix = jobject_str(env, "package")?; + let uri = env + .call_static_method( + "android/net/Uri", + "fromParts", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Landroid/net/Uri;", + &[ + JValue::Object(&package_prefix), + JValue::Object(&package_name), + JValue::Object(&null_obj), + ], + ) + .and_then(|value| value.l()) + .map_err(|error| format!("Uri.fromParts(package): {error}"))?; + start_settings_intent(env, context, &action_obj, Some(&uri)) + } + + pub fn launch_overlay_settings(env: &mut JNIEnv, context: &JObject) -> Result<(), String> { + if android_sdk_int(env)? < 23 { + return Ok(()); + } + let action_obj = jobject_str(env, "android.settings.action.MANAGE_OVERLAY_PERMISSION")?; + let null_obj = JObject::null(); + let package_name = env + .call_method(context, "getPackageName", "()Ljava/lang/String;", &[]) + .and_then(|value| value.l()) + .map_err(|error| format!("Context.getPackageName: {error}"))?; + let package_prefix = jobject_str(env, "package")?; + let uri = env + .call_static_method( + "android/net/Uri", + "fromParts", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Landroid/net/Uri;", + &[ + JValue::Object(&package_prefix), + JValue::Object(&package_name), + JValue::Object(&null_obj), + ], + ) + .and_then(|value| value.l()) + .map_err(|error| format!("Uri.fromParts(package): {error}"))?; + start_settings_intent(env, context, &action_obj, Some(&uri)) + } + + pub fn android_sdk_int(env: &mut JNIEnv) -> Result { + env.get_static_field("android/os/Build$VERSION", "SDK_INT", "I") + .and_then(|value| value.i()) + .map_err(|error| format!("read SDK_INT: {error}")) + } + + pub fn copy_to_clipboard( + env: &mut JNIEnv, + context: &JObject, + text: &str, + ) -> Result { + let clipboard_name = jobject_str(env, "clipboard")?; + let clipboard = env + .call_method( + context, + "getSystemService", + "(Ljava/lang/String;)Ljava/lang/Object;", + &[JValue::Object(&clipboard_name)], + ) + .and_then(|value| value.l()) + .map_err(|error| format!("get clipboard service: {error}"))?; + let label = jobject_str(env, "OpenLess")?; + let text_obj = jobject_str(env, text)?; + let clip = env + .call_static_method( + "android/content/ClipData", + "newPlainText", + "(Ljava/lang/CharSequence;Ljava/lang/CharSequence;)Landroid/content/ClipData;", + &[JValue::Object(&label), JValue::Object(&text_obj)], + ) + .and_then(|value| value.l()) + .map_err(|error| format!("new ClipData: {error}"))?; + env.call_method( + &clipboard, + "setPrimaryClip", + "(Landroid/content/ClipData;)V", + &[JValue::Object(&clip)], + ) + .map_err(|error| format!("setPrimaryClip: {error}"))?; + Ok(true) + } + + pub fn notify_overlay_bridge<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + state: &str, + message: Option<&str>, + ) -> Result<(), String> { + let state_obj = jobject_str(env, state)?; + let message_obj = jobject_str(env, message.unwrap_or(""))?; + call_static_void_with_context_class( + env, + context, + "com.openless.app.OpenLessOverlayBridge", + "onCapsuleStateChanged", + "(Ljava/lang/String;Ljava/lang/String;)V", + &[JValue::Object(&state_obj), JValue::Object(&message_obj)], + ) + } + + pub fn show_overlay_toast<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + message: &str, + ) -> Result<(), String> { + let message_obj = jobject_str(env, message)?; + call_static_void_with_context_class( + env, + context, + "com.openless.app.OpenLessOverlayBridge", + "showToast", + "(Ljava/lang/String;)V", + &[JValue::Object(&message_obj)], + ) + } + + pub fn accessibility_paste<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + ) -> Result { + call_static_bool_with_context_class( + env, + context, + "com.openless.app.OpenLessAccessibilityService", + "pasteToFocusedField", + "()Z", + &[], + ) + } + + pub fn accessibility_selected_text<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + ) -> Result, String> { + let class = load_context_class( + env, + context, + "com.openless.app.OpenLessAccessibilityService", + )?; + let value = env + .call_static_method(class, "captureSelectedText", "()Ljava/lang/String;", &[]) + .and_then(|value| value.l()) + .map_err(|error| { + format!("call com.openless.app.OpenLessAccessibilityService.captureSelectedText: {error}") + })?; + if value.is_null() { + return Ok(None); + } + let text = env + .get_string(&JString::from(value)) + .map_err(|error| format!("read selected text jstring: {error}"))? + .to_string_lossy() + .into_owned(); + if text.trim().is_empty() { + Ok(None) + } else { + Ok(Some(text)) + } + } + + pub fn accessibility_enabled<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + ) -> Result { + call_static_bool_with_context_class( + env, + context, + "com.openless.app.OpenLessAccessibilityService", + "isEnabled", + "(Landroid/content/Context;)Z", + &[JValue::Object(context)], + ) + } + + pub fn accessibility_operational<'local>( + env: &mut JNIEnv<'local>, + context: &JObject<'local>, + ) -> Result { + call_static_bool_with_context_class( + env, + context, + "com.openless.app.OpenLessAccessibilityService", + "isOperational", + "(Landroid/content/Context;)Z", + &[JValue::Object(context)], + ) + } + + pub fn launch_accessibility_settings( + env: &mut JNIEnv, + context: &JObject, + ) -> Result<(), String> { + let action_obj = jobject_str(env, "android.settings.ACCESSIBILITY_SETTINGS")?; + start_settings_intent(env, context, &action_obj, None) + } + + fn start_settings_intent( + env: &mut JNIEnv, + context: &JObject, + action_obj: &JObject, + data_uri: Option<&JObject>, + ) -> Result<(), String> { + let intent = env + .new_object( + "android/content/Intent", + "(Ljava/lang/String;)V", + &[JValue::Object(&action_obj)], + ) + .map_err(|error| format!("create settings intent: {error}"))?; + if let Some(uri) = data_uri { + env.call_method( + &intent, + "setData", + "(Landroid/net/Uri;)Landroid/content/Intent;", + &[JValue::Object(uri)], + ) + .map_err(|error| format!("set settings intent data: {error}"))?; + } + env.call_method( + &intent, + "addFlags", + "(I)Landroid/content/Intent;", + &[JValue::Int(0x10000000)], + ) + .map_err(|error| format!("set intent flags: {error}"))?; + env.call_method( + context, + "startActivity", + "(Landroid/content/Intent;)V", + &[JValue::Object(&intent)], + ) + .map_err(|error| format!("start settings activity: {error}"))?; + Ok(()) + } + + pub fn export_jstring(env: &mut JNIEnv, value: &str) -> jni::sys::jstring { + env.new_string(value) + .map(|s| s.into_raw()) + .unwrap_or(std::ptr::null_mut()) + } + + pub fn export_jboolean(value: bool) -> jni::sys::jboolean { + if value { + 1 + } else { + 0 + } + } +} diff --git a/openless-all/app/src-tauri/src/android/mod.rs b/openless-all/app/src-tauri/src/android/mod.rs new file mode 100644 index 00000000..0f58039e --- /dev/null +++ b/openless-all/app/src-tauri/src/android/mod.rs @@ -0,0 +1,25 @@ +//! Android platform integration (JNI, overlay, accessibility, insert). + +pub mod accessibility; +#[cfg(target_os = "android")] +pub mod insert; +pub mod jni; +pub mod native_bridge; +pub mod overlay; +pub use crate::types::android_types as types; + +pub use accessibility::{ + get_android_accessibility_status, paste_via_accessibility, + request_android_accessibility_permission, AndroidAccessibilityPermissionResult, +}; +#[cfg(target_os = "android")] +pub use insert::android_insert_with_strategy; +pub use native_bridge::{ + hide_overlay, is_overlay_visible, notify_capsule_state, refresh_overlay_if_visible, + refresh_overlay_layout, register_android_coordinator, replace_overlay, show_overlay, +}; +pub use overlay::{ + get_android_overlay_status, hide_android_overlay, refresh_android_overlay_if_visible, + refresh_android_overlay_layout, replace_android_overlay, request_android_overlay_permission, + show_android_overlay, AndroidOverlayPermissionResult, +}; diff --git a/openless-all/app/src-tauri/src/android/native_bridge.rs b/openless-all/app/src-tauri/src/android/native_bridge.rs new file mode 100644 index 00000000..9673e2ce --- /dev/null +++ b/openless-all/app/src-tauri/src/android/native_bridge.rs @@ -0,0 +1,396 @@ +//! JNI bridge between Kotlin overlay code and Rust Coordinator. + +use std::sync::{Arc, OnceLock}; + +use crate::coordinator::Coordinator; +use crate::types::{CapsulePayload, CapsuleState}; + +static COORDINATOR: OnceLock> = OnceLock::new(); +static OVERLAY_VISIBLE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +pub fn register_android_coordinator(coordinator: Arc) { + let _ = COORDINATOR.set(coordinator); +} + +pub fn notify_capsule_state(payload: &CapsulePayload) { + #[cfg(target_os = "android")] + { + let state = capsule_state_name(payload.state); + let message = payload.message.as_deref(); + if let Err(error) = crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::notify_overlay_bridge(env, context, state, message) + }) { + log::warn!("[android-native] notify overlay bridge failed: {error}"); + } + } + let _ = payload; +} + +pub fn show_overlay() -> Result<(), String> { + #[cfg(target_os = "android")] + { + crate::android::jni::android::with_android_env(|env, context| { + show_overlay_with_context(env, context) + })?; + } + Ok(()) +} + +pub fn hide_overlay() -> Result<(), String> { + #[cfg(target_os = "android")] + { + crate::android::jni::android::with_android_env(|env, context| { + hide_overlay_with_context(env, context) + })?; + } + Ok(()) +} + +pub fn replace_overlay() -> Result<(), String> { + #[cfg(target_os = "android")] + { + crate::android::jni::android::with_android_env(|env, context| { + replace_overlay_with_context(env, context) + })?; + } + Ok(()) +} + +pub fn refresh_overlay_layout() -> Result<(), String> { + #[cfg(target_os = "android")] + { + crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::start_service_action( + env, + context, + "com.openless.app.OpenLessOverlayService", + "com.openless.app.overlay.REFRESH_LAYOUT", + ) + })?; + } + Ok(()) +} + +pub fn refresh_overlay_if_visible() -> Result<(), String> { + if is_overlay_visible() { + refresh_overlay_layout() + } else { + Ok(()) + } +} + +#[cfg(target_os = "android")] +fn show_overlay_with_context( + env: &mut jni::JNIEnv, + context: &jni::objects::JObject, +) -> Result<(), String> { + crate::android::jni::android::start_service_action( + env, + context, + "com.openless.app.OpenLessOverlayService", + "com.openless.app.overlay.SHOW", + )?; + OVERLAY_VISIBLE.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) +} + +#[cfg(target_os = "android")] +fn hide_overlay_with_context( + env: &mut jni::JNIEnv, + context: &jni::objects::JObject, +) -> Result<(), String> { + crate::android::jni::android::start_service_action( + env, + context, + "com.openless.app.OpenLessOverlayService", + "com.openless.app.overlay.HIDE", + )?; + OVERLAY_VISIBLE.store(false, std::sync::atomic::Ordering::SeqCst); + Ok(()) +} + +#[cfg(target_os = "android")] +fn replace_overlay_with_context( + env: &mut jni::JNIEnv, + context: &jni::objects::JObject, +) -> Result<(), String> { + crate::android::jni::android::start_service_action( + env, + context, + "com.openless.app.OpenLessOverlayService", + "com.openless.app.overlay.REPLACE_OVERLAY", + )?; + OVERLAY_VISIBLE.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) +} + +pub fn is_overlay_visible() -> bool { + OVERLAY_VISIBLE.load(std::sync::atomic::Ordering::SeqCst) +} + +pub fn overlay_trigger_mode_name() -> &'static str { + let Some(coordinator) = COORDINATOR.get() else { + return "background"; + }; + match coordinator.android_overlay_trigger() { + crate::types::AndroidOverlayTrigger::Background => "background", + crate::types::AndroidOverlayTrigger::Keyboard => "keyboard", + crate::types::AndroidOverlayTrigger::Always => "always", + } +} + +fn spawn_start_dictation(translation: bool) { + let Some(coordinator) = COORDINATOR.get().cloned() else { + log::warn!("[android-native] coordinator unavailable"); + return; + }; + tauri::async_runtime::spawn(async move { + let result = if translation { + coordinator.start_dictation_with_translation().await + } else { + coordinator.start_dictation().await + }; + if let Err(error) = result { + log::warn!( + "[android-native] {} failed: {error}", + if translation { + "start_dictation_with_translation" + } else { + "start_dictation" + } + ); + } + }); +} + +fn spawn_stop_dictation() { + let Some(coordinator) = COORDINATOR.get().cloned() else { + log::warn!("[android-native] coordinator unavailable"); + return; + }; + tauri::async_runtime::spawn(async move { + if let Err(error) = coordinator.stop_dictation().await { + log::warn!("[android-native] stop_dictation failed: {error}"); + } + }); +} + +fn spawn_stop_dictation_with_translation(translation: bool) { + let Some(coordinator) = COORDINATOR.get().cloned() else { + log::warn!("[android-native] coordinator unavailable"); + return; + }; + tauri::async_runtime::spawn(async move { + if let Err(error) = coordinator + .stop_dictation_with_translation(translation) + .await + { + log::warn!("[android-native] stop_dictation_with_translation failed: {error}"); + } + }); +} + +fn spawn_cancel_dictation() { + let Some(coordinator) = COORDINATOR.get().cloned() else { + return; + }; + coordinator.cancel_dictation(); +} + +fn spawn_switch_style_pack() { + let Some(coordinator) = COORDINATOR.get().cloned() else { + log::warn!("[android-native] coordinator unavailable"); + return; + }; + coordinator.switch_to_previous_style_pack(); +} + +fn spawn_open_qa_from_overlay() { + let Some(coordinator) = COORDINATOR.get().cloned() else { + log::warn!("[android-native] coordinator unavailable"); + return; + }; + log::info!("[android-native] open_qa_from_overlay requested"); + tauri::async_runtime::spawn(async move { + if let Err(error) = coordinator.open_qa_from_overlay().await { + log::warn!("[android-native] open_qa_from_overlay failed: {error}"); + } + }); +} + +fn spawn_finalize_qa_from_overlay() { + let Some(coordinator) = COORDINATOR.get().cloned() else { + log::warn!("[android-native] coordinator unavailable"); + return; + }; + log::info!("[android-native] finalize_qa_from_overlay requested"); + tauri::async_runtime::spawn(async move { + if let Err(error) = coordinator.finalize_qa_from_overlay().await { + log::warn!("[android-native] finalize_qa_from_overlay failed: {error}"); + } + }); +} + +fn capsule_state_name(state: CapsuleState) -> &'static str { + match state { + CapsuleState::Idle => "idle", + CapsuleState::Recording => "recording", + CapsuleState::Transcribing => "transcribing", + CapsuleState::Polishing => "polishing", + CapsuleState::Done => "done", + CapsuleState::Cancelled => "cancelled", + CapsuleState::Error => "error", + } +} + +#[cfg(target_os = "android")] +mod jni_exports { + use super::*; + use jni::objects::{JClass, JObject}; + use jni::sys::{jboolean, jstring, JNIEnv}; + use jni::JNIEnv as JniEnv; + + unsafe fn with_jni_context( + env_ptr: *mut JNIEnv, + context: JObject, + f: impl for<'local> FnOnce(&mut JniEnv<'local>, &JObject<'local>) -> Result, + ) -> Result { + let mut env = + JniEnv::from_raw(env_ptr).map_err(|error| format!("attach JNI env: {error}"))?; + f(&mut env, &context) + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeStartDictation( + _env: *mut JNIEnv, + _class: JClass, + ) { + spawn_start_dictation(false); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeStartDictationWithTranslation( + _env: *mut JNIEnv, + _class: JClass, + translation: jboolean, + ) { + spawn_start_dictation(translation != 0); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeStopDictation( + _env: *mut JNIEnv, + _class: JClass, + ) { + spawn_stop_dictation(); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeStopDictationWithTranslation( + _env: *mut JNIEnv, + _class: JClass, + translation: jboolean, + ) { + spawn_stop_dictation_with_translation(translation != 0); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeCancelDictation( + _env: *mut JNIEnv, + _class: JClass, + ) { + spawn_cancel_dictation(); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeSwitchStylePack( + _env: *mut JNIEnv, + _class: JClass, + ) { + spawn_switch_style_pack(); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeOpenQaFromOverlay( + _env: *mut JNIEnv, + _class: JClass, + ) { + spawn_open_qa_from_overlay(); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeFinalizeQaFromOverlay( + _env: *mut JNIEnv, + _class: JClass, + ) { + spawn_finalize_qa_from_overlay(); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeShowOverlay( + env: *mut JNIEnv, + _class: JClass, + context: JObject, + ) { + let _ = with_jni_context(env, context, |env, context| { + show_overlay_with_context(env, context) + }); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeHideOverlay( + env: *mut JNIEnv, + _class: JClass, + context: JObject, + ) { + let _ = with_jni_context(env, context, |env, context| { + hide_overlay_with_context(env, context) + }); + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeCanDrawOverlays( + env: *mut JNIEnv, + _class: JClass, + context: JObject, + ) -> jboolean { + let visible = with_jni_context(env, context, |env, context| { + crate::android::jni::android::can_draw_overlays(env, context) + }) + .unwrap_or(false); + crate::android::jni::android::export_jboolean(visible) + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeIsOverlayVisible( + _env: *mut JNIEnv, + _class: JClass, + ) -> jboolean { + crate::android::jni::android::export_jboolean(is_overlay_visible()) + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeGetOverlayTriggerMode( + env: *mut JNIEnv, + _class: JClass, + ) -> jstring { + let mode = overlay_trigger_mode_name(); + match JniEnv::from_raw(env) { + Ok(mut env) => crate::android::jni::android::export_jstring(&mut env, mode), + Err(_) => std::ptr::null_mut(), + } + } + + #[no_mangle] + pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeNotifyOverlayPermissionChanged( + env: *mut JNIEnv, + _class: JClass, + context: JObject, + ) { + if overlay_trigger_mode_name() == "always" { + let _ = with_jni_context(env, context, |env, context| { + show_overlay_with_context(env, context) + }); + } + } +} diff --git a/openless-all/app/src-tauri/src/android/overlay.rs b/openless-all/app/src-tauri/src/android/overlay.rs new file mode 100644 index 00000000..4b45b455 --- /dev/null +++ b/openless-all/app/src-tauri/src/android/overlay.rs @@ -0,0 +1,145 @@ +//! Android overlay window permission and foreground service integration. + +use serde::Serialize; + +use crate::android::types::AndroidOverlayStatus; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AndroidOverlayPermissionResult { + pub launched: bool, + pub message: String, +} + +pub fn get_android_overlay_status() -> AndroidOverlayStatus { + #[cfg(target_os = "android")] + { + android_impl::get_android_overlay_status() + } + + #[cfg(not(target_os = "android"))] + { + use crate::android::types::AndroidOverlayPermissionState; + + AndroidOverlayStatus { + permission: AndroidOverlayPermissionState::NotAndroid, + overlay_visible: false, + message: "Android overlay is only available on Android".to_string(), + } + } +} + +pub fn request_android_overlay_permission() -> AndroidOverlayPermissionResult { + #[cfg(target_os = "android")] + { + android_impl::request_android_overlay_permission() + } + + #[cfg(not(target_os = "android"))] + { + AndroidOverlayPermissionResult { + launched: false, + message: "Android overlay permission is only available on Android".to_string(), + } + } +} + +pub fn show_android_overlay() -> Result<(), String> { + #[cfg(target_os = "android")] + { + return crate::android::native_bridge::show_overlay(); + } + #[cfg(not(target_os = "android"))] + { + Err("Android overlay is only available on Android".to_string()) + } +} + +pub fn hide_android_overlay() -> Result<(), String> { + #[cfg(target_os = "android")] + { + return crate::android::native_bridge::hide_overlay(); + } + #[cfg(not(target_os = "android"))] + { + Err("Android overlay is only available on Android".to_string()) + } +} + +pub fn refresh_android_overlay_if_visible() -> Result<(), String> { + #[cfg(target_os = "android")] + { + return crate::android::native_bridge::refresh_overlay_if_visible(); + } + #[cfg(not(target_os = "android"))] + { + Err("Android overlay is only available on Android".to_string()) + } +} + +pub fn refresh_android_overlay_layout() -> Result<(), String> { + #[cfg(target_os = "android")] + { + return crate::android::native_bridge::refresh_overlay_layout(); + } + #[cfg(not(target_os = "android"))] + { + Err("Android overlay is only available on Android".to_string()) + } +} + +pub fn replace_android_overlay() -> Result<(), String> { + #[cfg(target_os = "android")] + { + return crate::android::native_bridge::replace_overlay(); + } + #[cfg(not(target_os = "android"))] + { + Err("Android overlay is only available on Android".to_string()) + } +} + +#[cfg(target_os = "android")] +mod android_impl { + use super::{AndroidOverlayPermissionResult, AndroidOverlayStatus}; + use crate::android::types::{AndroidOverlayPermissionState, AndroidOverlayStatus as Status}; + + pub fn get_android_overlay_status() -> AndroidOverlayStatus { + let granted = crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::can_draw_overlays(env, context) + }) + .unwrap_or(false); + Status { + permission: if granted { + AndroidOverlayPermissionState::Granted + } else { + AndroidOverlayPermissionState::NotGranted + }, + overlay_visible: crate::android::native_bridge::is_overlay_visible(), + message: if granted { + "悬浮窗权限已授予".to_string() + } else { + "请在系统设置中授予悬浮窗权限".to_string() + }, + } + } + + pub fn request_android_overlay_permission() -> AndroidOverlayPermissionResult { + match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::start_activity_class( + env, + context, + "com.openless.app.OverlayPermissionActivity", + ) + }) { + Ok(()) => AndroidOverlayPermissionResult { + launched: true, + message: "已打开悬浮窗权限设置".to_string(), + }, + Err(error) => AndroidOverlayPermissionResult { + launched: false, + message: error, + }, + } + } +} diff --git a/openless-all/app/src-tauri/src/android/types.rs b/openless-all/app/src-tauri/src/android/types.rs new file mode 100644 index 00000000..c32d6e02 --- /dev/null +++ b/openless-all/app/src-tauri/src/android/types.rs @@ -0,0 +1,323 @@ +//! Android-specific preference types and status payloads. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum AndroidInsertStrategy { + Auto, + Ime, + Accessibility, + Clipboard, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum AndroidOverlayTrigger { + Background, + Keyboard, + Always, +} + +impl AndroidOverlayTrigger { + pub fn normalized(self) -> Self { + match self { + AndroidOverlayTrigger::Keyboard => AndroidOverlayTrigger::Background, + trigger => trigger, + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AndroidOverlayActivationMode { + Tap, + LongPress, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AndroidOverlayLeftSwipeAction { + Translation, + StylePack, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AndroidOverlayCancelSwipeDirection { + Up, + Down, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum AndroidAccessibilityState { + Enabled, + NotEnabled, + NotAndroid, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AndroidAccessibilityStatus { + pub state: AndroidAccessibilityState, + pub enabled: bool, + pub message: String, +} + +pub fn default_android_insert_strategy() -> AndroidInsertStrategy { + AndroidInsertStrategy::Accessibility +} + +pub fn default_android_overlay_trigger() -> AndroidOverlayTrigger { + AndroidOverlayTrigger::Background +} + +pub fn default_android_overlay_activation_mode() -> AndroidOverlayActivationMode { + AndroidOverlayActivationMode::Tap +} + +pub fn default_android_overlay_left_swipe_action() -> AndroidOverlayLeftSwipeAction { + AndroidOverlayLeftSwipeAction::Translation +} + +pub fn default_android_overlay_cancel_swipe_direction() -> AndroidOverlayCancelSwipeDirection { + AndroidOverlayCancelSwipeDirection::Up +} + +pub fn default_android_overlay_size_dp() -> u32 { + 72 +} + +pub fn normalize_android_insert_strategy(strategy: AndroidInsertStrategy) -> AndroidInsertStrategy { + match strategy { + AndroidInsertStrategy::Auto | AndroidInsertStrategy::Ime => { + AndroidInsertStrategy::Accessibility + } + strategy => strategy, + } +} + +pub fn normalize_android_overlay_size_dp(size_dp: u32) -> u32 { + size_dp.clamp(48, 120) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AndroidOverlaySettingsAction { + None, + RefreshLayout, + Transition { + from: AndroidOverlayTrigger, + to: AndroidOverlayTrigger, + }, +} + +pub fn classify_android_overlay_settings_change( + previous: &super::UserPreferences, + next: &super::UserPreferences, +) -> AndroidOverlaySettingsAction { + let trigger_changed = + previous.android_overlay_trigger.normalized() != next.android_overlay_trigger.normalized(); + let size_changed = normalize_android_overlay_size_dp(previous.android_overlay_size_dp) + != normalize_android_overlay_size_dp(next.android_overlay_size_dp); + + if trigger_changed { + return AndroidOverlaySettingsAction::Transition { + from: previous.android_overlay_trigger.normalized(), + to: next.android_overlay_trigger.normalized(), + }; + } + + if size_changed { + return AndroidOverlaySettingsAction::RefreshLayout; + } + + AndroidOverlaySettingsAction::None +} + +#[cfg(test)] +mod android_overlay_tests { + use super::*; + use crate::types::UserPreferences; + + fn overlay_prefs( + trigger: AndroidOverlayTrigger, + size_dp: u32, + activation: AndroidOverlayActivationMode, + ) -> UserPreferences { + let mut prefs = UserPreferences::default(); + prefs.android_overlay_trigger = trigger; + prefs.android_overlay_size_dp = size_dp; + prefs.android_overlay_activation_mode = activation; + prefs + } + + #[test] + fn size_only_change_returns_refresh_layout() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 96, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::RefreshLayout, + ); + } + + #[test] + fn trigger_only_change_returns_transition() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Background, + 72, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::Transition { + from: AndroidOverlayTrigger::Background, + to: AndroidOverlayTrigger::Always, + }, + ); + } + + #[test] + fn trigger_and_size_change_returns_transition_only() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Background, + 72, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 96, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::Transition { + from: AndroidOverlayTrigger::Background, + to: AndroidOverlayTrigger::Always, + }, + ); + } + + #[test] + fn activation_only_change_returns_none() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::LongPress, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::None, + ); + } + + #[test] + fn out_of_bounds_size_200_to_120_returns_none_after_normalize() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Always, + 200, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 120, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::None, + ); + } + + #[test] + fn out_of_bounds_size_below_min_normalizes_to_same_returns_none() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Always, + 30, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 48, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::None, + ); + } + + #[test] + fn identical_normalized_size_returns_none() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::None, + ); + } + + #[test] + fn keyboard_trigger_normalizes_to_background_for_transition() { + let previous = overlay_prefs( + AndroidOverlayTrigger::Keyboard, + 72, + AndroidOverlayActivationMode::Tap, + ); + let next = overlay_prefs( + AndroidOverlayTrigger::Always, + 72, + AndroidOverlayActivationMode::Tap, + ); + assert_eq!( + classify_android_overlay_settings_change(&previous, &next), + AndroidOverlaySettingsAction::Transition { + from: AndroidOverlayTrigger::Background, + to: AndroidOverlayTrigger::Always, + }, + ); + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum AndroidOverlayPermissionState { + Granted, + NotGranted, + NotAndroid, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AndroidOverlayStatus { + pub permission: AndroidOverlayPermissionState, + pub overlay_visible: bool, + pub message: String, +} diff --git a/openless-all/app/src-tauri/src/asr/local/download.rs b/openless-all/app/src-tauri/src/asr/local/download.rs index 81f4f04d..fa4d4e90 100644 --- a/openless-all/app/src-tauri/src/asr/local/download.rs +++ b/openless-all/app/src-tauri/src/asr/local/download.rs @@ -226,18 +226,20 @@ impl DownloadManager { pub(crate) fn build_client() -> Result { // native-tls (macOS=SecureTransport) 不像 rustls 那样把 CDN unclean close - // 当致命错误。 + // 当致命错误。Android/iOS 无 native-tls feature,走默认 rustls。 // // User-Agent 用 aria2 的——hfd(hf-mirror 官方推荐)就是 aria2 包装, // 实测 aria2 UA 在 HF 反滥用规则里走白名单不挨 throttle;自定义 UA // (`openless/x`) 在 sustained 传输后会被 mirror 主动切流。 - reqwest::Client::builder() - .use_native_tls() + let mut builder = reqwest::Client::builder() .user_agent("aria2/1.36.0") .connect_timeout(std::time::Duration::from_secs(30)) - .pool_idle_timeout(std::time::Duration::from_secs(60)) - .build() - .context("build reqwest client failed") + .pool_idle_timeout(std::time::Duration::from_secs(60)); + #[cfg(not(mobile))] + { + builder = builder.use_native_tls(); + } + builder.build().context("build reqwest client failed") } async fn run_download( diff --git a/openless-all/app/src-tauri/src/commands.rs b/openless-all/app/src-tauri/src/commands.rs index 7a2e12b9..d3839b93 100644 --- a/openless-all/app/src-tauri/src/commands.rs +++ b/openless-all/app/src-tauri/src/commands.rs @@ -8,18 +8,22 @@ use serde::Serialize; use serde_json::Value; use tauri::{AppHandle, Emitter, Manager, State, Window}; +#[cfg(not(mobile))] use crate::asr::local::foundry::{ model_alias_is_known, FoundryCatalogModel, FoundryPrepareProgressPayload, FoundryRuntimeStatus, DEFAULT_MODEL_ALIAS, PROVIDER_ID as FOUNDRY_LOCAL_PROVIDER_ID, }; +#[cfg(not(mobile))] use crate::asr::local::sherpa::{ model_alias_is_known as sherpa_model_alias_is_known, SherpaCatalogModel, SherpaPrepareProgressPayload, SherpaRuntimeStatus, DEFAULT_MODEL_ALIAS as SHERPA_DEFAULT_MODEL_ALIAS, }; +#[cfg(not(mobile))] use crate::asr::local::sherpa_download::{ fetch_remote_info as fetch_sherpa_remote_info, SherpaDownloadManager, SherpaRemoteInfo, }; +#[cfg(not(mobile))] use crate::asr::local::{FoundryLocalRuntime, SherpaOnnxRuntime}; use crate::coordinator::Coordinator; use crate::net; @@ -34,24 +38,31 @@ use crate::polish::{ CODEX_OAUTH_PROVIDER_ID, }; use crate::recorder::{AudioConsumer, Recorder}; +#[cfg(not(mobile))] +use crate::types::WindowsImeStatus; use crate::types::{ - builtin_style_pack_id, default_active_style_pack_id, ChineseScriptPreference, ComboBinding, - CorrectionRule, CredentialsStatus, DictationSession, DictionaryEntry, HotkeyCapability, - HotkeyStatus, OutputLanguagePreference, PolishMode, ShortcutBinding, StylePack, StylePackKind, - StylePackRuntimeDiagnostics, StyleSystemPrompts, UpdateChannel, UserPreferences, - VocabPresetStore, WindowsImeStatus, + builtin_style_pack_id, default_active_style_pack_id, AndroidOverlayStatus, + ChineseScriptPreference, ComboBinding, CorrectionRule, CredentialsStatus, DictationSession, + DictionaryEntry, HotkeyAdapterKind, HotkeyCapability, HotkeyInstallError, HotkeyStatus, + HotkeyStatusState, OutputLanguagePreference, PlatformCapabilities, PolishMode, ShortcutBinding, + StylePack, StylePackKind, StylePackRuntimeDiagnostics, StyleSystemPrompts, UpdateChannel, + UserPreferences, VocabPresetStore, }; type CoordinatorState<'a> = State<'a, Arc>; pub type MicrophoneMonitorState = Mutex>; + +#[cfg(not(mobile))] pub type TrayMicrophoneMenuState = Mutex>; +#[cfg(not(mobile))] pub struct TrayMicrophoneMenuItem { pub id: String, pub device_name: String, pub item: tauri::menu::CheckMenuItem, } +#[cfg(not(mobile))] pub fn sync_tray_microphone_selection(items: &[TrayMicrophoneMenuItem], device_name: &str) { for item in items { let _ = item.item.set_checked(item.device_name == device_name); @@ -180,6 +191,7 @@ fn persist_settings( let mut previous = coord.read_settings(); sync_dictation_hotkey_legacy_fields(&mut previous); sync_dictation_hotkey_legacy_fields(&mut prefs); + prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); reject_hotkey_collisions(&prefs)?; let dictation_shortcut_changed = previous.dictation_hotkey != prefs.dictation_hotkey; let dictation_mode_changed = previous.hotkey.mode != prefs.hotkey.mode; @@ -235,19 +247,35 @@ fn persist_settings( Ok(()) } -#[tauri::command] -pub fn set_settings( - coord: CoordinatorState<'_>, - app: AppHandle, - tray_microphones: State<'_, TrayMicrophoneMenuState>, +fn set_settings_common( + coord: &Coordinator, + app: &AppHandle, mut prefs: UserPreferences, -) -> Result<(), String> { +) -> Result { let packs = coord.style_packs().list().map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); + let previous = coord.prefs().get(); // 广播给所有 webview。issue #205:QaPanel 跑在独立 webview, // 没有 HotkeySettingsContext,必须靠事件感知录音键变化,否则面板可见时 // 用户改键会让浮窗里的 "{recordHotkey}" 文案一直停留在旧值。 - persist_settings(&*coord, prefs.clone())?; + persist_settings(coord, prefs.clone())?; + let _ = app.emit("prefs:changed", &prefs); + #[cfg(target_os = "android")] + { + coord.apply_android_overlay_settings_change(&previous, &prefs); + } + Ok(prefs) +} + +#[cfg(not(mobile))] +#[tauri::command] +pub fn set_settings( + coord: CoordinatorState<'_>, + app: AppHandle, + tray_microphones: State<'_, TrayMicrophoneMenuState>, + prefs: UserPreferences, +) -> Result<(), String> { + let prefs = set_settings_common(&*coord, &app, prefs)?; // refresh_tray_microphone_menu 内部会调用 NSStatusItem.set_menu,必须在主线程上跑。 // set_settings 本身是同步 Tauri command,在 IPC handler 线程上执行;从这里直接调 // 会触发 macOS 主线程断言或在 dispatch 队列上死锁,导致整个 UI 无响应(用户改 @@ -264,20 +292,35 @@ pub fn set_settings( ); } }); - // 抑制 unused 警告:tray_microphones 现在改在闭包里通过 app.state 取, - // 但函数签名保留 State 入参,以便 Tauri 在调用前注入。 let _ = tray_microphones; - let _ = app.emit("prefs:changed", &prefs); + Ok(()) +} + +#[cfg(mobile)] +#[tauri::command] +pub fn set_settings( + coord: CoordinatorState<'_>, + app: AppHandle, + prefs: UserPreferences, +) -> Result<(), String> { + set_settings_common(&*coord, &app, prefs)?; Ok(()) } fn refresh_tray_menu_async(app: &AppHandle) { - let app_for_main = app.clone(); - let _ = app.run_on_main_thread(move || { - if let Err(err) = crate::refresh_tray_microphone_menu(&app_for_main) { - log::warn!("[tray] refresh after style change failed: {err}"); - } - }); + #[cfg(not(mobile))] + { + let app_for_main = app.clone(); + let _ = app.run_on_main_thread(move || { + if let Err(err) = crate::refresh_tray_microphone_menu(&app_for_main) { + log::warn!("[tray] refresh after style change failed: {err}"); + } + }); + } + #[cfg(mobile)] + { + let _ = app; + } } fn emit_prefs_changed(app: &AppHandle, prefs: &UserPreferences) { @@ -475,6 +518,7 @@ fn extract_between(haystack: &str, open: &str, close: &str) -> Option { // 里是分开的两份文件 —— 即使代码逻辑写错把 Beta URL 传给 Stable 用户,HTTP 也是 // 直接 404,绝不会拿到错档。 +#[cfg(not(mobile))] #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct AppUpdateMetadata { @@ -492,6 +536,7 @@ pub struct AppUpdateMetadata { /// 渠道:显式传入 `channel` 时用它(关于页固定查 Stable、高级页 Beta 区查 Beta); /// 不传则回落到 `prefs.update_channel`(后台 AutoUpdateGate 自动检查走这条)。 /// 返回 None = 当前是最新;Some(metadata) = 有新版可装。 +#[cfg(not(mobile))] #[tauri::command] pub async fn app_check_update_with_channel( coord: CoordinatorState<'_>, @@ -537,6 +582,7 @@ pub async fn app_check_update_with_channel( /// 把 fetch_latest_beta_release 找到的最新 prerelease tag 拼成 -beta manifest URL 对。 /// 顺序:先镜像(fastgit.cc 代理 GitHub),后直连 —— 跟 tauri.conf 现有 Stable /// endpoints 一致,让国内访问优先打到 CDN。 +#[cfg(not(mobile))] async fn resolve_beta_manifest_endpoints() -> Result, String> { let Some(latest) = fetch_latest_beta_release().await? else { return Err("尚未发布过 Beta 版本".to_string()); @@ -591,30 +637,115 @@ pub async fn check_network() -> NetworkCheckResult { #[tauri::command] pub fn get_hotkey_status(coord: CoordinatorState<'_>) -> HotkeyStatus { + #[cfg(mobile)] + { + let _ = coord; + return HotkeyStatus { + adapter: HotkeyAdapterKind::Unavailable, + state: HotkeyStatusState::Failed, + message: Some("移动端不支持全局热键".into()), + last_error: Some(HotkeyInstallError { + code: "unavailable".into(), + message: "Global hotkeys are not available on mobile".into(), + }), + }; + } + #[cfg(not(mobile))] coord.hotkey_status() } #[tauri::command] pub fn get_hotkey_capability(coord: CoordinatorState<'_>) -> HotkeyCapability { + #[cfg(mobile)] + { + let _ = coord; + return HotkeyCapability::current(); + } + #[cfg(not(mobile))] coord.hotkey_capability() } +#[tauri::command] +pub fn get_platform_capabilities() -> PlatformCapabilities { + PlatformCapabilities::current() +} + +#[tauri::command] +pub fn get_android_overlay_status() -> AndroidOverlayStatus { + crate::android::get_android_overlay_status() +} + +#[tauri::command] +pub fn request_android_overlay_permission() -> crate::android::AndroidOverlayPermissionResult { + crate::android::request_android_overlay_permission() +} + +#[tauri::command] +pub fn show_android_overlay() -> Result<(), String> { + crate::android::show_android_overlay() +} + +#[tauri::command] +pub fn hide_android_overlay() -> Result<(), String> { + crate::android::hide_android_overlay() +} + +#[tauri::command] +pub fn get_android_accessibility_status() -> crate::types::AndroidAccessibilityStatus { + crate::android::get_android_accessibility_status() +} + +#[tauri::command] +pub fn request_android_accessibility_permission( +) -> crate::android::AndroidAccessibilityPermissionResult { + crate::android::request_android_accessibility_permission() +} + +#[tauri::command] +pub fn open_external_url(url: String) -> Result<(), String> { + crate::external_url::open_external_url(&url) +} + #[tauri::command] pub fn set_shortcut_recording_active(coord: CoordinatorState<'_>, active: bool) { + #[cfg(mobile)] + { + let _ = (coord, active); + return; + } + #[cfg(not(mobile))] coord.set_shortcut_recording_active(active); } #[tauri::command] +#[cfg(not(mobile))] pub fn get_windows_ime_status() -> WindowsImeStatus { crate::windows_ime_profile::get_windows_ime_status() } #[tauri::command] +#[cfg(mobile)] +pub fn list_microphone_devices() -> Result, String> { + Ok(Vec::new()) +} + +#[tauri::command] +#[cfg(not(mobile))] pub fn list_microphone_devices() -> Result, String> { crate::recorder::list_input_devices().map_err(|e| e.to_string()) } #[tauri::command] +#[cfg(mobile)] +pub async fn start_microphone_level_monitor( + _app: AppHandle, + _device_name: String, +) -> Result<(), String> { + Ok(()) +} + +#[tauri::command] +#[cfg(not(mobile))] pub async fn start_microphone_level_monitor( app: AppHandle, device_name: String, @@ -686,9 +817,10 @@ fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bo if provider == "volcengine" { return volcengine_configured(snap); } - if provider == crate::asr::local::PROVIDER_ID - || active_foundry_asr_is_supported(provider) - || active_sherpa_asr_is_supported(provider) + if !cfg!(mobile) + && (provider == crate::asr::local::PROVIDER_ID + || active_foundry_asr_is_supported(provider) + || active_sherpa_asr_is_supported(provider)) { // 本地 ASR 不依赖云端凭据。 return true; @@ -757,12 +889,14 @@ fn configured(field: &Option) -> bool { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg(not(mobile))] struct LocalAsrReleasePlan { qwen: bool, foundry: bool, sherpa: bool, } +#[cfg(not(mobile))] fn local_asr_release_plan_for_provider(provider: &str) -> LocalAsrReleasePlan { LocalAsrReleasePlan { qwen: provider != crate::asr::local::PROVIDER_ID, @@ -771,6 +905,7 @@ fn local_asr_release_plan_for_provider(provider: &str) -> LocalAsrReleasePlan { } } +#[cfg(not(mobile))] async fn release_foundry_runtime_if_inactive( runtime: &Arc, release_foundry: bool, @@ -783,6 +918,7 @@ async fn release_foundry_runtime_if_inactive( } } +#[cfg(not(mobile))] async fn release_sherpa_runtime_if_inactive( runtime: &Arc, release_sherpa: bool, @@ -811,6 +947,25 @@ pub fn set_credential(window: Window, account: String, value: String) -> Result< Ok(()) } +#[cfg(mobile)] +#[tauri::command] +pub async fn set_active_asr_provider( + _coord: CoordinatorState<'_>, + provider: String, +) -> Result<(), String> { + if provider == crate::asr::local::PROVIDER_ID + || provider == crate::asr::local::sherpa::PROVIDER_ID + || provider == crate::asr::local::foundry::PROVIDER_ID + { + return Err("Local ASR is not available on mobile".to_string()); + } + if CredentialsVault::get_active_asr() == provider { + return Ok(()); + } + CredentialsVault::set_active_asr_provider(&provider).map_err(|e| e.to_string()) +} + +#[cfg(not(mobile))] #[tauri::command] pub async fn set_active_asr_provider( coord: CoordinatorState<'_>, @@ -1104,17 +1259,18 @@ async fn validate_bailian_asr_provider() -> Result<(), String> { } fn active_asr_is_keyless_for_validation(provider: &str) -> bool { - provider == crate::asr::local::PROVIDER_ID - || active_foundry_asr_is_supported(provider) - || active_sherpa_asr_is_supported(provider) + !cfg!(mobile) + && (provider == crate::asr::local::PROVIDER_ID + || active_foundry_asr_is_supported(provider) + || active_sherpa_asr_is_supported(provider)) } fn active_foundry_asr_is_supported(provider: &str) -> bool { - #[cfg(target_os = "windows")] + #[cfg(all(not(mobile), target_os = "windows"))] { provider == FOUNDRY_LOCAL_PROVIDER_ID } - #[cfg(not(target_os = "windows"))] + #[cfg(not(all(not(mobile), target_os = "windows")))] { let _ = provider; false @@ -1122,11 +1278,11 @@ fn active_foundry_asr_is_supported(provider: &str) -> bool { } fn active_sherpa_asr_is_supported(provider: &str) -> bool { - #[cfg(target_os = "windows")] + #[cfg(all(not(mobile), target_os = "windows"))] { provider == crate::asr::local::sherpa::PROVIDER_ID } - #[cfg(not(target_os = "windows"))] + #[cfg(not(all(not(mobile), target_os = "windows")))] { let _ = provider; false @@ -1821,7 +1977,7 @@ pub fn request_microphone_permission(app: AppHandle) -> PermissionStatus { crate::request_microphone_from_foreground(&app) } -/// 跳到 macOS 系统设置的指定隐私面板。pane: "accessibility" | "microphone". +/// 跳到系统设置的指定权限面板。pane: "accessibility" | "microphone" | "overlay". #[tauri::command] pub fn open_system_settings(pane: String) -> Result<(), String> { #[cfg(target_os = "macos")] @@ -1877,10 +2033,25 @@ pub fn open_system_settings(pane: String) -> Result<(), String> { Ok(()) } } - #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] + #[cfg(target_os = "android")] + { + crate::android::jni::android::with_android_env(|env, context| match pane.as_str() { + "microphone" => crate::android::jni::android::launch_app_details_settings(env, context), + "accessibility" => { + crate::android::jni::android::launch_accessibility_settings(env, context) + } + "overlay" => crate::android::jni::android::launch_overlay_settings(env, context), + _ => crate::android::jni::android::launch_app_details_settings(env, context), + }) + } + #[cfg(all( + not(target_os = "macos"), + not(target_os = "windows"), + not(target_os = "android") + ))] { let _ = pane; - Err("open_system_settings is only supported on macOS and Windows".to_string()) + Err("open_system_settings is not supported on this platform".to_string()) } } @@ -1952,6 +2123,19 @@ pub fn qa_window_pin(coord: CoordinatorState<'_>, pinned: bool) { coord.qa_window_pin(pinned); } +/// 移动端 QA 面板录音按钮:Idle -> begin_qa_session,Recording -> end_qa_session。 +#[tauri::command] +pub async fn qa_toggle_recording(coord: CoordinatorState<'_>) -> Result<(), String> { + coord.qa_toggle_recording().await; + Ok(()) +} + +/// QA 面板键盘输入:复用语音 QA 的 LLM 管线,只替换问题来源。 +#[tauri::command] +pub async fn qa_submit_text(coord: CoordinatorState<'_>, text: String) -> Result<(), String> { + coord.qa_submit_text(text).await +} + /// 用户点 ✕ / 按 Esc 关 Less Computer 浮窗。 #[tauri::command] pub fn less_computer_window_dismiss(coord: CoordinatorState<'_>) { @@ -2380,11 +2564,13 @@ fn shortcut_bindings_overlap(left: &ShortcutBinding, right: &ShortcutBinding) -> // ─────────────────────────── local ASR (Qwen3-ASR) ─────────────────────────── +#[cfg(not(mobile))] use crate::asr::local::{ download::{fetch_remote_info, RemoteInfo}, DownloadManager, Mirror, ModelId, ModelStatus, PROVIDER_ID as LOCAL_PROVIDER_ID, }; +#[cfg(not(mobile))] #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct LocalAsrSettings { @@ -2397,6 +2583,7 @@ pub struct LocalAsrSettings { pub engine_available: bool, } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_get_settings(coord: CoordinatorState<'_>) -> LocalAsrSettings { let prefs = coord.prefs().get(); @@ -2414,6 +2601,7 @@ pub fn local_asr_get_settings(coord: CoordinatorState<'_>) -> LocalAsrSettings { } } +#[cfg(not(mobile))] fn non_empty_string(value: String) -> Option { let trimmed = value.trim(); if trimmed.is_empty() { @@ -2423,6 +2611,7 @@ fn non_empty_string(value: String) -> Option { } } +#[cfg(not(mobile))] #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct LocalAsrStorageSettings { @@ -2431,6 +2620,7 @@ pub struct LocalAsrStorageSettings { pub is_default: bool, } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_storage_settings( coord: CoordinatorState<'_>, @@ -2448,6 +2638,7 @@ pub fn local_asr_storage_settings( }) } +#[cfg(not(mobile))] #[tauri::command] pub async fn local_asr_set_models_base_dir( coord: CoordinatorState<'_>, @@ -2496,6 +2687,7 @@ pub async fn local_asr_set_models_base_dir( local_asr_storage_settings(coord) } +#[cfg(not(mobile))] fn same_path_for_command(left: &std::path::Path, right: &std::path::Path) -> bool { match (left.canonicalize(), right.canonicalize()) { (Ok(left), Ok(right)) => left == right, @@ -2503,6 +2695,7 @@ fn same_path_for_command(left: &std::path::Path, right: &std::path::Path) -> boo } } +#[cfg(not(mobile))] async fn quiesce_local_asr_storage_users( coord: &Arc, qwen_manager: &Arc, @@ -2543,6 +2736,7 @@ async fn quiesce_local_asr_storage_users( Err("local ASR downloads are still stopping; retry after cancellation finishes".to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_set_active_model( coord: CoordinatorState<'_>, @@ -2556,6 +2750,7 @@ pub fn local_asr_set_active_model( coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_set_mirror(coord: CoordinatorState<'_>, mirror: String) -> Result<(), String> { let _normalized = Mirror::from_str(&mirror); @@ -2564,6 +2759,7 @@ pub fn local_asr_set_mirror(coord: CoordinatorState<'_>, mirror: String) -> Resu coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_list_models() -> Vec { crate::asr::local::models::list_status() @@ -2571,6 +2767,7 @@ pub fn local_asr_list_models() -> Vec { /// 实时去 HuggingFace API 拉某个模型的真实文件清单 + 总尺寸; /// 前端在显示模型卡时调一次,避免硬编码尺寸过期。 +#[cfg(not(mobile))] #[tauri::command] pub async fn local_asr_fetch_remote_info( model_id: String, @@ -2581,6 +2778,7 @@ pub async fn local_asr_fetch_remote_info( fetch_remote_info(id, m).await.map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_download_model( app: AppHandle, @@ -2594,6 +2792,7 @@ pub fn local_asr_download_model( Ok(()) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_cancel_download( manager: State<'_, Arc>, @@ -2604,6 +2803,7 @@ pub fn local_asr_cancel_download( Ok(()) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_delete_model(coord: CoordinatorState<'_>, model_id: String) -> Result<(), String> { let id = ModelId::from_str(&model_id).ok_or_else(|| format!("unknown model id: {model_id}"))?; @@ -2615,6 +2815,7 @@ pub fn local_asr_delete_model(coord: CoordinatorState<'_>, model_id: String) -> crate::asr::local::models::delete_model(id).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_model_dir(model_id: String) -> Result { let id = ModelId::from_str(&model_id).ok_or_else(|| format!("unknown model id: {model_id}"))?; @@ -2623,6 +2824,7 @@ pub fn local_asr_model_dir(model_id: String) -> Result { .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_reveal_model_dir(model_id: String) -> Result<(), String> { let id = ModelId::from_str(&model_id).ok_or_else(|| format!("unknown model id: {model_id}"))?; @@ -2631,6 +2833,7 @@ pub fn local_asr_reveal_model_dir(model_id: String) -> Result<(), String> { open_path_in_file_manager(&dir) } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_reveal_models_root(coord: CoordinatorState<'_>) -> Result<(), String> { let prefs = coord.prefs().get(); @@ -2640,6 +2843,7 @@ pub fn local_asr_reveal_models_root(coord: CoordinatorState<'_>) -> Result<(), S open_path_in_file_manager(&dir) } +#[cfg(not(mobile))] #[tauri::command] pub async fn local_asr_test_model( model_id: String, @@ -2650,6 +2854,7 @@ pub async fn local_asr_test_model( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct LocalAsrEngineStatus { @@ -2658,6 +2863,7 @@ pub struct LocalAsrEngineStatus { pub keep_loaded_secs: u32, } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_engine_status(coord: CoordinatorState<'_>) -> LocalAsrEngineStatus { let prefs = coord.prefs().get(); @@ -2668,16 +2874,19 @@ pub fn local_asr_engine_status(coord: CoordinatorState<'_>) -> LocalAsrEngineSta } } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_release_engine(coord: CoordinatorState<'_>) { coord.release_local_asr_engine(); } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_preload(coord: tauri::State<'_, std::sync::Arc>) { coord.preload_local_asr_in_background(); } +#[cfg(not(mobile))] #[tauri::command] pub fn local_asr_set_keep_loaded_secs( coord: CoordinatorState<'_>, @@ -2690,6 +2899,7 @@ pub fn local_asr_set_keep_loaded_secs( // ───────────────────── Windows local ASR (Foundry Local Whisper) ───────────────────── +#[cfg(not(mobile))] fn active_foundry_model_from_prefs(prefs: &UserPreferences) -> String { if model_alias_is_known(&prefs.foundry_local_asr_model) { prefs.foundry_local_asr_model.clone() @@ -2698,6 +2908,7 @@ fn active_foundry_model_from_prefs(prefs: &UserPreferences) -> String { } } +#[cfg(not(mobile))] fn validate_foundry_model_alias(model_alias: &str) -> Result<(), String> { if model_alias_is_known(model_alias) { Ok(()) @@ -2708,6 +2919,7 @@ fn validate_foundry_model_alias(model_alias: &str) -> Result<(), String> { } } +#[cfg(not(mobile))] fn normalize_foundry_language_hint(language_hint: &str) -> Result { let normalized = language_hint.trim().to_string(); if normalized.is_empty() @@ -2719,10 +2931,12 @@ fn normalize_foundry_language_hint(language_hint: &str) -> Result String { crate::asr::local::foundry_native::normalize_runtime_source_str(source) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_status( coord: CoordinatorState<'_>, @@ -2735,6 +2949,7 @@ pub async fn foundry_local_asr_status( .await) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_catalog( runtime: State<'_, Arc>, @@ -2745,6 +2960,7 @@ pub async fn foundry_local_asr_catalog( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub fn foundry_local_asr_set_model( coord: CoordinatorState<'_>, @@ -2759,6 +2975,7 @@ pub fn foundry_local_asr_set_model( coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn foundry_local_asr_set_language_hint( coord: CoordinatorState<'_>, @@ -2773,6 +2990,7 @@ pub fn foundry_local_asr_set_language_hint( coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn foundry_local_asr_set_runtime_source( coord: CoordinatorState<'_>, @@ -2787,6 +3005,7 @@ pub fn foundry_local_asr_set_runtime_source( coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_prepare( app: AppHandle, @@ -2820,6 +3039,7 @@ pub async fn foundry_local_asr_prepare( } } +#[cfg(not(mobile))] #[tauri::command] pub fn foundry_local_asr_cancel_prepare( runtime: State<'_, Arc>, @@ -2828,6 +3048,7 @@ pub fn foundry_local_asr_cancel_prepare( Ok(()) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_release( runtime: State<'_, Arc>, @@ -2835,6 +3056,7 @@ pub async fn foundry_local_asr_release( runtime.release_now().await.map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_model_dir( runtime: State<'_, Arc>, @@ -2848,6 +3070,7 @@ pub async fn foundry_local_asr_model_dir( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_delete_model( runtime: State<'_, Arc>, @@ -2860,6 +3083,7 @@ pub async fn foundry_local_asr_delete_model( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub async fn foundry_local_asr_reveal_model_dir( runtime: State<'_, Arc>, @@ -2873,6 +3097,7 @@ pub async fn foundry_local_asr_reveal_model_dir( open_path_in_file_manager(&dir) } +#[cfg(not(mobile))] fn emit_foundry_prepare_progress(app: &AppHandle, payload: FoundryPrepareProgressPayload) { if let Err(error) = app.emit("foundry-local-asr-prepare-progress", payload) { log::warn!("[foundry-asr] emit prepare progress failed: {error}"); @@ -2885,6 +3110,7 @@ fn emit_foundry_prepare_progress(app: &AppHandle, payload: FoundryPrepareProgres // catalog / 下载 / prepare / release / 删除 / 状态查询,推理由 coordinator 的 // 听写链路触发。offline 模型停止录音后 batch decode;online 模型录音时输出 partial。 +#[cfg(not(mobile))] fn active_sherpa_model_from_prefs(prefs: &UserPreferences) -> String { if sherpa_model_alias_is_known(&prefs.sherpa_onnx_model) { prefs.sherpa_onnx_model.clone() @@ -2893,6 +3119,7 @@ fn active_sherpa_model_from_prefs(prefs: &UserPreferences) -> String { } } +#[cfg(not(mobile))] fn validate_sherpa_model_alias(model_alias: &str) -> Result<(), String> { if sherpa_model_alias_is_known(model_alias) { Ok(()) @@ -2901,6 +3128,7 @@ fn validate_sherpa_model_alias(model_alias: &str) -> Result<(), String> { } } +#[cfg(not(mobile))] fn normalize_sherpa_language_hint(language_hint: &str) -> Result { let normalized = language_hint.trim().to_lowercase(); if normalized.is_empty() @@ -2914,6 +3142,7 @@ fn normalize_sherpa_language_hint(language_hint: &str) -> Result } } +#[cfg(not(mobile))] #[tauri::command] pub async fn sherpa_onnx_asr_status( coord: CoordinatorState<'_>, @@ -2924,6 +3153,7 @@ pub async fn sherpa_onnx_asr_status( Ok(runtime.status_snapshot(&active_model).await) } +#[cfg(not(mobile))] #[tauri::command] pub async fn sherpa_onnx_asr_catalog( runtime: State<'_, Arc>, @@ -2934,6 +3164,7 @@ pub async fn sherpa_onnx_asr_catalog( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub async fn sherpa_onnx_asr_fetch_remote_info( model_alias: String, @@ -2946,6 +3177,7 @@ pub async fn sherpa_onnx_asr_fetch_remote_info( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_download_model( app: AppHandle, @@ -2959,6 +3191,7 @@ pub fn sherpa_onnx_asr_download_model( Ok(()) } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_cancel_download( manager: State<'_, Arc>, @@ -2969,6 +3202,7 @@ pub fn sherpa_onnx_asr_cancel_download( Ok(()) } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_set_model( coord: CoordinatorState<'_>, @@ -2983,6 +3217,7 @@ pub fn sherpa_onnx_asr_set_model( coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_set_language_hint( coord: CoordinatorState<'_>, @@ -2997,6 +3232,7 @@ pub fn sherpa_onnx_asr_set_language_hint( coord.prefs().set(prefs).map_err(|e| e.to_string()) } +#[cfg(not(mobile))] #[tauri::command] pub async fn sherpa_onnx_asr_prepare( app: AppHandle, @@ -3027,6 +3263,7 @@ pub async fn sherpa_onnx_asr_prepare( } } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_cancel_prepare( runtime: State<'_, Arc>, @@ -3035,6 +3272,7 @@ pub fn sherpa_onnx_asr_cancel_prepare( Ok(()) } +#[cfg(not(mobile))] #[tauri::command] pub async fn sherpa_onnx_asr_release( runtime: State<'_, Arc>, @@ -3042,6 +3280,7 @@ pub async fn sherpa_onnx_asr_release( runtime.release_now().await.map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_model_dir(model_alias: String) -> Result { validate_sherpa_model_alias(&model_alias)?; @@ -3050,6 +3289,7 @@ pub fn sherpa_onnx_asr_model_dir(model_alias: String) -> Result .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub async fn sherpa_onnx_asr_delete_model( runtime: State<'_, Arc>, @@ -3062,6 +3302,7 @@ pub async fn sherpa_onnx_asr_delete_model( .map_err(|e| format!("{e:#}")) } +#[cfg(not(mobile))] #[tauri::command] pub fn sherpa_onnx_asr_reveal_model_dir(model_alias: String) -> Result<(), String> { validate_sherpa_model_alias(&model_alias)?; @@ -3070,7 +3311,7 @@ pub fn sherpa_onnx_asr_reveal_model_dir(model_alias: String) -> Result<(), Strin open_path_in_file_manager(&dir) } -#[cfg(target_os = "windows")] +#[cfg(all(not(mobile), target_os = "windows"))] fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), String> { use windows::core::PCWSTR; use windows::Win32::UI::Shell::ShellExecuteW; @@ -3099,7 +3340,7 @@ fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), String> { } } -#[cfg(target_os = "macos")] +#[cfg(all(not(mobile), target_os = "macos"))] fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), String> { std::process::Command::new("open") .arg(path) @@ -3108,7 +3349,12 @@ fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), String> { .map_err(|e| e.to_string()) } -#[cfg(all(unix, not(target_os = "macos")))] +#[cfg(all( + not(mobile), + unix, + not(target_os = "macos"), + not(target_os = "android") +))] fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), String> { std::process::Command::new("xdg-open") .arg(path) @@ -3117,6 +3363,7 @@ fn open_path_in_file_manager(path: &std::path::Path) -> Result<(), String> { .map_err(|e| e.to_string()) } +#[cfg(not(mobile))] fn emit_sherpa_prepare_progress(app: &AppHandle, payload: SherpaPrepareProgressPayload) { if let Err(error) = app.emit("sherpa-onnx-asr-prepare-progress", payload) { log::warn!("[sherpa-asr] emit prepare progress failed: {error}"); diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index df34baed..625efefb 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -35,6 +35,7 @@ use crate::coordinator_state::{ publish_abort_idle_after_restore, start_processing_if_listening, startup_race_status, BeginOutcome, SessionId, SessionPhase, SessionState, StartupRaceStatus, }; +use crate::correction::apply_correction_rules; use crate::hotkey::{HotkeyEvent, HotkeyMonitor}; use crate::insertion::TextInserter; use crate::persistence::{ @@ -65,6 +66,17 @@ mod dictation; mod qa; mod resources; +pub(super) fn qa_event_target() -> &'static str { + #[cfg(target_os = "android")] + { + "main" + } + #[cfg(not(target_os = "android"))] + { + "qa" + } +} + #[cfg(test)] use dictation::dictation_error_code; use dictation::{ @@ -73,13 +85,16 @@ use dictation::{ }; #[cfg(any(debug_assertions, test))] use dictation::{handle_pressed, handle_released}; -use qa::{close_qa_panel, handle_qa_hotkey_pressed, QaPhase, QaSessionState}; +use qa::{ + close_qa_panel, handle_qa_hotkey_pressed, handle_qa_option_edge, open_qa_panel, QaPhase, + QaSessionState, +}; #[cfg(test)] use resources::discard_startup_resources_for_session; use resources::{ acquire_recording_mute, cancel_active_asr, release_recording_mute, selected_microphone_device_name, stop_microphone_preview_monitor, stop_qa_recorder, - SessionResource, SharedRecordingMuteState, + take_asr_for_session, take_recorder_for_session, SessionResource, SharedRecordingMuteState, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -490,6 +505,101 @@ impl Coordinator { *self.inner.app.lock() = Some(handle); } + pub fn android_insert_strategy(&self) -> crate::types::AndroidInsertStrategy { + self.inner.prefs.get().android_insert_strategy + } + + pub fn android_overlay_trigger(&self) -> crate::types::AndroidOverlayTrigger { + self.inner.prefs.get().android_overlay_trigger.normalized() + } + + pub fn apply_android_overlay_settings_change( + &self, + previous: &crate::types::UserPreferences, + next: &crate::types::UserPreferences, + ) { + #[cfg(target_os = "android")] + { + use crate::types::android_types::{ + classify_android_overlay_settings_change, AndroidOverlaySettingsAction, + }; + match classify_android_overlay_settings_change(previous, next) { + AndroidOverlaySettingsAction::None => {} + AndroidOverlaySettingsAction::RefreshLayout => { + self.refresh_android_overlay_layout(); + } + AndroidOverlaySettingsAction::Transition { from, to } => { + self.transition_android_overlay_trigger(from, to); + } + } + } + let _ = (previous, next); + } + + pub fn transition_android_overlay_trigger( + &self, + from: crate::types::AndroidOverlayTrigger, + to: crate::types::AndroidOverlayTrigger, + ) { + #[cfg(target_os = "android")] + { + use crate::types::AndroidOverlayTrigger; + fn overlay_trigger_log_name(trigger: AndroidOverlayTrigger) -> &'static str { + match trigger.normalized() { + AndroidOverlayTrigger::Background => "background", + AndroidOverlayTrigger::Keyboard => "keyboard", + AndroidOverlayTrigger::Always => "always", + } + } + if from == to { + return; + } + log::info!( + "[coord] overlay transition from={} to={}", + overlay_trigger_log_name(from), + overlay_trigger_log_name(to), + ); + match (from, to) { + ( + AndroidOverlayTrigger::Background | AndroidOverlayTrigger::Keyboard, + AndroidOverlayTrigger::Always, + ) => { + let _ = crate::android::replace_android_overlay(); + } + ( + AndroidOverlayTrigger::Always, + AndroidOverlayTrigger::Background | AndroidOverlayTrigger::Keyboard, + ) => { + let _ = crate::android::hide_android_overlay(); + } + _ => {} + } + } + let _ = (from, to); + } + + pub fn apply_android_overlay_on_startup(&self) { + #[cfg(target_os = "android")] + { + use crate::types::AndroidOverlayTrigger; + match self.android_overlay_trigger() { + AndroidOverlayTrigger::Always => { + let _ = crate::android::replace_android_overlay(); + } + AndroidOverlayTrigger::Background | AndroidOverlayTrigger::Keyboard => { + let _ = crate::android::hide_android_overlay(); + } + } + } + } + + pub fn refresh_android_overlay_layout(&self) { + #[cfg(target_os = "android")] + { + let _ = crate::android::refresh_android_overlay_layout(); + } + } + /// 让所有 hotkey supervisor loop(dictation / qa / combo / translation / /// switch_style / open_app)在下一轮 sleep / poll 后退出。生产场景下进程退出 /// 一并 reap 所有线程,但 integration test 和未来 RunEvent::Exit 钩子需要 @@ -988,6 +1098,15 @@ impl Coordinator { begin_session(&self.inner).await } + pub async fn start_dictation_with_translation(&self) -> Result<(), String> { + begin_session(&self.inner).await?; + self.inner + .translation_modifier_seen + .store(true, Ordering::SeqCst); + log::info!("[coord] android overlay translation dictation started"); + Ok(()) + } + pub async fn stop_dictation(&self) -> Result<(), String> { if self.inner.state.lock().phase == SessionPhase::Starting { request_stop_during_starting(&self.inner, "manual stop"); @@ -996,10 +1115,32 @@ impl Coordinator { end_session(&self.inner).await } + pub async fn stop_dictation_with_translation(&self, translation: bool) -> Result<(), String> { + if translation { + mark_translation_modifier_seen(&self.inner); + } + self.stop_dictation().await + } + pub fn cancel_dictation(&self) { cancel_session(&self.inner); } + pub fn switch_to_previous_style_pack(&self) { + switch_to_previous_style(&self.inner); + } + + pub async fn open_qa_from_overlay(&self) -> Result<(), String> { + log::info!("[coord] overlay QA open requested"); + open_qa_panel(&self.inner); + begin_qa_session(&self.inner).await + } + + pub async fn finalize_qa_from_overlay(&self) -> Result<(), String> { + log::info!("[coord] overlay QA finalize requested"); + finalize_dictation_as_qa_question(&self.inner).await + } + /// 返回当前听写阶段(read-only 快照),供 CLI 入口在 dispatch toggle 时决策。 /// 与原热键边沿走的 `handle_pressed` 分支完全相同的判定逻辑:Idle → start, /// Listening → stop。可用于桌面快捷键 → CLI 转发的备用触发路径。 @@ -1014,6 +1155,14 @@ impl Coordinator { handle_qa_hotkey_pressed(&self.inner).await; } + pub async fn qa_toggle_recording(&self) { + handle_qa_option_edge(&self.inner).await; + } + + pub async fn qa_submit_text(&self, text: String) -> Result<(), String> { + submit_qa_text_question(&self.inner, text).await + } + pub fn set_shortcut_recording_active(&self, active: bool) { self.inner .shortcut_recording_active @@ -3591,6 +3740,484 @@ fn enabled_hotwords(inner: &Arc) -> Vec { // ─────────────────────────── QA session lifecycle ─────────────────────────── +async fn finalize_dictation_as_qa_question(inner: &Arc) -> Result<(), String> { + log::info!("[coord] QA finalize from overlay: capturing selection before opening panel"); + let selection = capture_selection(); + let selection_preview_text = selection.as_ref().map(|s| s.text.clone()); + + log::info!("[coord] QA finalize from overlay: opening panel and waiting for ASR result"); + open_qa_panel(inner); + { + let mut state = inner.qa_state.lock(); + state.phase = QaPhase::Processing; + state.cancelled = false; + state.session_id = new_session_id(); + state.front_app = capture_frontmost_app(); + state.selection = selection; + } + inner.qa_stream_cancelled.store(false, Ordering::SeqCst); + + if let Some(app) = inner.app.lock().clone() { + let messages = inner.qa_state.lock().messages.clone(); + let _ = app.emit_to( + qa_event_target(), + "qa:state", + serde_json::json!({ + "kind": "loading", + "selection_preview": selection_preview_text, + "messages": messages, + }), + ); + } + + let raw = match take_current_dictation_transcript_for_qa(inner).await { + Ok(Some(raw)) => raw, + Ok(None) => { + log::info!("[coord] QA finalize from overlay: no transcript produced"); + finish_qa_idle_silently(inner); + return Ok(()); + } + Err(error) => { + finish_qa_with_error(inner, error.clone()); + return Err(error); + } + }; + log::info!( + "[coord] QA finalize from overlay: transcript ready chars={} duration_ms={}", + raw.text.chars().count(), + raw.duration_ms + ); + answer_qa_question_text(inner, raw.text.trim().to_string(), raw.duration_ms).await +} + +async fn submit_qa_text_question(inner: &Arc, text: String) -> Result<(), String> { + let question = text.trim().to_string(); + if question.is_empty() { + return Ok(()); + } + + { + let mut state = inner.qa_state.lock(); + if !state.panel_visible { + state.panel_visible = true; + state.messages.clear(); + state.front_app = capture_frontmost_app(); + state.qa_focus_target = capture_focus_target(); + } + if state.phase != QaPhase::Idle { + return Err("QA is busy".to_string()); + } + state.phase = QaPhase::Processing; + state.cancelled = false; + state.session_id = new_session_id(); + if state.selection.is_none() { + state.selection = capture_selection(); + } + } + inner.qa_stream_cancelled.store(false, Ordering::SeqCst); + + let selection_preview_text = inner + .qa_state + .lock() + .selection + .as_ref() + .map(|selection| selection.text.clone()); + if let Some(app) = inner.app.lock().clone() { + let messages = inner.qa_state.lock().messages.clone(); + let _ = app.emit_to( + qa_event_target(), + "qa:state", + serde_json::json!({ + "kind": "thinking", + "selection_preview": selection_preview_text, + "messages": messages, + }), + ); + } + + answer_qa_question_text(inner, question, 0).await +} + +async fn take_current_dictation_transcript_for_qa( + inner: &Arc, +) -> Result, String> { + wait_for_dictation_listening(inner).await?; + + let current_session_id = { + let mut state = inner.state.lock(); + let Some(session_id) = start_processing_if_listening(&mut state) else { + return Ok(None); + }; + session_id + }; + + let elapsed = inner.state.lock().started_at.elapsed().as_millis() as u64; + emit_capsule(inner, CapsuleState::Transcribing, 0.0, elapsed, None, None); + + if let Some(rec) = take_recorder_for_session(inner, current_session_id) { + rec.stop(); + release_recording_mute(inner, "dictation"); + } + + let Some(asr) = take_asr_for_session(inner, current_session_id) else { + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + return Ok(None); + }; + + let mut raw = match transcribe_overlay_dictation_asr(inner, current_session_id, asr).await { + Ok(raw) => raw, + Err(error) => { + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + finish_qa_with_error(inner, format!("识别失败: {error}")); + return Err(error); + } + }; + + if inner.state.lock().cancelled { + log::info!("[coord] overlay QA: cancel detected after ASR — discarding transcript"); + restore_prepared_windows_ime_session(inner, current_session_id); + { + let mut state = inner.state.lock(); + state.phase = SessionPhase::Idle; + state.focus_target = None; + } + return Ok(None); + } + + #[cfg(any(debug_assertions, test))] + if raw.text.trim().is_empty() { + if let Some(debug_text) = debug_transcript_override_text() { + raw.text = debug_text; + } + } + + if raw.text.trim().is_empty() { + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + finish_qa_idle_silently(inner); + return Ok(None); + } + + if let Ok(rules) = inner.correction_rules.list() { + let corrected = apply_correction_rules(&raw.text, &rules); + if corrected != raw.text { + raw.text = corrected; + } + } + + restore_prepared_windows_ime_session(inner, current_session_id); + { + let mut state = inner.state.lock(); + state.phase = SessionPhase::Idle; + state.focus_target = None; + } + Ok(Some(raw)) +} + +async fn wait_for_dictation_listening(inner: &Arc) -> Result<(), String> { + const MAX_WAIT_MS: u64 = 3_000; + const STEP_MS: u64 = 20; + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(MAX_WAIT_MS); + + loop { + let phase = { inner.state.lock().phase }; + match phase { + SessionPhase::Starting if std::time::Instant::now() < deadline => { + tokio::time::sleep(std::time::Duration::from_millis(STEP_MS)).await; + } + SessionPhase::Starting => { + return Err("dictation startup timed out before QA finalize".to_string()); + } + _ => return Ok(()), + } + } +} + +async fn transcribe_overlay_dictation_asr( + _inner: &Arc, + _current_session_id: SessionId, + asr: ActiveAsr, +) -> Result { + let uses_global_timeout = asr_transcribe_uses_global_timeout(&asr); + match asr { + ActiveAsr::Volcengine(asr) => { + debug_assert!(uses_global_timeout); + if let Err(error) = asr.send_last_frame().await { + log::error!("[coord] overlay QA: send last frame failed: {error}"); + } + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, asr.await_final_result()).await { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => { + asr.cancel(); + Err("global timeout".to_string()) + } + } + } + ActiveAsr::Bailian(asr) => { + debug_assert!(uses_global_timeout); + if let Err(error) = asr.send_last_frame().await { + log::error!("[coord] overlay QA: Bailian send last frame failed: {error}"); + } + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, asr.await_final_result()).await { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => { + asr.cancel(); + Err("bailian global timeout".to_string()) + } + } + } + ActiveAsr::Whisper(whisper) => { + debug_assert!(uses_global_timeout); + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, whisper.transcribe()).await { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => Err("whisper global timeout".to_string()), + } + } + ActiveAsr::Mimo(mimo) => { + debug_assert!(uses_global_timeout); + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, mimo.transcribe()).await { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => Err("mimo global timeout".to_string()), + } + } + #[cfg(target_os = "windows")] + ActiveAsr::FoundryLocalWhisper(local) => { + debug_assert!(!uses_global_timeout); + match local + .transcribe(foundry_audio_transcribe_timeout_duration()) + .await + { + Ok(raw) => { + schedule_foundry_local_asr_release( + _inner, + AsrReleaseSession::Dictation(_current_session_id), + ); + Ok(raw) + } + Err(error) => { + schedule_foundry_local_asr_release( + _inner, + AsrReleaseSession::Dictation(_current_session_id), + ); + Err(error.to_string()) + } + } + } + #[cfg(target_os = "windows")] + ActiveAsr::SherpaOnnxLocal(local) => { + debug_assert!(!uses_global_timeout); + match local + .transcribe(sherpa_audio_transcribe_timeout_duration()) + .await + { + Ok(raw) => { + schedule_sherpa_onnx_release( + _inner, + AsrReleaseSession::Dictation(_current_session_id), + ); + Ok(raw) + } + Err(error) => { + schedule_sherpa_onnx_release( + _inner, + AsrReleaseSession::Dictation(_current_session_id), + ); + Err(error.to_string()) + } + } + } + #[cfg(target_os = "macos")] + ActiveAsr::Local(local) => { + debug_assert!(uses_global_timeout); + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = local_qwen_transcribe_timeout(audio_secs); + let result = tokio::time::timeout(timeout_duration, local.transcribe()).await; + _inner.local_asr_cache.touch(); + schedule_local_asr_release(_inner); + match result { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => Err("local qwen transcribe timeout".to_string()), + } + } + } +} + +async fn answer_qa_question_text( + inner: &Arc, + question: String, + duration_ms: u64, +) -> Result<(), String> { + if question.trim().is_empty() { + finish_qa_idle_silently(inner); + return Ok(()); + } + + let user_content = { + let st = inner.qa_state.lock(); + let is_first_turn = st.messages.is_empty(); + let sel_text = st + .selection + .as_ref() + .map(|s| s.text.clone()) + .unwrap_or_default(); + if is_first_turn && !sel_text.trim().is_empty() { + format!( + "# 选区原文\n{}\n\n# 我的问题\n{}", + sel_text.trim(), + question + ) + } else { + question.clone() + } + }; + + inner + .qa_state + .lock() + .messages + .push(crate::types::QaChatMessage { + role: "user".to_string(), + content: user_content, + }); + + if let Some(app) = inner.app.lock().clone() { + let messages = inner.qa_state.lock().messages.clone(); + let _ = app.emit_to( + qa_event_target(), + "qa:state", + serde_json::json!({ + "kind": "thinking", + "messages": messages, + }), + ); + } + + emit_capsule(inner, CapsuleState::Polishing, 0.0, 0, None, None); + + let prefs = inner.prefs.get(); + let working_languages = prefs.working_languages.clone(); + let chinese_script_preference = prefs.chinese_script_preference; + let output_language_preference = prefs.output_language_preference; + let llm_thinking_enabled = prefs.llm_thinking_enabled; + let (messages_for_llm, front_app) = { + let st = inner.qa_state.lock(); + (st.messages.clone(), st.front_app.clone()) + }; + + let captured_session_id = inner.qa_state.lock().session_id; + let inner_for_delta = Arc::clone(inner); + let on_delta = move |chunk: &str| { + let cur_id = inner_for_delta.qa_state.lock().session_id; + if cur_id != captured_session_id { + return; + } + if let Some(app) = inner_for_delta.app.lock().clone() { + let _ = app.emit_to( + qa_event_target(), + "qa:state", + serde_json::json!({ + "kind": "answer_delta", + "chunk": chunk, + }), + ); + } + }; + + let cancel_flag = Arc::clone(&inner.qa_stream_cancelled); + let should_cancel = move || cancel_flag.load(Ordering::Relaxed); + + let answer = match answer_chat_dispatch( + &messages_for_llm, + &working_languages, + chinese_script_preference, + output_language_preference, + llm_thinking_enabled, + front_app.as_deref(), + on_delta, + should_cancel, + ) + .await + { + Ok(answer) => answer, + Err(error) => { + inner.qa_state.lock().messages.pop(); + finish_qa_with_error(inner, format!("回答失败: {error}")); + return Err(error.to_string()); + } + }; + + if inner.qa_state.lock().cancelled { + inner.qa_state.lock().messages.pop(); + finish_qa_idle_silently(inner); + return Ok(()); + } + + inner + .qa_state + .lock() + .messages + .push(crate::types::QaChatMessage { + role: "assistant".to_string(), + content: answer.clone(), + }); + + if let Some(app) = inner.app.lock().clone() { + let messages = inner.qa_state.lock().messages.clone(); + let _ = app.emit_to( + qa_event_target(), + "qa:state", + serde_json::json!({ + "kind": "answer", + "messages": messages, + }), + ); + } + + emit_capsule(inner, CapsuleState::Idle, 0.0, 0, None, None); + + if prefs.qa_save_history { + let session = DictationSession { + id: Uuid::new_v4().to_string(), + created_at: Utc::now().to_rfc3339(), + raw_transcript: question.clone(), + final_text: answer, + mode: PolishMode::Raw, + style_pack_id: None, + translation_active: false, + polish_source: None, + app_bundle_id: None, + app_name: front_app, + insert_status: InsertStatus::CopiedFallback, + error_code: Some("qaSession".to_string()), + duration_ms: Some(duration_ms), + dictionary_entry_count: None, + has_audio_recording: None, + }; + let prefs_snapshot = inner.prefs.get(); + if let Err(error) = inner.history.append_with_retention( + session, + prefs_snapshot.history_retention_days, + prefs_snapshot.history_max_entries, + ) { + log::error!("[coord] overlay QA history append failed: {error}"); + } + } + + inner.qa_state.lock().phase = QaPhase::Idle; + Ok(()) +} + /// 划词语音问答会话(issue #118)。 /// /// 与 dictation 完全分离: @@ -3656,7 +4283,7 @@ async fn begin_qa_session(inner: &Arc) -> Result<(), String> { if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "recording", @@ -3693,7 +4320,7 @@ async fn begin_qa_session(inner: &Arc) -> Result<(), String> { *inner.qa_asr.lock() = Some(qa_asr.active_asr()); // QA recorder 不需要 RMS 节流到胶囊;前端 QA 浮窗有自己的电平视图, - // 这里发一份事件给 "qa" label 用就够了。 + // Android 的 QA 面板嵌在 main WebView;桌面端仍发给独立 qa 窗口。 let inner_for_level = Arc::clone(inner); let last_emit_at = Arc::new(Mutex::new(None::)); const LEVEL_EMIT_MIN_INTERVAL_MS: u64 = 33; @@ -3713,7 +4340,11 @@ async fn begin_qa_session(inner: &Arc) -> Result<(), String> { *last = Some(now); } if let Some(app) = inner_for_level.app.lock().clone() { - let _ = app.emit_to("qa", "qa:level", serde_json::json!({ "level": level })); + let _ = app.emit_to( + qa_event_target(), + "qa:level", + serde_json::json!({ "level": level }), + ); } // 同步把电平推给底部胶囊,让 QA 录音也有跟主听写一致的可视反馈。 emit_capsule( @@ -3795,7 +4426,11 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { emit_capsule(inner, CapsuleState::Transcribing, 0.0, 0, None, None); if let Some(app) = inner.app.lock().clone() { - let _ = app.emit_to("qa", "qa:state", serde_json::json!({ "kind": "loading" })); + let _ = app.emit_to( + qa_event_target(), + "qa:state", + serde_json::json!({ "kind": "loading" }), + ); } stop_qa_recorder(inner); @@ -4032,7 +4667,7 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "thinking", @@ -4069,7 +4704,7 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { } if let Some(app) = inner_for_delta.app.lock().clone() { let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "answer_delta", @@ -4126,7 +4761,7 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "answer", @@ -4183,7 +4818,7 @@ fn finish_qa_with_error(inner: &Arc, message: String) { if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "error", @@ -4205,7 +4840,7 @@ fn finish_qa_idle_silently(inner: &Arc) { if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "idle", @@ -5675,6 +6310,9 @@ fn emit_capsule( operating, }; + #[cfg(target_os = "android")] + crate::android::notify_capsule_state(&payload); + // visible / translation 是「这一帧 capsule:state event 的 payload」内容 —— // 必须在 call-site(即音频线程触发 emit_capsule 时)就算定,否则 main thread // 闭包里读到的将是「下一帧」的 state,跟实际下发给 JS 的 payload 不一致。 diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index 130e6d2a..df0050fc 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -230,7 +230,8 @@ async fn run_streaming_polish( }; // 把 final_text 写回剪贴板(默认 on,macOS/Windows 适用)。 // Linux:fcitx5 插件已直写文字到目标 app,跳过剪贴板避免破坏用户数据。 - #[cfg(not(target_os = "linux"))] + // Android/iOS:无 arboard 剪贴板路径,v1 依赖 IME commit。 + #[cfg(not(any(target_os = "linux", target_os = "android", target_os = "ios")))] if inner.prefs.get().streaming_insert_save_clipboard { match arboard::Clipboard::new() { Ok(mut cb) => match cb.set_text(final_text.clone()) { @@ -2231,44 +2232,55 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { polish_error ); InsertStatus::Inserted - } else if focus_ready_for_paste { - #[cfg(target_os = "windows")] + } else { + #[cfg(target_os = "android")] { - let ime_target = capture_ime_submit_target(); - insert_with_windows_ime_first( - inner, - current_session_id, + crate::android::android_insert_with_strategy( + &inner.inserter, &polished, - restore_clipboard, - allow_non_tsf_insertion_fallback, - paste_shortcut, - ime_target, + inner.prefs.get().android_insert_strategy, ) - .await } - #[cfg(not(target_os = "windows"))] - { - inner - .inserter - .insert(&polished, restore_clipboard, paste_shortcut) - } - } else { - #[cfg(target_os = "linux")] - { - // Linux: fcitx5 commitString 无需窗口焦点,始终尝试插入。 - inner - .inserter - .insert(&polished, restore_clipboard, paste_shortcut) - } - #[cfg(not(target_os = "linux"))] - { - log::warn!( - "[coord] original insertion target is not foreground; copied output without paste" - ); - if allow_non_tsf_insertion_fallback { - inner.inserter.copy_fallback(&polished) - } else { - InsertStatus::Failed + #[cfg(not(target_os = "android"))] + if focus_ready_for_paste { + #[cfg(target_os = "windows")] + { + let ime_target = capture_ime_submit_target(); + insert_with_windows_ime_first( + inner, + current_session_id, + &polished, + restore_clipboard, + allow_non_tsf_insertion_fallback, + paste_shortcut, + ime_target, + ) + .await + } + #[cfg(not(target_os = "windows"))] + { + inner + .inserter + .insert(&polished, restore_clipboard, paste_shortcut) + } + } else { + #[cfg(target_os = "linux")] + { + // Linux: fcitx5 commitString 无需窗口焦点,始终尝试插入。 + inner + .inserter + .insert(&polished, restore_clipboard, paste_shortcut) + } + #[cfg(not(target_os = "linux"))] + { + log::warn!( + "[coord] original insertion target is not foreground; copied output without paste" + ); + if allow_non_tsf_insertion_fallback { + inner.inserter.copy_fallback(&polished) + } else { + InsertStatus::Failed + } } } }; @@ -2763,4 +2775,9 @@ mod tests { fn platform_type_error() -> crate::unicode_keystroke::TypeError { crate::unicode_keystroke::TypeError::EnigoText("fail".into()) } + + #[cfg(target_os = "android")] + fn platform_type_error() -> crate::unicode_keystroke::TypeError { + crate::unicode_keystroke::TypeError::Unavailable + } } diff --git a/openless-all/app/src-tauri/src/coordinator/qa.rs b/openless-all/app/src-tauri/src/coordinator/qa.rs index 69614742..b0e2a445 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa.rs @@ -8,7 +8,7 @@ use crate::types::CapsuleState; use super::{ begin_qa_session, cancel_qa_session, capture_focus_target, capture_frontmost_app, emit_capsule, - end_qa_session, Inner, + end_qa_session, qa_event_target, Inner, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -108,7 +108,7 @@ pub(super) fn open_qa_panel(inner: &Arc) { if let Some(app) = inner.app.lock().clone() { crate::show_qa_window(&app, "idle"); let _ = app.emit_to( - "qa", + qa_event_target(), "qa:state", serde_json::json!({ "kind": "idle", diff --git a/openless-all/app/src-tauri/src/external_url.rs b/openless-all/app/src-tauri/src/external_url.rs new file mode 100644 index 00000000..0fdebeab --- /dev/null +++ b/openless-all/app/src-tauri/src/external_url.rs @@ -0,0 +1,70 @@ +pub fn open_external_url(url: &str) -> Result<(), String> { + let parsed = url::Url::parse(url).map_err(|error| format!("invalid URL: {error}"))?; + match parsed.scheme() { + "http" | "https" => {} + scheme => return Err(format!("unsupported URL scheme: {scheme}")), + } + + platform_open_external_url(parsed.as_str()) +} + +#[cfg(target_os = "android")] +fn platform_open_external_url(url: &str) -> Result<(), String> { + use jni::objects::{JObject, JValue}; + + let android_context = ndk_context::android_context(); + let vm = unsafe { + jni::JavaVM::from_raw(android_context.vm().cast()) + .map_err(|error| format!("attach Android JVM: {error}"))? + }; + let mut env = vm + .attach_current_thread() + .map_err(|error| format!("attach Android thread: {error}"))?; + let context = unsafe { JObject::from_raw(android_context.context() as jni::sys::jobject) }; + + let action = env + .new_string("android.intent.action.VIEW") + .map_err(|error| format!("create Intent action: {error}"))?; + let url = env + .new_string(url) + .map_err(|error| format!("create URL string: {error}"))?; + let uri = env + .call_static_method( + "android/net/Uri", + "parse", + "(Ljava/lang/String;)Landroid/net/Uri;", + &[JValue::Object(&JObject::from(url))], + ) + .and_then(|value| value.l()) + .map_err(|error| format!("parse URL into Android Uri: {error}"))?; + let intent = env + .new_object( + "android/content/Intent", + "(Ljava/lang/String;Landroid/net/Uri;)V", + &[JValue::Object(&JObject::from(action)), JValue::Object(&uri)], + ) + .map_err(|error| format!("create Android Intent: {error}"))?; + + // Context may be an application context; NEW_TASK keeps startActivity valid there. + env.call_method( + &intent, + "addFlags", + "(I)Landroid/content/Intent;", + &[JValue::Int(0x10000000)], + ) + .map_err(|error| format!("set Android Intent flags: {error}"))?; + env.call_method( + &context, + "startActivity", + "(Landroid/content/Intent;)V", + &[JValue::Object(&intent)], + ) + .map_err(|error| format!("start Android URL activity: {error}"))?; + + Ok(()) +} + +#[cfg(not(target_os = "android"))] +fn platform_open_external_url(_url: &str) -> Result<(), String> { + Err("native external URL fallback is only wired on Android".to_string()) +} diff --git a/openless-all/app/src-tauri/src/insertion.rs b/openless-all/app/src-tauri/src/insertion.rs index d2502021..d4c5ac18 100644 --- a/openless-all/app/src-tauri/src/insertion.rs +++ b/openless-all/app/src-tauri/src/insertion.rs @@ -5,22 +5,19 @@ //! - macOS:用 CoreGraphics CGEvent 直接 post Cmd+V。 //! - Windows / Linux:用 enigo 按 `PasteShortcut` 模拟。 -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] use std::sync::atomic::{AtomicU64, Ordering}; -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] use std::time::Duration; -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] use once_cell::sync::Lazy; -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] use parking_lot::Mutex; use crate::types::{InsertStatus, PasteShortcut}; -#[cfg(target_os = "windows")] -const CLIPBOARD_RESTORE_DELAY: Duration = Duration::from_millis(750); - -#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] const CLIPBOARD_RESTORE_DELAY: Duration = Duration::from_millis(750); pub struct TextInserter; @@ -63,7 +60,7 @@ impl TextInserter { insert_with_clipboard_restore(text, restore_clipboard_after_paste, paste_shortcut) } - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] pub fn insert_via_clipboard_fallback( &self, text: &str, @@ -107,6 +104,20 @@ impl TextInserter { macos_insert_status_after_paste(simulate_paste()) } + /// Android:跨应用输入由 dictation 流程按用户策略处理;通用插入只写剪贴板兜底。 + #[cfg(target_os = "android")] + pub fn insert( + &self, + text: &str, + _restore_clipboard_after_paste: bool, + _paste_shortcut: PasteShortcut, + ) -> InsertStatus { + if text.is_empty() { + return InsertStatus::CopiedFallback; + } + self.copy_fallback(text) + } + /// 只写剪贴板、不模拟粘贴。用于目标控件活跃状态无法验证时的兜底路径。 pub fn copy_fallback(&self, text: &str) -> InsertStatus { if text.is_empty() { @@ -163,27 +174,28 @@ impl Default for TextInserter { } } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] #[derive(Debug)] struct ClipboardRestorePlan { inserted_text: String, previous_text: Option, } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] #[derive(Debug, Clone)] struct PendingClipboardRestore { latest_restore_id: u64, original_text: Option, } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] static NEXT_CLIPBOARD_RESTORE_ID: AtomicU64 = AtomicU64::new(1); -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] static PENDING_CLIPBOARD_RESTORE: Lazy>> = Lazy::new(|| Mutex::new(None)); +#[cfg(not(any(target_os = "android", target_os = "ios")))] fn copy_to_clipboard(text: &str) -> bool { let mut clipboard = match arboard::Clipboard::new() { Ok(c) => c, @@ -199,7 +211,28 @@ fn copy_to_clipboard(text: &str) -> bool { true } -#[cfg(not(target_os = "macos"))] +#[cfg(any(target_os = "android", target_os = "ios"))] +fn copy_to_clipboard(text: &str) -> bool { + #[cfg(target_os = "android")] + { + return crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::copy_to_clipboard(env, context, text) + }) + .unwrap_or_else(|error| { + log::error!("[insertion] android clipboard failed: {error}"); + false + }); + } + + #[cfg(target_os = "ios")] + { + let _ = text; + log::warn!("[insertion] mobile clipboard fallback unavailable"); + false + } +} + +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn copy_to_clipboard_with_restore_plan(text: &str) -> Result { let mut clipboard = arboard::Clipboard::new().map_err(|e| e.to_string())?; let previous_text = match clipboard.get_text() { @@ -221,7 +254,7 @@ fn copy_to_clipboard_with_restore_plan(text: &str) -> Result) -> (u64, Option) { let restore_id = NEXT_CLIPBOARD_RESTORE_ID.fetch_add(1, Ordering::SeqCst); let original_text = { @@ -273,7 +306,7 @@ fn remember_pending_clipboard_restore(previous_text: Option) -> (u64, Op (restore_id, original_text) } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn restore_clipboard_after_delay( plan: ClipboardRestorePlan, original_text: Option, @@ -324,7 +357,7 @@ fn restore_clipboard_after_delay( clear_pending_clipboard_restore(restore_id); } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn is_latest_clipboard_restore(restore_id: u64) -> bool { matches!( PENDING_CLIPBOARD_RESTORE.lock().as_ref(), @@ -332,7 +365,7 @@ fn is_latest_clipboard_restore(restore_id: u64) -> bool { ) } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn clear_pending_clipboard_restore(restore_id: u64) { let mut pending = PENDING_CLIPBOARD_RESTORE.lock(); if matches!(pending.as_ref(), Some(batch) if batch.latest_restore_id == restore_id) { @@ -340,7 +373,7 @@ fn clear_pending_clipboard_restore(restore_id: u64) { } } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn should_restore_clipboard(current_text: Option<&str>, inserted_text: &str) -> bool { matches!(current_text, Some(current) if current == inserted_text) } @@ -357,7 +390,7 @@ fn simulate_paste() -> Result<(), String> { } /// 把 `PasteShortcut` 拆成 `(modifiers, primary)`,顺序决定按下/释放顺序。 -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn paste_keys(shortcut: PasteShortcut) -> (Vec, enigo::Key) { use enigo::Key; match shortcut { @@ -367,7 +400,7 @@ fn paste_keys(shortcut: PasteShortcut) -> (Vec, enigo::Key) { } } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn simulate_paste(shortcut: PasteShortcut) -> Result<(), String> { use enigo::{Direction, Enigo, Keyboard, Settings}; let (modifiers, primary) = paste_keys(shortcut); @@ -411,7 +444,7 @@ fn insertion_success_status() -> InsertStatus { InsertStatus::Inserted } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn insertion_success_status() -> InsertStatus { InsertStatus::PasteSent } @@ -561,7 +594,7 @@ mod tests { use std::time::Duration; #[test] - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn restore_only_when_clipboard_still_holds_inserted_text() { assert!(should_restore_clipboard( Some("dictated text"), @@ -576,7 +609,7 @@ mod tests { /// 配置的快捷键必须真实映射到对应按键。只比较 modifier 数 + 主键,规避 enigo 内部 PartialEq。 #[test] - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn paste_keys_match_configured_shortcut() { use enigo::Key; @@ -605,7 +638,7 @@ mod tests { inserter.insert("", true, PasteShortcut::CtrlV), InsertStatus::CopiedFallback ); - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] { assert_eq!( inserter.insert_via_clipboard_fallback("", true, PasteShortcut::CtrlV), @@ -691,7 +724,7 @@ mod tests { } #[test] - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn pending_clipboard_restore_keeps_first_original_until_latest_restore() { *PENDING_CLIPBOARD_RESTORE.lock() = None; @@ -713,7 +746,7 @@ mod tests { } #[test] - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "android", target_os = "ios")))] fn clipboard_restore_skips_when_clipboard_no_longer_matches_inserted_text() { assert!(should_restore_clipboard( Some("dictated text"), diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 36334163..2cb2456a 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -14,34 +14,68 @@ //! - coordinator: dictation state machine glue //! - commands: Tauri IPC surface +mod android; mod asr; mod audio_mute; mod cli; mod coding_agent; +#[cfg(not(mobile))] +mod combo_hotkey; +#[cfg(mobile)] +#[path = "mobile_stubs/combo_hotkey.rs"] mod combo_hotkey; mod commands; mod coordinator; mod coordinator_state; mod correction; +mod external_url; +#[cfg(not(mobile))] mod global_hotkey_runtime; +#[cfg(not(mobile))] +#[path = "hotkey.rs"] +mod hotkey; +#[cfg(mobile)] +#[path = "mobile_stubs/hotkey.rs"] mod hotkey; mod insertion; #[cfg(target_os = "linux")] mod linux_fcitx; mod llm_gemini; +#[cfg(mobile)] +mod mobile_runtime; mod net; mod permissions; mod persistence; mod polish; +#[cfg(not(mobile))] +mod qa_hotkey; +#[cfg(mobile)] +#[path = "mobile_stubs/qa_hotkey.rs"] mod qa_hotkey; mod recorder; +#[cfg(not(mobile))] +#[path = "selection.rs"] mod selection; +#[cfg(mobile)] +#[path = "mobile_stubs/selection.rs"] +mod selection; +#[cfg(not(mobile))] +mod shortcut_binding; +#[cfg(mobile)] +#[path = "mobile_stubs/shortcut_binding.rs"] mod shortcut_binding; mod types; +#[cfg(not(mobile))] +mod unicode_keystroke; +#[cfg(mobile)] +#[path = "mobile_stubs/unicode_keystroke.rs"] mod unicode_keystroke; +#[cfg(target_os = "windows")] mod windows_ime_ipc; mod windows_ime_profile; +#[cfg(target_os = "windows")] mod windows_ime_protocol; +#[cfg(target_os = "windows")] mod windows_ime_session; use std::sync::atomic::{AtomicBool, Ordering}; @@ -57,10 +91,13 @@ const OPENLESS_BUNDLE_ID: &str = "com.openless.app"; /// 第一次 show 时把 QA 浮窗摆到屏幕底部居中;之后的 show 不再 reposition, /// 让用户拖动后的位置在 hide → show 之间得以保持。详见 issue #118 v2。 static QA_WINDOW_POSITIONED: AtomicBool = AtomicBool::new(false); +#[cfg(not(mobile))] static TRAY_MICROPHONE_WATCHER_STOPPING: AtomicBool = AtomicBool::new(false); +#[cfg(not(mobile))] use tauri::menu::{ CheckMenuItemBuilder, Menu, MenuBuilder, MenuItemBuilder, Submenu, SubmenuBuilder, }; +#[cfg(not(mobile))] use tauri::tray::{MouseButton, TrayIconBuilder, TrayIconEvent}; use tauri::{ AppHandle, Emitter, LogicalPosition, LogicalSize, Manager, PhysicalPosition, PhysicalSize, @@ -71,6 +108,261 @@ use crate::types::PolishMode; #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { + #[cfg(mobile)] + { + mobile_runtime::run(); + return; + } + #[cfg(not(mobile))] + run_desktop(); +} + +macro_rules! app_invoke_handler_desktop { + () => { + tauri::generate_handler![ + commands::get_settings, + commands::get_default_style_system_prompts, + commands::set_settings, + commands::get_update_channel, + commands::set_update_channel, + commands::fetch_latest_beta_release, + commands::app_check_update_with_channel, + commands::check_network, + commands::get_hotkey_status, + commands::get_hotkey_capability, + commands::set_shortcut_recording_active, + commands::get_windows_ime_status, + commands::get_platform_capabilities, + commands::get_android_overlay_status, + commands::request_android_overlay_permission, + commands::show_android_overlay, + commands::hide_android_overlay, + commands::get_android_accessibility_status, + commands::request_android_accessibility_permission, + commands::open_external_url, + commands::list_microphone_devices, + commands::start_microphone_level_monitor, + commands::stop_microphone_level_monitor, + commands::get_credentials, + commands::set_credential, + commands::list_history, + commands::delete_history_entry, + commands::clear_history, + commands::read_audio_recording, + commands::marketplace_list, + commands::marketplace_detail, + commands::marketplace_install, + commands::marketplace_upload, + commands::marketplace_like, + commands::marketplace_my_likes, + commands::marketplace_my_packs, + commands::marketplace_delete, + commands::github_device_flow_start, + commands::github_device_flow_poll, + commands::list_vocab, + commands::add_vocab, + commands::remove_vocab, + commands::set_vocab_enabled, + commands::list_correction_rules, + commands::add_correction_rule, + commands::remove_correction_rule, + commands::set_correction_rule_enabled, + commands::list_vocab_presets, + commands::save_vocab_presets, + commands::start_dictation, + commands::stop_dictation, + commands::cancel_dictation, + coding_agent::commands::coding_agent_detect, + coding_agent::commands::coding_agent_run_test, + coding_agent::commands::coding_agent_cancel_test, + coding_agent::commands::coding_agent_command_risk, + commands::handle_window_hotkey_event, + #[cfg(debug_assertions)] + commands::inject_hotkey_click_for_dev, + commands::repolish, + commands::list_style_packs, + commands::create_style_pack_from_template, + commands::save_style_pack, + commands::preview_style_pack_runtime, + commands::set_active_style_pack, + commands::set_style_pack_enabled, + commands::reset_builtin_style_pack, + commands::delete_style_pack, + commands::import_style_pack_from_zip, + commands::export_style_pack_to_zip, + commands::set_default_polish_mode, + commands::set_style_enabled, + commands::check_accessibility_permission, + commands::request_accessibility_permission, + commands::check_microphone_permission, + commands::request_microphone_permission, + commands::open_system_settings, + commands::trigger_microphone_prompt, + commands::read_credential, + commands::set_active_asr_provider, + commands::set_active_llm_provider, + commands::get_qa_hotkey_label, + commands::set_qa_hotkey, + commands::validate_shortcut_binding, + commands::set_dictation_hotkey, + commands::set_translation_hotkey, + commands::set_switch_style_hotkey, + commands::set_open_app_hotkey, + commands::qa_window_dismiss, + commands::qa_window_pin, + commands::less_computer_window_dismiss, + commands::less_computer_window_resize, + commands::less_computer_approve, + commands::validate_combo_hotkey, + commands::set_combo_hotkey, + commands::validate_provider_credentials, + commands::list_provider_models, + commands::local_asr_get_settings, + commands::local_asr_storage_settings, + commands::local_asr_set_models_base_dir, + commands::local_asr_set_active_model, + commands::local_asr_set_mirror, + commands::local_asr_list_models, + commands::local_asr_fetch_remote_info, + commands::local_asr_download_model, + commands::local_asr_cancel_download, + commands::local_asr_delete_model, + commands::local_asr_model_dir, + commands::local_asr_reveal_model_dir, + commands::local_asr_reveal_models_root, + commands::local_asr_test_model, + commands::local_asr_engine_status, + commands::local_asr_release_engine, + commands::local_asr_preload, + commands::local_asr_set_keep_loaded_secs, + commands::foundry_local_asr_status, + commands::foundry_local_asr_catalog, + commands::foundry_local_asr_set_model, + commands::foundry_local_asr_set_language_hint, + commands::foundry_local_asr_set_runtime_source, + commands::foundry_local_asr_prepare, + commands::foundry_local_asr_cancel_prepare, + commands::foundry_local_asr_release, + commands::foundry_local_asr_model_dir, + commands::foundry_local_asr_delete_model, + commands::foundry_local_asr_reveal_model_dir, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_status, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_catalog, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_fetch_remote_info, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_download_model, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_cancel_download, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_set_model, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_set_language_hint, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_prepare, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_cancel_prepare, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_release, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_model_dir, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_delete_model, + #[cfg(target_os = "windows")] + commands::sherpa_onnx_asr_reveal_model_dir, + commands::export_error_log, + restart_app, + ] + }; +} + +/// Android/iOS: only commands usable without desktop hotkeys, tray, updater, or local ASR. +#[macro_export] +macro_rules! app_invoke_handler_mobile { + () => { + tauri::generate_handler![ + $crate::commands::get_settings, + $crate::commands::get_default_style_system_prompts, + $crate::commands::set_settings, + $crate::commands::check_network, + $crate::commands::get_platform_capabilities, + $crate::commands::get_android_overlay_status, + $crate::commands::request_android_overlay_permission, + $crate::commands::show_android_overlay, + $crate::commands::hide_android_overlay, + $crate::commands::get_android_accessibility_status, + $crate::commands::request_android_accessibility_permission, + $crate::commands::open_external_url, + $crate::commands::list_microphone_devices, + $crate::commands::start_microphone_level_monitor, + $crate::commands::stop_microphone_level_monitor, + $crate::commands::get_credentials, + $crate::commands::set_credential, + $crate::commands::read_credential, + $crate::commands::set_active_asr_provider, + $crate::commands::set_active_llm_provider, + $crate::commands::validate_provider_credentials, + $crate::commands::list_provider_models, + $crate::commands::list_history, + $crate::commands::delete_history_entry, + $crate::commands::clear_history, + $crate::commands::read_audio_recording, + $crate::commands::marketplace_list, + $crate::commands::marketplace_detail, + $crate::commands::marketplace_install, + $crate::commands::marketplace_upload, + $crate::commands::marketplace_like, + $crate::commands::marketplace_my_likes, + $crate::commands::marketplace_my_packs, + $crate::commands::marketplace_delete, + $crate::commands::github_device_flow_start, + $crate::commands::github_device_flow_poll, + $crate::commands::list_vocab, + $crate::commands::add_vocab, + $crate::commands::remove_vocab, + $crate::commands::set_vocab_enabled, + $crate::commands::list_correction_rules, + $crate::commands::add_correction_rule, + $crate::commands::remove_correction_rule, + $crate::commands::set_correction_rule_enabled, + $crate::commands::list_vocab_presets, + $crate::commands::save_vocab_presets, + $crate::commands::start_dictation, + $crate::commands::stop_dictation, + $crate::commands::cancel_dictation, + $crate::commands::qa_window_dismiss, + $crate::commands::qa_window_pin, + $crate::commands::qa_toggle_recording, + $crate::commands::qa_submit_text, + $crate::commands::repolish, + $crate::commands::list_style_packs, + $crate::commands::create_style_pack_from_template, + $crate::commands::save_style_pack, + $crate::commands::preview_style_pack_runtime, + $crate::commands::set_active_style_pack, + $crate::commands::set_style_pack_enabled, + $crate::commands::reset_builtin_style_pack, + $crate::commands::delete_style_pack, + $crate::commands::import_style_pack_from_zip, + $crate::commands::export_style_pack_to_zip, + $crate::commands::set_default_polish_mode, + $crate::commands::set_style_enabled, + $crate::commands::check_accessibility_permission, + $crate::commands::request_accessibility_permission, + $crate::commands::check_microphone_permission, + $crate::commands::request_microphone_permission, + $crate::commands::open_system_settings, + $crate::commands::trigger_microphone_prompt, + $crate::commands::export_error_log, + $crate::restart_app, + ] + }; +} + +#[cfg(not(mobile))] +fn run_desktop() { let foundry_local_runtime = Arc::new(asr::local::FoundryLocalRuntime::new()); let sherpa_onnx_runtime = Arc::new(asr::local::SherpaOnnxRuntime::new()); let sherpa_download_manager = @@ -343,154 +635,7 @@ pub fn run() { Ok(()) }) - .invoke_handler(tauri::generate_handler![ - commands::get_settings, - commands::get_default_style_system_prompts, - commands::set_settings, - commands::get_update_channel, - commands::set_update_channel, - commands::fetch_latest_beta_release, - commands::app_check_update_with_channel, - commands::check_network, - commands::get_hotkey_status, - commands::get_hotkey_capability, - commands::set_shortcut_recording_active, - commands::get_windows_ime_status, - commands::list_microphone_devices, - commands::start_microphone_level_monitor, - commands::stop_microphone_level_monitor, - commands::get_credentials, - commands::set_credential, - commands::list_history, - commands::delete_history_entry, - commands::clear_history, - commands::read_audio_recording, - commands::marketplace_list, - commands::marketplace_detail, - commands::marketplace_install, - commands::marketplace_upload, - commands::marketplace_like, - commands::marketplace_my_likes, - commands::marketplace_my_packs, - commands::marketplace_delete, - commands::github_device_flow_start, - commands::github_device_flow_poll, - commands::list_vocab, - commands::add_vocab, - commands::remove_vocab, - commands::set_vocab_enabled, - commands::list_correction_rules, - commands::add_correction_rule, - commands::remove_correction_rule, - commands::set_correction_rule_enabled, - commands::list_vocab_presets, - commands::save_vocab_presets, - commands::start_dictation, - commands::stop_dictation, - commands::cancel_dictation, - coding_agent::commands::coding_agent_detect, - coding_agent::commands::coding_agent_run_test, - coding_agent::commands::coding_agent_cancel_test, - coding_agent::commands::coding_agent_command_risk, - commands::handle_window_hotkey_event, - #[cfg(debug_assertions)] - commands::inject_hotkey_click_for_dev, - commands::repolish, - commands::list_style_packs, - commands::create_style_pack_from_template, - commands::save_style_pack, - commands::preview_style_pack_runtime, - commands::set_active_style_pack, - commands::set_style_pack_enabled, - commands::reset_builtin_style_pack, - commands::delete_style_pack, - commands::import_style_pack_from_zip, - commands::export_style_pack_to_zip, - commands::set_default_polish_mode, - commands::set_style_enabled, - commands::check_accessibility_permission, - commands::request_accessibility_permission, - commands::check_microphone_permission, - commands::request_microphone_permission, - commands::open_system_settings, - commands::trigger_microphone_prompt, - commands::read_credential, - commands::set_active_asr_provider, - commands::set_active_llm_provider, - commands::get_qa_hotkey_label, - commands::set_qa_hotkey, - commands::validate_shortcut_binding, - commands::set_dictation_hotkey, - commands::set_translation_hotkey, - commands::set_switch_style_hotkey, - commands::set_open_app_hotkey, - commands::qa_window_dismiss, - commands::qa_window_pin, - commands::less_computer_window_dismiss, - commands::less_computer_window_resize, - commands::less_computer_approve, - commands::validate_combo_hotkey, - commands::set_combo_hotkey, - commands::validate_provider_credentials, - commands::list_provider_models, - commands::local_asr_get_settings, - commands::local_asr_storage_settings, - commands::local_asr_set_models_base_dir, - commands::local_asr_set_active_model, - commands::local_asr_set_mirror, - commands::local_asr_list_models, - commands::local_asr_fetch_remote_info, - commands::local_asr_download_model, - commands::local_asr_cancel_download, - commands::local_asr_delete_model, - commands::local_asr_model_dir, - commands::local_asr_reveal_model_dir, - commands::local_asr_reveal_models_root, - commands::local_asr_test_model, - commands::local_asr_engine_status, - commands::local_asr_release_engine, - commands::local_asr_preload, - commands::local_asr_set_keep_loaded_secs, - commands::foundry_local_asr_status, - commands::foundry_local_asr_catalog, - commands::foundry_local_asr_set_model, - commands::foundry_local_asr_set_language_hint, - commands::foundry_local_asr_set_runtime_source, - commands::foundry_local_asr_prepare, - commands::foundry_local_asr_cancel_prepare, - commands::foundry_local_asr_release, - commands::foundry_local_asr_model_dir, - commands::foundry_local_asr_delete_model, - commands::foundry_local_asr_reveal_model_dir, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_status, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_catalog, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_fetch_remote_info, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_download_model, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_cancel_download, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_set_model, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_set_language_hint, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_prepare, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_cancel_prepare, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_release, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_model_dir, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_delete_model, - #[cfg(target_os = "windows")] - commands::sherpa_onnx_asr_reveal_model_dir, - commands::export_error_log, - restart_app, - ]) + .invoke_handler(app_invoke_handler_desktop!()) .build(tauri::generate_context!()) .expect("error while building tauri application") .run(|app, event| match event { @@ -531,21 +676,25 @@ pub fn run() { }); } +#[cfg(not(mobile))] struct MicrophoneTrayMenu { submenu: Submenu, items: Vec, } +#[cfg(not(mobile))] struct StyleTrayMenu { submenu: Submenu, } +#[cfg(not(mobile))] struct TrayMenu { menu: Menu, microphone_items: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] +#[cfg(not(mobile))] struct TrayPolishModeMenuEntry { id: String, label: &'static str, @@ -554,9 +703,13 @@ struct TrayPolishModeMenuEntry { } fn tray_style_menu_enabled() -> bool { - cfg!(target_os = "windows") + #[cfg(all(not(mobile), target_os = "windows"))] + return true; + #[cfg(not(all(not(mobile), target_os = "windows")))] + false } +#[cfg(not(mobile))] fn tray_polish_mode_menu_entries(selected: PolishMode) -> Vec { [ (PolishMode::Raw, "style-raw"), @@ -574,6 +727,7 @@ fn tray_polish_mode_menu_entries(selected: PolishMode) -> Vec Option { match id { "style-raw" => Some(PolishMode::Raw), @@ -584,6 +738,7 @@ fn parse_tray_polish_mode_id(id: &str) -> Option { } } +#[cfg(not(mobile))] fn build_tray_menu>( app: &M, coordinator: &Arc, @@ -609,6 +764,7 @@ fn build_tray_menu>( }) } +#[cfg(not(mobile))] fn build_style_tray_menu>( app: &M, coordinator: &Arc, @@ -631,6 +787,7 @@ fn build_style_tray_menu>( }) } +#[cfg(not(mobile))] fn build_microphone_tray_menu>( app: &M, coordinator: &Arc, @@ -689,6 +846,7 @@ fn build_microphone_tray_menu>( }) } +#[cfg(not(mobile))] pub(crate) fn refresh_tray_microphone_menu(app: &AppHandle) -> tauri::Result<()> { let coordinator = app.state::>(); let tray_menu = build_tray_menu(app, &coordinator)?; @@ -700,6 +858,7 @@ pub(crate) fn refresh_tray_microphone_menu(app: &AppHandle) -> tauri::Result<()> Ok(()) } +#[cfg(not(mobile))] fn microphone_device_signature() -> Option> { match recorder::list_input_devices() { Ok(devices) => Some( @@ -715,6 +874,7 @@ fn microphone_device_signature() -> Option> { } } +#[cfg(not(mobile))] fn start_tray_microphone_watcher(app: AppHandle) { TRAY_MICROPHONE_WATCHER_STOPPING.store(false, Ordering::Relaxed); if let Err(err) = std::thread::Builder::new() @@ -748,6 +908,7 @@ fn start_tray_microphone_watcher(app: AppHandle) { } } +#[cfg(not(mobile))] fn handle_microphone_tray_menu_event(app: &AppHandle, id: &str) { let tray_items = app.state::(); let items = tray_items.lock(); @@ -767,6 +928,7 @@ fn handle_microphone_tray_menu_event(app: &AppHandle, id: &str) { commands::sync_tray_microphone_selection(&items, &selected.device_name); } +#[cfg(not(mobile))] fn handle_style_tray_menu_event(app: &AppHandle, id: &str) -> bool { let Some(mode) = parse_tray_polish_mode_id(id) else { return false; @@ -782,6 +944,11 @@ fn handle_style_tray_menu_event(app: &AppHandle, id: &str) -> bool { true } +#[cfg(mobile)] +pub(crate) fn refresh_tray_microphone_menu(_app: &AppHandle) -> tauri::Result<()> { + Ok(()) +} + /// 把 Win11 原生标题栏底色刷成白色,与应用 sidebar 视觉统一。需要 Win11 22H2+ /// (Build 22621+) 才支持 `DWMWA_CAPTION_COLOR`(35);老 Windows 上 DwmSetWindowAttribute /// 返回错误,仅打 warn 不阻塞启动。 @@ -947,7 +1114,7 @@ pub fn log_dir_path() -> std::path::PathBuf { .join("Logs"); } } - #[cfg(all(unix, not(target_os = "macos")))] + #[cfg(all(unix, not(target_os = "macos"), not(target_os = "android")))] { if let Ok(home) = std::env::var("HOME") { return std::path::PathBuf::from(home) @@ -957,6 +1124,12 @@ pub fn log_dir_path() -> std::path::PathBuf { .join("logs"); } } + #[cfg(target_os = "android")] + { + if let Ok(dir) = std::env::var("TAURI_ANDROID_APP_DATA_DIR") { + return std::path::PathBuf::from(dir).join("logs"); + } + } std::env::temp_dir().join("OpenLess") } @@ -964,6 +1137,7 @@ pub(crate) fn show_main_window(app: &AppHandle) { activate_window_mode(app); if let Some(w) = app.get_webview_window("main") { let _ = w.show(); + #[cfg(not(mobile))] let _ = w.unminimize(); let _ = w.set_focus(); } @@ -1235,6 +1409,33 @@ fn position_qa_window(window: &tauri::WebviewWindow) -> ta /// 让前端 React 视图自行决定渲染哪一种。**不**抢前台 app 焦点(保证 Cmd+C /// fallback 仍能从原 app 拿到选区)。 pub(crate) fn show_qa_window(app: &AppHandle, content_kind: &str) { + #[cfg(target_os = "android")] + { + const FLAG_ACTIVITY_NEW_TASK: i32 = 0x10000000; + const FLAG_ACTIVITY_REORDER_TO_FRONT: i32 = 0x00020000; + const FLAG_ACTIVITY_SINGLE_TOP: i32 = 0x20000000; + let flags = + FLAG_ACTIVITY_NEW_TASK | FLAG_ACTIVITY_REORDER_TO_FRONT | FLAG_ACTIVITY_SINGLE_TOP; + match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::start_activity_class_with_flags( + env, + context, + "com.openless.app.MainActivity", + flags, + ) + }) { + Ok(()) => log::info!("[qa] android requested MainActivity foreground for QA"), + Err(error) => log::warn!("[qa] android failed to foreground MainActivity: {error}"), + } + log::info!("[qa] android emit qa:state to main kind={content_kind}"); + let _ = app.emit_to( + "main", + "qa:state", + serde_json::json!({ "kind": content_kind }), + ); + return; + } + let Some(window) = app.get_webview_window("qa") else { log::info!("[qa] show 跳过:qa 窗口不存在 (content_kind={content_kind})"); return; @@ -1327,6 +1528,12 @@ fn make_qa_window_draggable_macos(window: &tauri::WebviewWind /// 隐藏 QA 窗口。供 commands::qa_window_dismiss / coordinator session 收尾共用。 pub(crate) fn hide_qa_window(app: &AppHandle) { + #[cfg(target_os = "android")] + { + let _ = app.emit_to("main", "qa:dismiss", serde_json::json!({})); + return; + } + if let Some(window) = app.get_webview_window("qa") { let _ = window.hide(); } diff --git a/openless-all/app/src-tauri/src/mobile_runtime.rs b/openless-all/app/src-tauri/src/mobile_runtime.rs new file mode 100644 index 00000000..5b466af5 --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_runtime.rs @@ -0,0 +1,81 @@ +//! Minimal Tauri mobile runtime — single main window, no tray/hotkey/updater. + +use std::sync::Arc; + +use tauri::{AppHandle, Manager, RunEvent}; + +use crate::commands::MicrophoneMonitorState; +use crate::coordinator::Coordinator; + +pub fn run() { + let coordinator = Arc::new(Coordinator::new()); + + tauri::Builder::default() + .plugin(tauri_plugin_shell::init()) + .plugin(tauri_plugin_dialog::init()) + .manage(coordinator.clone()) + .manage(MicrophoneMonitorState::new(None)) + .setup(move |app| { + crate::init_file_logger(); + log::info!("=== OpenLess mobile 启动 ==="); + initialize_android_ndk_context_for_audio(); + + if let Some(main) = app.get_webview_window("main") { + let _ = main.show(); + } + if let Some(qa) = app.get_webview_window("qa") { + let _ = qa.hide(); + } + + coordinator.bind_app(app.handle().clone()); + #[cfg(target_os = "android")] + { + crate::android::register_android_coordinator(coordinator.clone()); + coordinator.apply_android_overlay_on_startup(); + } + Ok(()) + }) + .invoke_handler(crate::app_invoke_handler_mobile!()) + .build(tauri::generate_context!()) + .expect("error while building tauri mobile application") + .run(|app, event| match event { + RunEvent::Exit => { + let coordinator = app.state::>(); + coordinator.stop_hotkey_listener(); + } + _ => {} + }); +} + +#[allow(dead_code)] +pub(crate) fn show_main_window(app: &AppHandle) { + if let Some(w) = app.get_webview_window("main") { + let _ = w.show(); + let _ = w.set_focus(); + } +} + +#[cfg(target_os = "android")] +fn initialize_android_ndk_context_for_audio() { + static INIT: std::sync::Once = std::sync::Once::new(); + + INIT.call_once(|| { + let Some(context) = tao::platform::android::prelude::main_android_context() else { + log::warn!("[android] tao Android context unavailable; audio backend may fail"); + return; + }; + + let result = std::panic::catch_unwind(|| unsafe { + ndk_context::initialize_android_context(context.java_vm, context.context_jobject); + }); + + if result.is_ok() { + log::info!("[android] initialized ndk-context for audio backend"); + } else { + log::warn!("[android] ndk-context was already initialized or rejected initialization"); + } + }); +} + +#[cfg(not(target_os = "android"))] +fn initialize_android_ndk_context_for_audio() {} diff --git a/openless-all/app/src-tauri/src/mobile_stubs/combo_hotkey.rs b/openless-all/app/src-tauri/src/mobile_stubs/combo_hotkey.rs new file mode 100644 index 00000000..3a7907ef --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_stubs/combo_hotkey.rs @@ -0,0 +1,46 @@ +//! Mobile stub — combo hotkeys are unavailable on Android/iOS. + +use std::sync::mpsc::Sender; + +use crate::types::ShortcutBinding; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ComboHotkeyEvent { + Pressed, + Released, +} + +#[derive(Debug, thiserror::Error)] +pub enum ComboHotkeyError { + #[error("不支持的修饰键: {0}")] + UnsupportedModifier(String), + #[error("不支持的主键: {0}")] + UnsupportedKey(String), + #[error("注册全局快捷键失败: {0}")] + RegisterFailed(String), + #[error("初始化全局快捷键管理器失败: {0}")] + ManagerInitFailed(String), +} + +pub struct ComboHotkeyMonitor; + +impl ComboHotkeyMonitor { + pub fn start( + _binding: ShortcutBinding, + _tx: Sender, + ) -> Result { + Err(ComboHotkeyError::RegisterFailed( + "Combo hotkeys are not available on mobile".into(), + )) + } + + pub fn update_binding(&self, _binding: ShortcutBinding) -> Result<(), ComboHotkeyError> { + Ok(()) + } +} + +pub fn validate_binding(_binding: &ShortcutBinding) -> Result<(), ComboHotkeyError> { + Err(ComboHotkeyError::RegisterFailed( + "Combo hotkeys are not available on mobile".into(), + )) +} diff --git a/openless-all/app/src-tauri/src/mobile_stubs/hotkey.rs b/openless-all/app/src-tauri/src/mobile_stubs/hotkey.rs new file mode 100644 index 00000000..59c6c7e7 --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_stubs/hotkey.rs @@ -0,0 +1,49 @@ +//! Mobile stub — global hotkeys are unavailable on Android/iOS. + +use std::sync::mpsc::Sender; + +use crate::types::{ + HotkeyAdapterKind, HotkeyBinding, HotkeyCapability, HotkeyInstallError, HotkeyTrigger, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HotkeyEvent { + Pressed, + Released, + Cancelled, + TranslationModifierPressed, + QaShortcutPressed, +} + +pub struct HotkeyMonitor; + +impl HotkeyMonitor { + pub fn start( + _binding: HotkeyBinding, + _tx: Sender, + ) -> Result { + Err(HotkeyInstallError { + code: "unavailable".into(), + message: "Global hotkeys are not available on mobile".into(), + }) + } + + pub fn update_binding(&self, _binding: HotkeyBinding) {} + + pub fn update_modifier_shortcuts( + &self, + _qa_trigger: Option, + _translation_trigger: Option, + ) { + } + + pub fn kind(&self) -> HotkeyAdapterKind { + HotkeyAdapterKind::Unavailable + } + + pub fn reset_held_state(&self) {} + + pub fn capability() -> HotkeyCapability { + HotkeyCapability::current() + } +} diff --git a/openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs b/openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs new file mode 100644 index 00000000..b94c7acb --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs @@ -0,0 +1,33 @@ +//! Mobile stub — QA hotkeys are unavailable on Android/iOS. + +use std::sync::mpsc::Sender; + +use crate::types::ShortcutBinding; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QaHotkeyEvent { + Pressed, +} + +#[derive(Debug, thiserror::Error)] +pub enum QaHotkeyError { + #[error("注册 QA 快捷键失败: {0}")] + RegisterFailed(String), +} + +pub struct QaHotkeyMonitor; + +impl QaHotkeyMonitor { + pub fn start( + _binding: ShortcutBinding, + _tx: Sender, + ) -> Result { + Err(QaHotkeyError::RegisterFailed( + "QA hotkeys are not available on mobile".into(), + )) + } + + pub fn update_binding(&self, _binding: ShortcutBinding) -> Result<(), QaHotkeyError> { + Ok(()) + } +} diff --git a/openless-all/app/src-tauri/src/mobile_stubs/selection.rs b/openless-all/app/src-tauri/src/mobile_stubs/selection.rs new file mode 100644 index 00000000..ef48faa6 --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_stubs/selection.rs @@ -0,0 +1,54 @@ +//! Mobile selection capture. + +const SELECTION_MAX_CHARS: usize = 4000; +const SELECTION_TRUNCATE_HEAD: usize = 2000; +const SELECTION_TRUNCATE_TAIL: usize = 2000; +const SELECTION_TRUNCATED_MARKER: &str = "\n[…truncated…]\n"; + +#[derive(Debug, Clone)] +pub struct SelectionContext { + pub text: String, + pub source_app: Option, +} + +#[cfg(target_os = "android")] +pub fn capture_selection() -> Option { + let text = match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::accessibility_selected_text(env, context) + }) { + Ok(Some(text)) => text, + Ok(None) => return None, + Err(error) => { + log::warn!("[selection] Android accessibility selection read failed: {error}"); + return None; + } + }; + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + log::info!( + "[selection] Android accessibility read OK ({} chars)", + trimmed.chars().count() + ); + Some(SelectionContext { + text: truncate_selection(trimmed), + source_app: Some("Android accessibility".to_string()), + }) +} + +#[cfg(not(target_os = "android"))] +pub fn capture_selection() -> Option { + None +} + +fn truncate_selection(text: &str) -> String { + let total: usize = text.chars().count(); + if total <= SELECTION_MAX_CHARS { + return text.to_string(); + } + let head: String = text.chars().take(SELECTION_TRUNCATE_HEAD).collect(); + let tail_start = total.saturating_sub(SELECTION_TRUNCATE_TAIL); + let tail: String = text.chars().skip(tail_start).collect(); + format!("{head}{SELECTION_TRUNCATED_MARKER}{tail}") +} diff --git a/openless-all/app/src-tauri/src/mobile_stubs/shortcut_binding.rs b/openless-all/app/src-tauri/src/mobile_stubs/shortcut_binding.rs new file mode 100644 index 00000000..7946a44d --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_stubs/shortcut_binding.rs @@ -0,0 +1,38 @@ +//! Mobile stub — shortcut binding validation is unavailable on mobile. + +use crate::types::{HotkeyTrigger, ShortcutBinding}; + +#[derive(Debug, thiserror::Error)] +pub enum ShortcutBindingError { + #[error("快捷键在移动端不可用")] + Unavailable, +} + +pub fn validate_binding(_binding: &ShortcutBinding) -> Result<(), ShortcutBindingError> { + Err(ShortcutBindingError::Unavailable) +} + +pub fn parse_global_hotkey(_binding: &ShortcutBinding) -> Result<(), ShortcutBindingError> { + Err(ShortcutBindingError::Unavailable) +} + +pub fn legacy_modifier_trigger(_binding: &ShortcutBinding) -> Option { + None +} + +pub fn binding_from_legacy_trigger(trigger: HotkeyTrigger) -> ShortcutBinding { + let primary = match trigger { + HotkeyTrigger::RightOption | HotkeyTrigger::RightAlt => "RightOption", + HotkeyTrigger::LeftOption => "LeftOption", + HotkeyTrigger::RightControl => "RightControl", + HotkeyTrigger::LeftControl => "LeftControl", + HotkeyTrigger::RightCommand => "RightCommand", + HotkeyTrigger::Fn => "Fn", + HotkeyTrigger::MediaPlayPause => "MediaPlayPause", + HotkeyTrigger::Custom => "RightOption", + }; + ShortcutBinding { + primary: primary.into(), + modifiers: Vec::new(), + } +} diff --git a/openless-all/app/src-tauri/src/mobile_stubs/unicode_keystroke.rs b/openless-all/app/src-tauri/src/mobile_stubs/unicode_keystroke.rs new file mode 100644 index 00000000..9adc27ed --- /dev/null +++ b/openless-all/app/src-tauri/src/mobile_stubs/unicode_keystroke.rs @@ -0,0 +1,40 @@ +//! Mobile stub — unicode keystroke streaming is unavailable on mobile. + +use tauri::{AppHandle, Runtime}; + +#[derive(Debug, thiserror::Error)] +pub enum TypeError { + #[allow(dead_code)] + #[error("{source} after {typed_chars} chars were sent")] + Partial { + typed_chars: usize, + #[source] + source: Box, + }, + #[error("unicode keystroke unavailable on mobile")] + Unavailable, +} + +impl TypeError { + pub fn typed_chars(&self) -> usize { + match self { + TypeError::Partial { typed_chars, .. } => *typed_chars, + _ => 0, + } + } +} + +pub async fn switch_to_ascii(_app: &AppHandle) -> Result, TypeError> { + Err(TypeError::Unavailable) +} + +pub async fn restore_input_source( + _app: &AppHandle, + _previous: Option<()>, +) -> Result<(), TypeError> { + Ok(()) +} + +pub fn type_unicode_chunk(_text: &str) -> Result { + Err(TypeError::Unavailable) +} diff --git a/openless-all/app/src-tauri/src/permissions.rs b/openless-all/app/src-tauri/src/permissions.rs index ef1452d4..c525a92d 100644 --- a/openless-all/app/src-tauri/src/permissions.rs +++ b/openless-all/app/src-tauri/src/permissions.rs @@ -244,9 +244,54 @@ mod platform { } } -// ─────────────────────────── Windows / 其他 ─────────────────────────── +// ─────────────────────────── Android ─────────────────────────── -#[cfg(not(target_os = "macos"))] +#[cfg(target_os = "android")] +mod platform { + use super::PermissionStatus; + + pub fn check_accessibility() -> PermissionStatus { + PermissionStatus::NotApplicable + } + + pub fn request_accessibility() -> PermissionStatus { + PermissionStatus::NotApplicable + } + + pub fn check_microphone() -> PermissionStatus { + match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::check_self_permission( + env, + context, + "android.permission.RECORD_AUDIO", + ) + }) { + Ok(true) => PermissionStatus::Granted, + Ok(false) => PermissionStatus::Denied, + Err(error) => { + log::warn!("[mic] Android RECORD_AUDIO permission check failed: {error}"); + PermissionStatus::NotDetermined + } + } + } + + pub fn request_microphone() -> PermissionStatus { + match crate::android::jni::android::with_android_env(|env, context| { + crate::android::jni::android::request_record_audio_permission(env, context) + }) { + Ok(true) => PermissionStatus::Granted, + Ok(false) => PermissionStatus::NotDetermined, + Err(error) => { + log::warn!("[mic] Android RECORD_AUDIO permission request failed: {error}"); + check_microphone() + } + } + } +} + +// ─────────────────────────── Windows / Linux / 其他 ─────────────────────────── + +#[cfg(all(not(target_os = "macos"), not(target_os = "android")))] mod platform { use super::PermissionStatus; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; diff --git a/openless-all/app/src-tauri/src/persistence.rs b/openless-all/app/src-tauri/src/persistence.rs index e4557472..386f18c1 100644 --- a/openless-all/app/src-tauri/src/persistence.rs +++ b/openless-all/app/src-tauri/src/persistence.rs @@ -48,6 +48,8 @@ const LEGACY_CREDS_FILE: &str = "credentials.json"; const KEYRING_CREDENTIALS_ACCOUNT: &str = "credentials.v1"; const KEYRING_CREDENTIALS_CHUNK_PREFIX: &str = "credentials.v1.chunk."; +#[cfg(target_os = "android")] +const ANDROID_CREDENTIALS_FILE: &str = "credentials.enc.json"; // Windows Credential Manager caps one credential blob at 2560 bytes. keyring stores // passwords as UTF-16 on Windows, so keep each JSON chunk comfortably below that. const KEYRING_CHUNK_MAX_UTF16_UNITS: usize = 1000; @@ -111,7 +113,7 @@ fn data_dir() -> Result { Ok(PathBuf::from(appdata).join("OpenLess")) } - #[cfg(all(unix, not(target_os = "macos")))] + #[cfg(all(unix, not(target_os = "macos"), not(target_os = "android")))] { if let Ok(xdg) = std::env::var("XDG_DATA_HOME") { if !xdg.is_empty() { @@ -124,6 +126,14 @@ fn data_dir() -> Result { .join("share") .join("OpenLess")) } + + #[cfg(target_os = "android")] + { + if let Ok(dir) = std::env::var("TAURI_ANDROID_APP_DATA_DIR") { + return Ok(PathBuf::from(dir).join("OpenLess")); + } + Ok(std::env::temp_dir().join("OpenLess")) + } } fn ensure_dir(dir: &Path) -> Result<()> { @@ -640,15 +650,54 @@ fn credentials_path() -> Result { } } +#[cfg(not(target_os = "android"))] fn keyring_entry() -> Result { keyring_entry_for(KEYRING_CREDENTIALS_ACCOUNT) } +#[cfg(not(target_os = "android"))] fn keyring_entry_for(account: &str) -> Result { keyring::Entry::new(CredentialsVault::SERVICE_NAME, account) .context("open system credential vault") } +#[cfg(target_os = "android")] +fn android_credentials_path() -> Result { + Ok(data_dir()?.join(ANDROID_CREDENTIALS_FILE)) +} + +#[cfg(target_os = "android")] +fn load_android_credentials() -> Result> { + let path = android_credentials_path()?; + if !path.exists() { + return Ok(None); + } + let bytes = fs::read(&path).with_context(|| format!("read failed: {}", path.display()))?; + if bytes.is_empty() { + return Ok(None); + } + // Stub: base64 envelope — replace with Keystore-backed AES when JNI lands. + use base64::Engine; + let decoded = base64::engine::general_purpose::STANDARD + .decode(bytes) + .context("decode android credentials envelope")?; + let root = + serde_json::from_slice::(&decoded).context("parse android credentials json")?; + Ok(Some(root)) +} + +#[cfg(target_os = "android")] +fn save_android_credentials(root: &CredsRoot) -> Result<()> { + let cleaned = clean_credentials(root); + let json = serde_json::to_string(&cleaned).context("encode credentials failed")?; + use base64::Engine; + let encoded = base64::engine::general_purpose::STANDARD.encode(json.as_bytes()); + let path = android_credentials_path()?; + ensure_dir(path.parent().unwrap_or_else(|| Path::new(".")))?; + fs::write(&path, encoded).with_context(|| format!("write failed: {}", path.display()))?; + Ok(()) +} + fn clean_credentials(root: &CredsRoot) -> CredsRoot { let mut cleaned = root.clone(); cleaned.providers.asr.retain(|_, v| !v.is_empty()); @@ -742,6 +791,7 @@ fn read_chunk_manifest(json: &str) -> Option { } } +#[cfg(not(target_os = "android"))] fn get_keyring_password(account: &str) -> Result> { match keyring_entry_for(account)?.get_password() { Ok(value) => Ok(Some(value)), @@ -752,6 +802,7 @@ fn get_keyring_password(account: &str) -> Result> { } } +#[cfg(not(target_os = "android"))] fn delete_keyring_password(account: &str) { match keyring_entry_for(account).and_then(|entry| { entry @@ -762,6 +813,7 @@ fn delete_keyring_password(account: &str) { } } +#[cfg(not(target_os = "android"))] fn load_keyring_credentials() -> Result> { let Some(json_or_manifest) = get_keyring_password(KEYRING_CREDENTIALS_ACCOUNT)? else { return Ok(None); @@ -782,6 +834,7 @@ fn load_keyring_credentials() -> Result> { .context("decode system credential vault payload") } +#[cfg(not(target_os = "android"))] fn load_legacy_keyring_credentials() -> CredsRoot { match load_legacy_keyring_credentials_for_update() { Ok(root) => root, @@ -792,6 +845,7 @@ fn load_legacy_keyring_credentials() -> CredsRoot { } } +#[cfg(not(target_os = "android"))] fn load_legacy_keyring_credentials_for_update() -> Result { let mut root = CredsRoot::default(); for account in CredentialAccount::all() { @@ -805,6 +859,7 @@ fn load_legacy_keyring_credentials_for_update() -> Result { Ok(clean_credentials(&root)) } +#[cfg(not(target_os = "android"))] fn remove_legacy_keyring_credentials() { for account in CredentialAccount::all() { delete_keyring_password(account.keyring_account()); @@ -826,9 +881,12 @@ fn load_legacy_sources_without_migration() -> CredsRoot { return legacy; } - let legacy_vault = load_legacy_keyring_credentials(); - if legacy_vault_has_credentials(&legacy_vault) { - return legacy_vault; + #[cfg(not(target_os = "android"))] + { + let legacy_vault = load_legacy_keyring_credentials(); + if legacy_vault_has_credentials(&legacy_vault) { + return legacy_vault; + } } CredsRoot::default() @@ -847,15 +905,19 @@ fn migrate_legacy_sources() -> CredsRoot { fn migrate_legacy_sources_for_update() -> Result { if let Some(legacy) = load_legacy_credentials() { save_credentials(&legacy)?; + #[cfg(not(target_os = "android"))] remove_legacy_keyring_credentials(); return Ok(legacy); } - let legacy_vault = load_legacy_keyring_credentials_for_update()?; - if legacy_vault_has_credentials(&legacy_vault) { - save_credentials(&legacy_vault)?; - remove_legacy_keyring_credentials(); - return Ok(legacy_vault); + #[cfg(not(target_os = "android"))] + { + let legacy_vault = load_legacy_keyring_credentials_for_update()?; + if legacy_vault_has_credentials(&legacy_vault) { + save_credentials(&legacy_vault)?; + remove_legacy_keyring_credentials(); + return Ok(legacy_vault); + } } Ok(CredsRoot::default()) @@ -865,6 +927,22 @@ fn load_credentials() -> CredsRoot { if let Some(cached) = credentials_cache().lock().as_ref().cloned() { return cached; } + + #[cfg(target_os = "android")] + { + let root = match load_android_credentials() { + Ok(Some(root)) => root, + Ok(None) => CredsRoot::default(), + Err(e) => { + log::warn!("[vault] android credential read failed: {e}"); + CredsRoot::default() + } + }; + store_credentials_cache(&root); + return root; + } + + #[cfg(not(target_os = "android"))] match load_keyring_credentials() { Ok(Some(root)) => { // 不在这里调 remove_legacy_keyring_credentials() —— 它内部对每个 @@ -901,6 +979,18 @@ fn load_credentials_for_update() -> Result { if let Some(cached) = credentials_cache().lock().as_ref().cloned() { return Ok(cached); } + + #[cfg(target_os = "android")] + { + let root = match load_android_credentials()? { + Some(root) => root, + None => CredsRoot::default(), + }; + store_credentials_cache(&root); + return Ok(root); + } + + #[cfg(not(target_os = "android"))] match load_keyring_credentials() { Ok(Some(root)) => { // 同 load_credentials:不再每次 update 都尝试 delete legacy keyring @@ -924,59 +1014,70 @@ fn load_credentials_for_update() -> Result { fn save_credentials(root: &CredsRoot) -> Result<()> { let cleaned = clean_credentials(root); - let json = serde_json::to_string(&cleaned).context("encode credentials failed")?; - let previous_manifest = get_keyring_password(KEYRING_CREDENTIALS_ACCOUNT) - .ok() - .flatten() - .and_then(|value| read_chunk_manifest(&value)); - let chunks = chunk_json_payload(&json); - - // 先写所有 chunks(稳定名),再写 manifest —— 保证 partial-write 不会让 - // manifest 指向不完整 chunks。stable name 让 macOS Keychain ACL 一次允许后 - // 长期有效,不再因 UUID 轮换反复弹窗(这是 PR #277 早期 UUID-rotation - // 设计的回退)。 - for (index, chunk) in chunks.iter().enumerate() { - let account = chunk_account(None, index); - keyring_entry_for(&account)? - .set_password(chunk) - .with_context(|| format!("write system credential vault chunk {index}"))?; - } - - let manifest = CredsChunkManifest { - openless_credentials_storage: "chunked".to_string(), - version: 1, - generation: None, - chunks: chunks.len(), - }; - let manifest_json = - serde_json::to_string(&manifest).context("encode credential manifest failed")?; - keyring_entry()? - .set_password(&manifest_json) - .context("write system credential vault manifest")?; - - // 清理旧 chunks: - // 1) 旧 manifest 用 UUID generation → 那一代 chunks 全删(迁移到 stable name) - // 2) 旧 manifest 也是 stable name,但 chunks 数量比这次多 → 删多余的 idx - if let Some(previous) = previous_manifest { - match previous.generation.as_deref() { - Some(prev_gen) => { - for index in 0..previous.chunks { - delete_keyring_password(&chunk_account(Some(prev_gen), index)); + + #[cfg(target_os = "android")] + { + save_android_credentials(&cleaned)?; + store_credentials_cache(&cleaned); + return Ok(()); + } + + #[cfg(not(target_os = "android"))] + { + let json = serde_json::to_string(&cleaned).context("encode credentials failed")?; + let previous_manifest = get_keyring_password(KEYRING_CREDENTIALS_ACCOUNT) + .ok() + .flatten() + .and_then(|value| read_chunk_manifest(&value)); + let chunks = chunk_json_payload(&json); + + // 先写所有 chunks(稳定名),再写 manifest —— 保证 partial-write 不会让 + // manifest 指向不完整 chunks。stable name 让 macOS Keychain ACL 一次允许后 + // 长期有效,不再因 UUID 轮换反复弹窗(这是 PR #277 早期 UUID-rotation + // 设计的回退)。 + for (index, chunk) in chunks.iter().enumerate() { + let account = chunk_account(None, index); + keyring_entry_for(&account)? + .set_password(chunk) + .with_context(|| format!("write system credential vault chunk {index}"))?; + } + + let manifest = CredsChunkManifest { + openless_credentials_storage: "chunked".to_string(), + version: 1, + generation: None, + chunks: chunks.len(), + }; + let manifest_json = + serde_json::to_string(&manifest).context("encode credential manifest failed")?; + keyring_entry()? + .set_password(&manifest_json) + .context("write system credential vault manifest")?; + + // 清理旧 chunks: + // 1) 旧 manifest 用 UUID generation → 那一代 chunks 全删(迁移到 stable name) + // 2) 旧 manifest 也是 stable name,但 chunks 数量比这次多 → 删多余的 idx + if let Some(previous) = previous_manifest { + match previous.generation.as_deref() { + Some(prev_gen) => { + for index in 0..previous.chunks { + delete_keyring_password(&chunk_account(Some(prev_gen), index)); + } } - } - None => { - for index in chunks.len()..previous.chunks { - delete_keyring_password(&chunk_account(None, index)); + None => { + for index in chunks.len()..previous.chunks { + delete_keyring_password(&chunk_account(None, index)); + } } } } - } - remove_legacy_credentials_file_best_effort(); - // 写完成功后立刻刷新 process cache —— 同进程后续读不再回 Keychain。 - // 见 CREDENTIALS_CACHE 的 doc。 - store_credentials_cache(&cleaned); - Ok(()) + remove_legacy_credentials_file_best_effort(); + // 写完成功后立刻刷新 process cache —— 同进程后续读不再回 Keychain。 + // 见 CREDENTIALS_CACHE 的 doc。 + store_credentials_cache(&cleaned); + Ok(()) + } } fn lookup_account(root: &CredsRoot, account: CredentialAccount) -> Option { diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index a943b783..a87e1ee9 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -311,7 +311,8 @@ fn build_input_stream( .map_err(|e| classify_default_config_err(e.to_string()))?; let sample_format = supported.sample_format(); - let config: StreamConfig = supported.config(); + let default_config: StreamConfig = supported.config(); + let config = stable_input_config_for_platform(&default_config); let input_sr = config.sample_rate.0; let channels = config.channels as usize; @@ -324,21 +325,59 @@ fn build_input_stream( ); let state = Arc::new(StreamState::new()); - let stream = build_stream_for_format( + let stream = match build_stream_for_format( &device, &config, sample_format, - consumer, - level_handler, - archiver, + Arc::clone(&consumer), + Arc::clone(&level_handler), + archiver.clone(), Arc::clone(&state), input_sr, channels, - runtime_error_tx, - )?; + runtime_error_tx.clone(), + ) { + Ok(stream) => stream, + Err(err) if config != default_config => { + log::warn!( + "[recorder] stable input config failed; falling back to default config: {err}" + ); + build_stream_for_format( + &device, + &default_config, + sample_format, + consumer, + level_handler, + archiver, + Arc::clone(&state), + default_config.sample_rate.0, + default_config.channels as usize, + runtime_error_tx, + )? + } + Err(err) => return Err(err), + }; Ok((stream, state)) } +#[cfg(target_os = "android")] +fn stable_input_config_for_platform(default_config: &StreamConfig) -> StreamConfig { + let mut config = default_config.clone(); + if config.channels > 1 { + log::info!( + "[recorder] android forcing mono input channels: {} -> 1", + config.channels + ); + config.channels = 1; + } + config +} + +#[cfg(not(target_os = "android"))] +fn stable_input_config_for_platform(default_config: &StreamConfig) -> StreamConfig { + default_config.clone() +} + fn select_input_device( host: &cpal::Host, microphone_device_name: Option<&str>, diff --git a/openless-all/app/src-tauri/src/types.rs b/openless-all/app/src-tauri/src/types.rs index 7528cfd4..07f0a752 100644 --- a/openless-all/app/src-tauri/src/types.rs +++ b/openless-all/app/src-tauri/src/types.rs @@ -3,6 +3,22 @@ use serde::{Deserialize, Serialize}; +#[path = "android/types.rs"] +pub mod android_types; + +use android_types::{ + default_android_insert_strategy, default_android_overlay_activation_mode, + default_android_overlay_cancel_swipe_direction, default_android_overlay_left_swipe_action, + default_android_overlay_size_dp, default_android_overlay_trigger, + normalize_android_insert_strategy, normalize_android_overlay_size_dp, +}; +pub use android_types::{ + AndroidAccessibilityState, AndroidAccessibilityStatus, AndroidInsertStrategy, + AndroidOverlayActivationMode, AndroidOverlayCancelSwipeDirection, + AndroidOverlayLeftSwipeAction, AndroidOverlayPermissionState, AndroidOverlayStatus, + AndroidOverlayTrigger, +}; + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] #[derive(Default)] @@ -749,6 +765,24 @@ pub struct UserPreferences { /// 上传 / 点赞需要带这个 header;空时上传被后端 401。 #[serde(default)] pub marketplace_dev_login: String, + /// Android: text insertion strategy for cross-app dictation results. + #[serde(default = "default_android_insert_strategy")] + pub android_insert_strategy: AndroidInsertStrategy, + /// Android: when to show the floating overlay control. + #[serde(default = "default_android_overlay_trigger")] + pub android_overlay_trigger: AndroidOverlayTrigger, + /// Android: how the floating overlay enters the armed interaction state. + #[serde(default = "default_android_overlay_activation_mode")] + pub android_overlay_activation_mode: AndroidOverlayActivationMode, + /// Android: action performed by left swiping while the overlay is armed. + #[serde(default = "default_android_overlay_left_swipe_action")] + pub android_overlay_left_swipe_action: AndroidOverlayLeftSwipeAction, + /// Android: vertical swipe direction that cancels recording. + #[serde(default = "default_android_overlay_cancel_swipe_direction")] + pub android_overlay_cancel_swipe_direction: AndroidOverlayCancelSwipeDirection, + /// Android: floating overlay control diameter in dp. + #[serde(default = "default_android_overlay_size_dp")] + pub android_overlay_size_dp: u32, } fn default_local_asr_model() -> String { @@ -898,6 +932,18 @@ struct UserPreferencesWire { marketplace_base_url: String, #[serde(default)] marketplace_dev_login: String, + #[serde(default = "default_android_insert_strategy")] + android_insert_strategy: AndroidInsertStrategy, + #[serde(default = "default_android_overlay_trigger")] + android_overlay_trigger: AndroidOverlayTrigger, + #[serde(default = "default_android_overlay_activation_mode")] + android_overlay_activation_mode: AndroidOverlayActivationMode, + #[serde(default = "default_android_overlay_left_swipe_action")] + android_overlay_left_swipe_action: AndroidOverlayLeftSwipeAction, + #[serde(default = "default_android_overlay_cancel_swipe_direction")] + android_overlay_cancel_swipe_direction: AndroidOverlayCancelSwipeDirection, + #[serde(default = "default_android_overlay_size_dp")] + android_overlay_size_dp: u32, } impl Default for UserPreferencesWire { @@ -965,6 +1011,12 @@ impl Default for UserPreferencesWire { audio_recording_max_entries: prefs.audio_recording_max_entries, marketplace_base_url: prefs.marketplace_base_url, marketplace_dev_login: prefs.marketplace_dev_login, + android_insert_strategy: prefs.android_insert_strategy, + android_overlay_trigger: prefs.android_overlay_trigger, + android_overlay_activation_mode: prefs.android_overlay_activation_mode, + android_overlay_left_swipe_action: prefs.android_overlay_left_swipe_action, + android_overlay_cancel_swipe_direction: prefs.android_overlay_cancel_swipe_direction, + android_overlay_size_dp: prefs.android_overlay_size_dp, } } } @@ -1061,6 +1113,16 @@ impl<'de> Deserialize<'de> for UserPreferences { audio_recording_max_entries: wire.audio_recording_max_entries, marketplace_base_url: wire.marketplace_base_url, marketplace_dev_login: wire.marketplace_dev_login, + android_insert_strategy: normalize_android_insert_strategy( + wire.android_insert_strategy, + ), + android_overlay_trigger: wire.android_overlay_trigger.normalized(), + android_overlay_activation_mode: wire.android_overlay_activation_mode, + android_overlay_left_swipe_action: wire.android_overlay_left_swipe_action, + android_overlay_cancel_swipe_direction: wire.android_overlay_cancel_swipe_direction, + android_overlay_size_dp: normalize_android_overlay_size_dp( + wire.android_overlay_size_dp, + ), }) } } @@ -1781,6 +1843,13 @@ impl Default for UserPreferences { audio_recording_max_entries: None, marketplace_base_url: String::new(), marketplace_dev_login: String::new(), + android_insert_strategy: default_android_insert_strategy(), + android_overlay_trigger: default_android_overlay_trigger(), + android_overlay_activation_mode: default_android_overlay_activation_mode(), + android_overlay_left_swipe_action: default_android_overlay_left_swipe_action(), + android_overlay_cancel_swipe_direction: default_android_overlay_cancel_swipe_direction( + ), + android_overlay_size_dp: default_android_overlay_size_dp(), } } } @@ -1989,6 +2058,8 @@ pub enum HotkeyAdapterKind { MacEventTap, WindowsLowLevel, Fcitx5, + /// Mobile platforms do not expose desktop global hotkey adapters. + Unavailable, } impl HotkeyAdapterKind { @@ -1997,6 +2068,7 @@ impl HotkeyAdapterKind { HotkeyAdapterKind::MacEventTap => "macOS Event Tap", HotkeyAdapterKind::WindowsLowLevel => "Windows 低层键盘 hook", HotkeyAdapterKind::Fcitx5 => "fcitx5 输入法插件", + HotkeyAdapterKind::Unavailable => "不可用", } } } @@ -2152,6 +2224,21 @@ pub struct HotkeyCapability { impl HotkeyCapability { pub fn current() -> Self { + #[cfg(mobile)] + { + return Self { + adapter: HotkeyAdapterKind::Unavailable, + available_triggers: Vec::new(), + requires_accessibility_permission: false, + supports_modifier_only_trigger: false, + supports_side_specific_modifiers: false, + explicit_fallback_available: false, + status_hint: Some( + "移动端不支持全局热键;请使用应用内录音按钮或悬浮窗(需授权)。".into(), + ), + }; + } + #[cfg(target_os = "macos")] { Self { @@ -2196,7 +2283,7 @@ impl HotkeyCapability { }; } - #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] + #[cfg(all(not(target_os = "macos"), not(target_os = "windows"), not(mobile)))] { Self { adapter: HotkeyAdapterKind::Fcitx5, @@ -2258,6 +2345,68 @@ pub struct WindowsImeStatus { pub dll_path: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct PlatformCapabilities { + pub platform: String, + pub supports_ime_input: bool, + pub supports_overlay: bool, + pub supports_desktop_hotkey: bool, + pub supports_tray: bool, + pub supports_local_asr: bool, + pub supports_in_app_dictation: bool, + pub supports_auto_update: bool, +} + +impl PlatformCapabilities { + pub fn current() -> Self { + #[cfg(target_os = "android")] + { + Self { + platform: "android".to_string(), + supports_ime_input: false, + supports_overlay: true, + supports_desktop_hotkey: false, + supports_tray: false, + supports_local_asr: false, + supports_in_app_dictation: true, + supports_auto_update: false, + } + } + + #[cfg(all( + any(target_os = "android", target_os = "ios"), + not(target_os = "android") + ))] + { + Self { + platform: "mobile".to_string(), + supports_ime_input: false, + supports_overlay: false, + supports_desktop_hotkey: false, + supports_tray: false, + supports_local_asr: false, + supports_in_app_dictation: false, + supports_auto_update: false, + } + } + + #[cfg(not(any(target_os = "android", target_os = "ios")))] + { + Self { + platform: "desktop".to_string(), + supports_ime_input: cfg!(target_os = "windows"), + supports_overlay: true, + supports_desktop_hotkey: true, + supports_tray: true, + supports_local_asr: cfg!(any(target_os = "macos", target_os = "windows")), + supports_in_app_dictation: false, + supports_auto_update: true, + } + } + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub enum HotkeyStatusState { diff --git a/openless-all/app/src-tauri/tauri.android.conf.json b/openless-all/app/src-tauri/tauri.android.conf.json new file mode 100644 index 00000000..05ae8f0a --- /dev/null +++ b/openless-all/app/src-tauri/tauri.android.conf.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "identifier": "com.openless.app", + "app": { + "windows": [ + { + "label": "main", + "title": "OpenLess", + "width": 1240, + "height": 800, + "minWidth": 360, + "minHeight": 640, + "resizable": true, + "decorations": true, + "visible": true + } + ] + } +} diff --git a/openless-all/app/src-tauri/tauri.conf.json b/openless-all/app/src-tauri/tauri.conf.json index 5e79db90..d00d7cf8 100644 --- a/openless-all/app/src-tauri/tauri.conf.json +++ b/openless-all/app/src-tauri/tauri.conf.json @@ -128,6 +128,9 @@ "infoPlist": "Info.plist", "entitlements": "Entitlements.plist" }, + "android": { + "minSdkVersion": 26 + }, "windows": { "nsis": { "installMode": "perMachine", diff --git a/openless-all/app/src/App.tsx b/openless-all/app/src/App.tsx index 919f56dc..39f02c9d 100644 --- a/openless-all/app/src/App.tsx +++ b/openless-all/app/src/App.tsx @@ -9,9 +9,12 @@ import { checkMicrophonePermission, getHotkeyStatus, getSettings, + getPlatformCapabilities, handleWindowHotkeyEvent, isTauri, + qaWindowDismiss, } from './lib/ipc'; +import type { PlatformCapabilities } from './lib/types'; import { isWindowHotkeyKeyboardCandidate, windowMouseHotkeyCode, @@ -31,6 +34,7 @@ interface AppProps { } type Gate = 'onboarding' | 'ready'; +const ANDROID_SETUP_WIZARD_COMPLETE_KEY = 'openless.androidSetupWizardComplete'; export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, forcedOs }: AppProps) { if (isCapsule) { @@ -49,7 +53,14 @@ export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, force const os = forcedOs ?? detectOS(); // Windows 启动不应被权限探测阻塞首屏。 const [gate, setGate] = useState('ready'); - + const [platformCaps, setPlatformCaps] = useState(null); + const [mobileQaOpen, setMobileQaOpen] = useState(false); + const completeOnboarding = () => { + if (platformCaps?.platform === 'android') { + localStorage.setItem(ANDROID_SETUP_WIZARD_COMPLETE_KEY, '1'); + } + setGate('ready'); + }; useEffect(() => { applyAppTheme(readAppTheme()); const syncTheme = (event: StorageEvent) => { @@ -60,6 +71,58 @@ export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, force return () => window.removeEventListener('storage', syncTheme); }, []); + useEffect(() => { + if (!isTauri) return; + void getPlatformCapabilities().then(setPlatformCaps); + }, []); + + useEffect(() => { + if (!isTauri || platformCaps?.platform !== 'android') return; + let unlistenState: (() => void) | undefined; + let unlistenDismiss: (() => void) | undefined; + let cancelled = false; + (async () => { + try { + const { listen } = await import('@tauri-apps/api/event'); + const stateHandle = await listen('qa:state', () => { + console.info('[qa] android qa:state received; opening embedded panel'); + setMobileQaOpen(true); + }); + const dismissHandle = await listen('qa:dismiss', () => { + console.info('[qa] android qa:dismiss received; closing embedded panel'); + setMobileQaOpen(false); + }); + if (cancelled) { + stateHandle(); + dismissHandle(); + } else { + unlistenState = stateHandle; + unlistenDismiss = dismissHandle; + } + } catch (error) { + console.warn('[qa] mobile route listener setup failed', error); + } + })(); + return () => { + cancelled = true; + unlistenState?.(); + unlistenDismiss?.(); + }; + }, [platformCaps?.platform]); + + useEffect(() => { + if (!mobileQaOpen || platformCaps?.platform !== 'android') return; + window.history.pushState({ openlessQa: true }, '', window.location.href); + const onPopState = () => { + setMobileQaOpen(false); + void qaWindowDismiss().catch(error => console.warn('[qa] mobile back dismiss failed', error)); + }; + window.addEventListener('popstate', onPopState); + return () => { + window.removeEventListener('popstate', onPopState); + }; + }, [mobileQaOpen, platformCaps?.platform]); + useEffect(() => { if (!isTauri) return; let cancelled = false; @@ -105,13 +168,30 @@ export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, force if (!isTauri) return; let cancelled = false; - if (os === 'win') { - // 超时保护:50 次 × 200ms = 10s。hotkey hook 永远 starting(被反作弊 / EDR - // / UAC 拦)时不让 UI 死锁灰屏,过 10s 强 setGate('ready') 让用户进 - // Permissions 页看 hotkey_status.lastError 处理。详见 issue #163。 - const POLL_INTERVAL_MS = 200; - const POLL_MAX_ATTEMPTS = 50; - const pollHotkeyStatus = async () => { + void (async () => { + const caps = await getPlatformCapabilities(); + if (cancelled) return; + + if (caps.platform === 'android') { + if (localStorage.getItem(ANDROID_SETUP_WIZARD_COMPLETE_KEY) !== '1') { + setGate('onboarding'); + return; + } + const m = await checkMicrophonePermission(); + if (cancelled) return; + // notDetermined is non-blocking on Android — show grant flow in-app instead + // of trapping users on onboarding while JNI/runtime permission is pending. + const blocked = m === 'denied' || m === 'restricted'; + setGate(blocked ? 'onboarding' : 'ready'); + return; + } + + if (os === 'win') { + // 超时保护:50 次 × 200ms = 10s。hotkey hook 永远 starting(被反作弊 / EDR + // / UAC 拦)时不让 UI 死锁灰屏,过 10s 强 setGate('ready') 让用户进 + // Permissions 页看 hotkey_status.lastError 处理。详见 issue #163。 + const POLL_INTERVAL_MS = 200; + const POLL_MAX_ATTEMPTS = 50; let attempts = 0; while (!cancelled && attempts < POLL_MAX_ATTEMPTS) { attempts += 1; @@ -129,19 +209,9 @@ export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, force ); setGate('ready'); } - }; - void pollHotkeyStatus().catch(error => { - console.warn('[startup] hotkey status polling failed', error); - if (!cancelled) { - setGate('ready'); - } - }); - return () => { - cancelled = true; - }; - } + return; + } - (async () => { const [a, m] = await Promise.all([ checkAccessibilityPermission(), checkMicrophonePermission(), @@ -150,7 +220,13 @@ export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, force const aOk = a === 'granted' || a === 'notApplicable'; const mOk = m === 'granted' || m === 'notApplicable'; setGate(aOk && mOk ? 'ready' : 'onboarding'); - })(); + })().catch(error => { + console.warn('[startup] permission gate failed', error); + if (!cancelled) { + setGate('ready'); + } + }); + return () => { cancelled = true; }; @@ -191,8 +267,25 @@ export function App({ isCapsule, isQa, isLessComputer, isLessComputerGlow, force return ( - {gate === 'onboarding' ? setGate('ready')} /> : } - {gate === 'ready' && } + {platformCaps?.platform === 'android' && ( +
+ { + setMobileQaOpen(false); + if (window.history.state?.openlessQa === true) { + window.history.back(); + } + }} + /> +
+ )} + {!mobileQaOpen && (gate === 'onboarding' ? ( + + ) : ( + + ))} + {gate === 'ready' && platformCaps?.supportsAutoUpdate === true && }
); } diff --git a/openless-all/app/src/components/AudioCue.tsx b/openless-all/app/src/components/AudioCue.tsx index 5ad4f2b2..e3f5f4ad 100644 --- a/openless-all/app/src/components/AudioCue.tsx +++ b/openless-all/app/src/components/AudioCue.tsx @@ -1,9 +1,9 @@ // 录音提示音:监听 capsule:state 事件,在"开始录音"边沿播放合成提示音。 // 独立组件,不依赖胶囊窗口显示——Linux 上胶囊隐藏也能正常工作。 -// 全平台通用,在 FloatingShellBody 中渲染。 +// Android Web Audio 输出会触发部分设备的录音输入路由切换,移动端禁用。 import { useEffect, useRef } from 'react'; -import { isTauri } from '../lib/ipc'; +import { isAndroid, isTauri } from '../lib/ipc'; import { playRecordStartCue, primeAudioCue, stopAudioCue } from '../lib/audioCue'; import type { CapsuleState, UserPreferences } from '../lib/types'; @@ -18,10 +18,11 @@ interface CapsulePayload { export function AudioCueListener() { const audioCueEnabledRef = useRef(true); const prevStateRef = useRef('idle' as CapsuleState); + const audioCueRuntimeEnabled = !isAndroid(); // 读取设置(默认开启) useEffect(() => { - if (!isTauri) return; + if (!isTauri || !audioCueRuntimeEnabled) return; let cancelled = false; (async () => { try { @@ -33,11 +34,11 @@ export function AudioCueListener() { } })(); return () => { cancelled = true; }; - }, []); + }, [audioCueRuntimeEnabled]); // 监听设置变更 useEffect(() => { - if (!isTauri) return; + if (!isTauri || !audioCueRuntimeEnabled) return; let unlisten: (() => void) | undefined; let cancelled = false; (async () => { @@ -48,17 +49,17 @@ export function AudioCueListener() { }).then(fn => { if (!cancelled) unlisten = fn; }).catch(() => {}); })(); return () => { cancelled = true; unlisten?.(); }; - }, []); + }, [audioCueRuntimeEnabled]); // 预热 AudioContext useEffect(() => { - if (!isTauri) return; + if (!isTauri || !audioCueRuntimeEnabled) return; primeAudioCue(); - }, []); + }, [audioCueRuntimeEnabled]); // 监听 capsule 状态边沿 useEffect(() => { - if (!isTauri) return; + if (!isTauri || !audioCueRuntimeEnabled) return; let unlisten: (() => void) | undefined; let cancelled = false; (async () => { @@ -75,7 +76,7 @@ export function AudioCueListener() { }).then(fn => { if (!cancelled) unlisten = fn; }).catch(() => {}); })(); return () => { cancelled = true; unlisten?.(); }; - }, []); + }, [audioCueRuntimeEnabled]); return null; } diff --git a/openless-all/app/src/components/AutoUpdate.tsx b/openless-all/app/src/components/AutoUpdate.tsx index c6a5f096..0d0cfa0e 100644 --- a/openless-all/app/src/components/AutoUpdate.tsx +++ b/openless-all/app/src/components/AutoUpdate.tsx @@ -2,30 +2,21 @@ // 状态机 + 对话框 UI。两边各自调用 useAutoUpdate(),dialog 渲染条件相同。 // // 渠道感知:check 不再走 plugin-updater 的 JS check()(它只看 tauri.conf 配的 -// Stable manifest URL),改为 invoke('app_check_update_with_channel')。 +// Stable manifest URL),改为 appCheckUpdateWithChannel()(ipc 层按 +// supportsAutoUpdate 在 Android 上 no-op)。 // Rust 那边按 prefs.update_channel 决定 manifest URL;返回的 metadata 直接 // `new Update(metadata)` 复用 plugin 的 download / install / close 实现, // 我们不重复造下载和签名校验。 import { useEffect, useRef, useState } from 'react'; -import { invoke } from '@tauri-apps/api/core'; import type { DownloadEvent } from '@tauri-apps/plugin-updater'; import { Update } from '@tauri-apps/plugin-updater'; import { useTranslation } from 'react-i18next'; -import { isTauri, restartApp, type UpdateChannel } from '../lib/ipc'; +import { appCheckUpdateWithChannel, isTauri, restartApp, type AppUpdateMetadata, type UpdateChannel } from '../lib/ipc'; import { Btn } from '../pages/_atoms'; const UPDATE_CHECK_TIMEOUT_MS = 15_000; -interface AppUpdateMetadata { - rid: number; - currentVersion: string; - version: string; - date?: string | null; - body?: string | null; - rawJson: Record; -} - export type UpdateStatus = | 'idle' | 'checking' @@ -102,10 +93,10 @@ export function useAutoUpdate(): UseAutoUpdate { } // Rust 侧按 update_channel 拼 manifest URL:Stable → tauri.conf 默认; // Beta → fetch_latest_beta_release 拼出 -beta manifest URL 后再 check。 - const metadata = await invoke('app_check_update_with_channel', { - timeoutMs: UPDATE_CHECK_TIMEOUT_MS, - channel: channel ?? null, - }); + const metadata = await appCheckUpdateWithChannel( + UPDATE_CHECK_TIMEOUT_MS, + channel ?? null, + ); if (!metadata) { setStatus('none'); return; diff --git a/openless-all/app/src/components/AutoUpdateGate.tsx b/openless-all/app/src/components/AutoUpdateGate.tsx index a1953dac..7ff1b6b7 100644 --- a/openless-all/app/src/components/AutoUpdateGate.tsx +++ b/openless-all/app/src/components/AutoUpdateGate.tsx @@ -2,8 +2,10 @@ // 受 prefs.autoUpdateCheck 开关控制;关闭时只走 Settings → 关于 的手动按钮。 // 找到新版本时直接挂 UpdateDialog;不弹自定义通知,沿用既有 dialog 视觉。 -import { useEffect, useRef } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { isDialogStatus, UpdateDialog, useAutoUpdate } from './AutoUpdate'; +import { getPlatformCapabilities } from '../lib/ipc'; +import type { PlatformCapabilities } from '../lib/types'; import { useHotkeySettings } from '../state/HotkeySettingsContext'; const AUTO_CHECK_INTERVAL_MS = 60 * 60 * 1000; @@ -12,7 +14,12 @@ const STARTUP_DELAY_MS = 4_000; export function AutoUpdateGate() { const { prefs } = useHotkeySettings(); const u = useAutoUpdate(); - const enabled = prefs?.autoUpdateCheck ?? true; + const [platformCaps, setPlatformCaps] = useState(null); + const enabled = (prefs?.autoUpdateCheck ?? true) && platformCaps?.supportsAutoUpdate === true; + + useEffect(() => { + void getPlatformCapabilities().then(setPlatformCaps); + }, []); // 用 ref 保持 tick 闭包始终读到最新的 useAutoUpdate 返回值。 // 之前直接捕获 `u` 会让 60min interval 触发时读旧 status 闭包——例如用户已经 @@ -43,6 +50,8 @@ export function AutoUpdateGate() { }; }, [enabled]); + if (platformCaps?.supportsAutoUpdate !== true) return null; + if (!isDialogStatus(u.status)) return null; return ( (); const [providerPromptOpen, setProviderPromptOpen] = useState(false); const [hotkeyModePromptOpen, setHotkeyModePromptOpen] = useState(false); + const mobileLayout = useMobileLayout(); // tab 切换的 cross-fade:旧页 blur+fade out(180ms),结束后挂载新页(走 ol-page-slide enter)。 // displayTab 是实际渲染的 tab,currentTab 是用户点中的目标 tab。 @@ -96,12 +98,17 @@ function FloatingShellBody({ os, initialTab, initialSettings }: { os: OS; initia [t], ); const Page = (NAV.find((n) => n.id === displayTab) ?? NAV[0]).cmp; + const activeNav = NAV.find(n => n.id === currentTab) ?? NAV[0]; // sidebar nav 滑动指示器:测量当前 active button 的 offsetTop / height, // 用一个 absolute pill 平滑滑过去,而不是每个按钮各自瞬切背景色。 const navItemRefs = useRef>([]); const [pillRect, setPillRect] = useState<{ top: number; height: number } | null>(null); useLayoutEffect(() => { + if (mobileLayout) { + setPillRect(null); + return; + } if (settingsOpen) { setPillRect(null); return; @@ -114,11 +121,13 @@ function FloatingShellBody({ os, initialTab, initialSettings }: { os: OS; initia const el = navItemRefs.current[idx]; if (!el) return; setPillRect({ top: el.offsetTop, height: el.offsetHeight }); - }, [currentTab, settingsOpen, NAV]); + }, [currentTab, settingsOpen, NAV, mobileLayout]); useEffect(() => { let cancelled = false; (async () => { + const caps = await getPlatformCapabilities(); + if (cancelled || caps.platform === 'android') return; const credentials = await getCredentials(); const promptDeferredValue = window.sessionStorage.getItem(PROVIDER_SETUP_PROMPT_DEFERRED_KEY); if (!cancelled && shouldShowProviderSetupPrompt(credentials, promptDeferredValue)) { @@ -131,11 +140,18 @@ function FloatingShellBody({ os, initialTab, initialSettings }: { os: OS; initia }, []); useEffect(() => { - const acknowledgedValue = window.localStorage.getItem(HOTKEY_MODE_MIGRATION_ACK_KEY); - const deferredValue = window.sessionStorage.getItem(HOTKEY_MODE_MIGRATION_DEFERRED_KEY); - if (shouldShowHotkeyModeMigrationPrompt(acknowledgedValue, deferredValue)) { - setHotkeyModePromptOpen(true); - } + let cancelled = false; + void getPlatformCapabilities().then((caps) => { + if (cancelled || caps.platform === 'android') return; + const acknowledgedValue = window.localStorage.getItem(HOTKEY_MODE_MIGRATION_ACK_KEY); + const deferredValue = window.sessionStorage.getItem(HOTKEY_MODE_MIGRATION_DEFERRED_KEY); + if (shouldShowHotkeyModeMigrationPrompt(acknowledgedValue, deferredValue)) { + setHotkeyModePromptOpen(true); + } + }); + return () => { + cancelled = true; + }; }, []); // 之前监听的 NAVIGATE_LOCAL_ASR_EVENT 已无意义——「模型设置」独立 tab 已下线, @@ -191,21 +207,45 @@ function FloatingShellBody({ os, initialTab, initialSettings }: { os: OS; initia style={{ flex: 1, minHeight: 0, display: 'flex', + flexDirection: mobileLayout ? 'column' : 'row', background: 'transparent', overflow: 'hidden', position: 'relative', zIndex: 1, }}> - {/* Sidebar — 透明地坐在外层磨砂底板上,让 LOGO/导航/快捷键/BETA/footer 共用同一片磨砂玻璃 */} - + )} {/* Main content — Linux 禁用透明窗口后使用不透明面;其他平台保留玻璃层。 */} -
+
+ + {mobileLayout && ( + + )}
{/* Settings modal — rendered inside this window */} @@ -397,6 +467,9 @@ function FloatingShellBody({ os, initialTab, initialSettings }: { os: OS; initia height: 26px; border-radius: 8px; box-shadow: none; + box-sizing: border-box; + padding: 3px; + object-fit: contain; } .ol-aura-sidebar-brand-title { font-size: 14px; @@ -474,7 +547,97 @@ function FloatingShellBody({ os, initialTab, initialSettings }: { os: OS; initia box-shadow: none; } .ol-aura-console-main { - border-radius: var(--ol-panel-radius); + border-radius: ${mobileLayout ? '0' : 'var(--ol-panel-radius)'}; + } + .ol-aura-mobile-topbar { + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: calc(10px + env(safe-area-inset-top, 0px)) 14px 10px; + border-bottom: 1px solid var(--ol-sidebar-border); + background: var(--ol-sidebar-bg); + } + .ol-aura-mobile-brand { + min-width: 0; + display: flex; + align-items: center; + gap: 10px; + } + .ol-aura-mobile-brand-mark { + width: 30px; + height: 30px; + border-radius: 8px; + flex-shrink: 0; + box-sizing: border-box; + padding: 3px; + object-fit: contain; + } + .ol-aura-mobile-brand-title { + font-size: 14px; + font-weight: 700; + color: var(--ol-ink); + line-height: 1.15; + } + .ol-aura-mobile-brand-section { + margin-top: 2px; + font-size: 11px; + color: var(--ol-ink-4); + font-family: var(--ol-font-mono); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .ol-aura-mobile-settings { + width: 36px; + height: 36px; + flex-shrink: 0; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 12px; + color: var(--ol-ink-3); + background: var(--ol-sidebar-settings-bg); + border: 1px solid var(--ol-sidebar-settings-border); + } + .ol-aura-mobile-settings-active { + color: var(--ol-ink); + background: var(--ol-sidebar-settings-active-bg); + } + .ol-aura-mobile-nav { + flex-shrink: 0; + display: grid; + grid-template-columns: repeat(${NAV.length}, minmax(0, 1fr)); + gap: 2px; + padding: 7px 8px calc(7px + env(safe-area-inset-bottom, 0px)); + border-top: 1px solid var(--ol-sidebar-border); + background: var(--ol-sidebar-bg); + } + .ol-aura-mobile-nav-btn { + min-width: 0; + height: 50px; + display: inline-flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 4px; + border-radius: 12px; + color: var(--ol-ink-4); + font-size: 10px; + font-weight: 600; + line-height: 1.1; + } + .ol-aura-mobile-nav-btn span { + max-width: 100%; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } + .ol-aura-mobile-nav-btn-active { + color: var(--ol-ink); + background: var(--ol-sidebar-pill-bg); + border: 1px solid var(--ol-sidebar-pill-border); } .ol-nav-btn.ol-nav-btn-active { color: var(--ol-ink); diff --git a/openless-all/app/src/components/Onboarding.tsx b/openless-all/app/src/components/Onboarding.tsx index 2d23ed21..bb518005 100644 --- a/openless-all/app/src/components/Onboarding.tsx +++ b/openless-all/app/src/components/Onboarding.tsx @@ -1,26 +1,280 @@ -// Onboarding.tsx — 首次运行权限引导。 -// -// 触发条件:App.tsx 启动检查 accessibility + microphone,任一未授权则渲染本组件而非主 Shell。 -// 与 Swift `Sources/OpenLessApp/Onboarding/` 同语义,但简化为单页三步。 +// Onboarding.tsx — first-run permission and service setup. -import { useEffect, useRef, useState } from 'react'; +import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react'; import { useTranslation } from 'react-i18next'; +import { AndroidPermissionsPanel } from '@android/components/AndroidPermissionsPanel'; +import { checkAndroidMicrophoneAccess, requestAndroidMicrophoneAccess } from '@android/lib/androidMicrophonePermission'; import { checkAccessibilityPermission, checkMicrophonePermission, + getPlatformCapabilities, openSystemSettings, requestAccessibilityPermission, requestMicrophonePermission, } from '../lib/ipc'; import { getHotkeyTriggerLabel } from '../lib/hotkey'; -import type { PermissionStatus } from '../lib/types'; +import type { PermissionStatus, PlatformCapabilities } from '../lib/types'; import { useHotkeySettings } from '../state/HotkeySettingsContext'; +import { ProvidersSection } from '../pages/settings/ProvidersSection'; interface OnboardingProps { onComplete: () => void; } +type AndroidStepId = + | 'microphone' + | 'accessibility' + | 'overlayPermission' + | 'overlayConfig' + | 'asr' + | 'llm'; + export function Onboarding({ onComplete }: OnboardingProps) { + const { t } = useTranslation(); + const [platformCaps, setPlatformCaps] = useState(null); + + useEffect(() => { + void getPlatformCapabilities().then(setPlatformCaps); + }, []); + + if (!platformCaps) { + return ; + } + + if (platformCaps.platform === 'android') { + return ; + } + + return ; +} + +function AndroidOnboarding({ onComplete }: OnboardingProps) { + const { t } = useTranslation(); + const [stepIndex, setStepIndex] = useState(0); + + const steps = useMemo>( + () => [ + { + id: 'microphone', + title: t('onboarding.androidSteps.microphoneTitle'), + desc: t('onboarding.androidSteps.microphoneDesc'), + }, + { + id: 'accessibility', + title: t('onboarding.androidSteps.accessibilityTitle'), + desc: t('onboarding.androidSteps.accessibilityDesc'), + }, + { + id: 'overlayPermission', + title: t('onboarding.androidSteps.overlayPermissionTitle'), + desc: t('onboarding.androidSteps.overlayPermissionDesc'), + }, + { + id: 'overlayConfig', + title: t('onboarding.androidSteps.overlayConfigTitle'), + desc: t('onboarding.androidSteps.overlayConfigDesc'), + }, + { + id: 'asr', + title: t('onboarding.androidSteps.asrTitle'), + desc: t('onboarding.androidSteps.asrDesc'), + }, + { + id: 'llm', + title: t('onboarding.androidSteps.llmTitle'), + desc: t('onboarding.androidSteps.llmDesc'), + }, + ], + [t], + ); + + const current = steps[stepIndex] ?? steps[0]; + const isFirst = stepIndex === 0; + const isLast = stepIndex === steps.length - 1; + + const goNext = () => { + if (isLast) { + onComplete(); + return; + } + setStepIndex((value) => Math.min(value + 1, steps.length - 1)); + }; + + return ( + +
+ + +
+ {steps.map((step, index) => ( +
+ ))} +
+ +
+
+
+ {t('onboarding.androidStepCounter', { current: stepIndex + 1, total: steps.length })} +
+
{current.title}
+
+ {current.desc} +
+
+ + +
+ +
+ + +
+ + +
+ + ); +} + +function AndroidStepContent({ step }: { step: AndroidStepId }) { + if (step === 'microphone') { + return ; + } + if (step === 'accessibility') { + return ; + } + if (step === 'overlayPermission') { + return ; + } + if (step === 'overlayConfig') { + return ; + } + if (step === 'asr') { + return ; + } + return ; +} + +function AndroidMicrophoneStep() { + const { t } = useTranslation(); + const [status, setStatus] = useState('notDetermined'); + const [busy, setBusy] = useState(false); + + const refresh = async () => { + setStatus(await checkAndroidMicrophoneAccess()); + }; + + useEffect(() => { + void refresh(); + const id = window.setInterval(refresh, 3000); + const onFocus = () => { void refresh(); }; + window.addEventListener('focus', onFocus); + return () => { + window.clearInterval(id); + window.removeEventListener('focus', onFocus); + }; + }, []); + + const request = async () => { + setBusy(true); + try { + if (status === 'denied' || status === 'restricted') { + await openSystemSettings('microphone'); + } else { + setStatus(await requestAndroidMicrophoneAccess()); + } + await refresh(); + } finally { + setBusy(false); + } + }; + + const granted = status === 'granted' || status === 'notApplicable'; + return ( + +
+
+
{t('onboarding.micTitle')}
+
+ {t('onboarding.micDesc')} +
+
+ +
+ +
+ ); +} + +function DesktopOnboarding({ + onComplete, + platformCaps: _platformCaps, +}: OnboardingProps & { platformCaps: PlatformCapabilities }) { const { t } = useTranslation(); const [accessibility, setAccessibility] = useState('notDetermined'); const [microphone, setMicrophone] = useState('notDetermined'); @@ -28,6 +282,8 @@ export function Onboarding({ onComplete }: OnboardingProps) { const refreshTimeoutRef = useRef(null); const { capability } = useHotkeySettings(); + const requiresAccessibility = !!capability?.requiresAccessibilityPermission; + const refresh = async () => { const [a, m] = await Promise.all([ checkAccessibilityPermission(), @@ -35,7 +291,9 @@ export function Onboarding({ onComplete }: OnboardingProps) { ]); setAccessibility(a); setMicrophone(m); - if ((a === 'granted' || a === 'notApplicable') && (m === 'granted' || m === 'notApplicable')) { + const aOk = !requiresAccessibility || a === 'granted' || a === 'notApplicable'; + const mOk = m === 'granted' || m === 'notApplicable'; + if (aOk && mOk) { onComplete(); } }; @@ -43,7 +301,6 @@ export function Onboarding({ onComplete }: OnboardingProps) { useEffect(() => { refresh(); const id = window.setInterval(refresh, 1000); - // 用户从系统设置切回来时立刻刷新 const onFocus = () => refresh(); window.addEventListener('focus', onFocus); return () => { @@ -51,7 +308,7 @@ export function Onboarding({ onComplete }: OnboardingProps) { window.removeEventListener('focus', onFocus); if (refreshTimeoutRef.current) clearTimeout(refreshTimeoutRef.current); }; - }, []); + }, [requiresAccessibility]); const onGrantAccessibility = async () => { setBusy(true); @@ -83,74 +340,45 @@ export function Onboarding({ onComplete }: OnboardingProps) { }; return ( -
+
-
-
- OL -
-
-
{t('onboarding.welcome')}
-
- {t('onboarding.intro')} -
-
-
+ - + {requiresAccessibility && ( + + )} -
+
{t('onboarding.footerHint')}
+ + ); +} + +function OnboardingLoading({ label }: { label: string }) { + return ( +
+ {label}
); } +function OnboardingSurface({ children }: { children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +function BrandHeader({ title, desc, compact = false }: { title: string; desc: string; compact?: boolean }) { + return ( +
+ OpenLess +
+
{title}
+
+ {desc} +
+
+
+ ); +} + +function AndroidStepCard({ children }: { children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +function StatusBadge({ granted, label }: { granted: boolean; label: string }) { + return ( + + {label} + + ); +} + interface StepProps { index: number; title: string; @@ -255,3 +571,54 @@ function PermissionStep({ index, title, desc, status, actionLabel, onAction, dis
); } + +const primaryButtonStyle = { + flex: 1, + minHeight: 42, + padding: '10px 14px', + fontSize: 13, + fontWeight: 600, + fontFamily: 'inherit', + border: 0, + borderRadius: 10, + background: 'var(--ol-ink)', + color: '#fff', + cursor: 'default', +} as const; + +const secondaryButtonStyle = { + flex: 1, + minHeight: 42, + padding: '10px 14px', + fontSize: 13, + fontWeight: 600, + fontFamily: 'inherit', + border: '0.5px solid var(--ol-line-strong)', + borderRadius: 10, + background: 'var(--ol-surface)', + color: 'var(--ol-ink-2)', + cursor: 'default', +} as const; + +const plainButtonStyle = { + width: '100%', + padding: '10px 14px', + fontSize: 12.5, + fontWeight: 500, + fontFamily: 'inherit', + border: 0, + borderRadius: 8, + background: 'transparent', + color: 'var(--ol-ink-4)', + cursor: 'default', +} as const; + +const footerHintStyle = { + marginTop: 18, + padding: '12px 14px', + borderRadius: 8, + background: 'var(--ol-surface-2)', + fontSize: 11.5, + color: 'var(--ol-ink-3)', + lineHeight: 1.6, +} as const; diff --git a/openless-all/app/src/components/SettingsModal.tsx b/openless-all/app/src/components/SettingsModal.tsx index 1609ff94..f55a682f 100644 --- a/openless-all/app/src/components/SettingsModal.tsx +++ b/openless-all/app/src/components/SettingsModal.tsx @@ -2,7 +2,7 @@ // // 重构(2026-05):原本是「外层弹窗侧栏 + 设置页内层侧栏」双层嵌套,用户点 // 「设置」还要再面对第二个侧栏。现在拍平成单层 —— 通用 / 服务 / 隐私 / 高级 / -// 个性化 / 关于 六个 tab + 帮助外链组。每个 tab 的内容见 pages/settings/。 +// 个性化 / 关于 六个 tab。每个 tab 的内容见 pages/settings/。 // // 设计原则:每个可见控件都必须可用。没有后端支撑的占位(账号 / 主题切换 等) // 不在此弹窗出现。 @@ -12,10 +12,10 @@ import { useTranslation } from 'react-i18next'; import { Icon } from './Icon'; import { SavedToast } from './SavedToast'; import { useSavedToastListener } from '../lib/savedEvent'; -import { openExternal } from '../lib/ipc'; import type { OS } from './WindowChrome'; import { GeneralTab, ServicesTab, PrivacyTab, AdvancedTab } from '../pages/settings/tabs'; import { AboutSection } from '../pages/settings/AboutSection'; +import { useMobileLayout } from '../lib/useMobileLayout'; // 稳定 tab ID(与 i18n key `modal.sections.*` 一致)。 export type SettingsSectionId = @@ -34,14 +34,8 @@ interface SettingsModalProps { interface ModalNavItem { id: string; icon: string; - external?: boolean; - href?: string; } -const HELP_URL = 'https://github.com/appergb/openless#readme'; -const RELEASE_NOTES_URL = 'https://github.com/appergb/openless/releases'; - -// 第一组:可选中的 tab;第二组:外部链接(永远不 active)。 const TAB_ITEMS: ModalNavItem[] = [ { id: 'general', icon: 'settings' }, { id: 'services', icon: 'cloud' }, @@ -49,25 +43,26 @@ const TAB_ITEMS: ModalNavItem[] = [ { id: 'advanced', icon: 'bolt' }, { id: 'about', icon: 'info' }, ]; -const LINK_ITEMS: ModalNavItem[] = [ - { id: 'helpCenter', icon: 'help', external: true, href: HELP_URL }, - { id: 'releaseNotes', icon: 'doc', external: true, href: RELEASE_NOTES_URL }, -]; export function SettingsModal({ os: _os, onClose, initialSettingsSection }: SettingsModalProps) { const { t } = useTranslation(); const [section, setSection] = useState(initialSettingsSection ?? 'general'); const savedToast = useSavedToastListener(); + const mobile = useMobileLayout(); // 与 sidebar nav 一致的滑动指示器:仅 tab 组有 pill;外链组永远不画 pill。 const tabRefs = useRef>([]); const [pillRect, setPillRect] = useState<{ top: number; height: number } | null>(null); useLayoutEffect(() => { + if (mobile) { + setPillRect(null); + return; + } const idx = TAB_ITEMS.findIndex(it => it.id === section); const el = tabRefs.current[idx]; if (!el) return; setPillRect({ top: el.offsetTop, height: el.offsetHeight }); - }, [section]); + }, [section, mobile]); return (
@@ -90,8 +85,9 @@ export function SettingsModal({ os: _os, onClose, initialSettingsSection }: Sett width: '100%', maxWidth: 920, height: '100%', - maxHeight: 620, + maxHeight: mobile ? 'none' : 620, display: 'flex', + flexDirection: mobile ? 'column' : 'row', overflow: 'hidden', animation: 'ol-modal-card-in 0.24s var(--ol-motion-spring)', position: 'relative', @@ -101,14 +97,25 @@ export function SettingsModal({ os: _os, onClose, initialSettingsSection }: Sett {/* ─── 内容区 ────────────────────────────────────────────── @@ -197,7 +189,14 @@ export function SettingsModal({ os: _os, onClose, initialSettingsSection }: Sett
+ style={{ + flex: 1, + minHeight: 0, + overflow: 'auto', + padding: mobile + ? '8px 14px calc(18px + env(safe-area-inset-bottom, 0px))' + : '10px 28px 28px', + }}> {/* key=section 让切 tab 时整块重挂载,ol-tab-fade 轻微淡入。 */}
{` .ol-aura-settings { background: var(--ol-panel-bg); - border-radius: var(--ol-shell-radius); + border-radius: ${mobile ? '0' : 'var(--ol-shell-radius)'}; border: 1px solid var(--ol-panel-border); box-shadow: var(--ol-panel-shadow); } .ol-aura-settings-rail { - padding: 20px 14px; - gap: 16px; + padding: ${mobile ? 'calc(10px + env(safe-area-inset-top, 0px)) 10px 8px' : '20px 14px'}; + gap: ${mobile ? '8px' : '16px'}; background: var(--ol-settings-rail-bg); - border-right: 1px solid var(--ol-settings-rail-border); + border-right: ${mobile ? '0' : '1px solid var(--ol-settings-rail-border)'}; + border-bottom: ${mobile ? '1px solid var(--ol-settings-rail-border)' : '0'}; } .ol-aura-settings-pill { background: var(--ol-sidebar-pill-bg); @@ -231,7 +231,7 @@ export function SettingsModal({ os: _os, onClose, initialSettingsSection }: Sett box-shadow: none; } .ol-aura-settings-nav-btn { - padding: 7px 10px; + padding: ${mobile ? '8px 11px' : '7px 10px'}; border-radius: 12px; border: 0; background: transparent; @@ -243,12 +243,19 @@ export function SettingsModal({ os: _os, onClose, initialSettingsSection }: Sett z-index: 1; transition: color 0.16s var(--ol-motion-quick), background 0.16s var(--ol-motion-quick); } + .ol-aura-settings-nav-btn.ol-nav-btn-active { + background: ${mobile ? 'var(--ol-sidebar-pill-bg)' : 'transparent'}; + border: ${mobile ? '1px solid var(--ol-sidebar-pill-border)' : '0'}; + } .ol-aura-settings-links { display: flex; - flex-direction: column; + flex-direction: ${mobile ? 'row' : 'column'}; gap: 1px; - padding-top: 10px; - border-top: 1px solid var(--ol-settings-links-border); + padding-top: ${mobile ? '0' : '10px'}; + padding-left: ${mobile ? '8px' : '0'}; + border-top: ${mobile ? '0' : '1px solid var(--ol-settings-links-border)'}; + border-left: ${mobile ? '1px solid var(--ol-settings-links-border)' : '0'}; + overflow-x: ${mobile ? 'auto' : 'visible'}; } .ol-aura-settings-content { background: var(--ol-settings-content-bg); @@ -262,8 +269,8 @@ export function SettingsModal({ os: _os, onClose, initialSettingsSection }: Sett background: var(--ol-settings-close-hover-bg); } .ol-aura-settings-title { - padding: 24px 28px 10px; - font-size: 22px; + padding: ${mobile ? '16px 48px 8px 16px' : '24px 28px 10px'}; + font-size: ${mobile ? '20px' : '22px'}; font-weight: 600; letter-spacing: -0.02em; font-family: var(--ol-font-display); diff --git a/openless-all/app/src/components/WindowChrome.tsx b/openless-all/app/src/components/WindowChrome.tsx index e7068354..3a2d25bf 100644 --- a/openless-all/app/src/components/WindowChrome.tsx +++ b/openless-all/app/src/components/WindowChrome.tsx @@ -1,6 +1,6 @@ import { type CSSProperties, type ReactNode, useCallback, useEffect, useRef, useState } from 'react'; -export type OS = 'mac' | 'win' | 'linux'; +export type OS = 'mac' | 'win' | 'linux' | 'android'; export function detectOS(): OS { if (typeof navigator === 'undefined') return 'mac'; @@ -9,6 +9,7 @@ export function detectOS(): OS { ).userAgentData?.platform ?? ''; const hints = `${navigator.userAgent || ''} ${navigator.platform || ''} ${uaDataPlatform}`; if (/Mac|iPhone|iPad|iPod/.test(hints)) return 'mac'; + if (/Android/i.test(hints)) return 'android'; if (/Windows|Win32|Win64/.test(hints)) return 'win'; if (/Linux|X11|Wayland/.test(hints)) return 'linux'; return 'mac'; @@ -33,8 +34,8 @@ export function WindowChrome({ }: WindowChromeProps) { // Windows: decorations:true 时外层不画圆角/边框/阴影/标题栏,避免与原生窗口重叠。 // Linux: decorations:false 时外层画 14px 圆角 + 自定义标题栏。 - const shellRadius = os === 'mac' ? 0 : os === 'win' ? 0 : 14; - const consoleRadius = os === 'mac' ? 20 : os === 'win' ? WIN_CONSOLE_RADIUS : 14; + const shellRadius = os === 'mac' ? 0 : os === 'win' || os === 'android' ? 0 : 14; + const consoleRadius = os === 'mac' ? 20 : os === 'win' ? WIN_CONSOLE_RADIUS : os === 'android' ? 0 : 14; const titlebarHeight = os === 'mac' ? MAC_TITLEBAR_HEIGHT : os === 'linux' ? LINUX_TITLEBAR_HEIGHT : 0; // macOS / Windows 共用半透明玻璃 background + backdropFilter。 diff --git a/openless-all/app/src/components/ui/Row.tsx b/openless-all/app/src/components/ui/Row.tsx index a2723017..867693ac 100644 --- a/openless-all/app/src/components/ui/Row.tsx +++ b/openless-all/app/src/components/ui/Row.tsx @@ -1,6 +1,7 @@ // Row — two-column row used in the Settings modal sub-sections. import type { ReactNode } from 'react'; +import { useMobileLayout } from '../../lib/useMobileLayout'; interface RowProps { label: string; @@ -9,13 +10,14 @@ interface RowProps { } export function Row({ label, desc, children }: RowProps) { + const mobile = useMobileLayout(); return ( -
-
+
+
{label}
{desc &&
{desc}
}
-
{children}
+
{children}
); } diff --git a/openless-all/app/src/components/ui/SelectLite.tsx b/openless-all/app/src/components/ui/SelectLite.tsx index 4822e50a..ec3ebebe 100644 --- a/openless-all/app/src/components/ui/SelectLite.tsx +++ b/openless-all/app/src/components/ui/SelectLite.tsx @@ -29,6 +29,7 @@ import { } from 'react'; import { createPortal } from 'react-dom'; import { Icon } from '../Icon'; +import { useMobileLayout } from '../../lib/useMobileLayout'; export interface SelectOption { value: string; @@ -81,6 +82,7 @@ export function SelectLite({ ariaLabel, onOpenChange, }: SelectLiteProps) { + const mobile = useMobileLayout(); const [open, setOpen] = useState(false); // leaving 让 popover 在卸载前播完 exit keyframe(用户报"没有收缩动画"——之前直接 unmount) const [leaving, setLeaving] = useState(false); @@ -264,6 +266,14 @@ export function SelectLite({ const triggerStyle: CSSProperties = { ...DEFAULT_TRIGGER_STYLE, ...style, + boxSizing: 'border-box', + ...(mobile + ? { + width: style?.width ?? '100%', + minWidth: 0, + maxWidth: '100%', + } + : null), opacity: disabled ? 0.5 : 1, cursor: disabled ? 'not-allowed' : 'default', }; diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 64a58c4a..b83e2210 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -49,6 +49,11 @@ export const en: typeof zhCN = { emptyTitle: 'Press {{recordHotkey}} to ask', emptyDesc: 'Select text in any app, press {{recordHotkey}} once to start recording, press it again to submit. Answers appear here. You can ask follow-up questions in the same panel.', recordingHint: 'Recording… press {{recordHotkey}} again to submit', + mobileRecordLabel: 'record button', + mobileRecordStart: 'Start recording', + mobileRecordStop: 'Stop and submit', + composerPlaceholder: 'Type a question. Enter to send, Shift+Enter for a new line', + composerSend: 'Send', statusIdle: 'Press {{recordHotkey}} to ask', statusRecording: 'Recording', statusThinking: 'Thinking', @@ -223,6 +228,28 @@ export const en: typeof zhCN = { actionRequestMic: 'Request access', accessibilityHint: 'After granting, you must **fully quit OpenLess** and reopen it (a macOS TCC requirement).', footerHint: 'This onboarding closes automatically once both permissions are granted. If it persists, quit OpenLess from the menu bar and relaunch.', + androidContinue: 'Continue to app', + androidFooterHint: 'Microphone access is required for dictation. Tap Request access above, or continue and grant it later from Overview.', + androidTitle: 'Set up OpenLess', + androidIntro: 'Complete mobile permissions and services step by step.', + androidStepCounter: 'Step {{current}} of {{total}}', + androidBack: 'Back', + androidNext: 'Next', + androidFinish: 'Finish and enter', + androidSteps: { + microphoneTitle: 'Microphone permission', + microphoneDesc: 'Show the Android system permission sheet and allow OpenLess to record voice.', + accessibilityTitle: 'Accessibility service', + accessibilityDesc: 'Paste recognition results back into the active input field and help detect the input context.', + overlayPermissionTitle: 'Floating window permission', + overlayPermissionDesc: 'Allow OpenLess to show the recording control over other apps.', + overlayConfigTitle: 'Floating window settings', + overlayConfigDesc: 'Configure visibility, activation, swipe actions, and button size.', + asrTitle: 'ASR cloud service', + asrDesc: 'Configure the speech-to-text provider, key, endpoint, and model.', + llmTitle: 'LLM service', + llmDesc: 'Configure the language model used for polishing, translation, and Q&A.', + }, }, overview: { kicker: 'DASHBOARD', @@ -258,6 +285,20 @@ export const en: typeof zhCN = { recentLoadFailed: 'Could not load recent transcripts. Please retry.', historyRetry: 'Retry', weekDays: ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'], + inAppDictation: { + title: 'In-app dictation', + start: 'Start recording', + stop: 'Stop recording', + idle: 'Tap to start recording', + recording: 'Recording…', + processing: 'Processing…', + }, + androidMicBanner: { + title: 'Microphone permission needed', + desc: 'Grant microphone access to use in-app dictation and voice input.', + grant: 'Request access', + openSettings: 'Open settings', + }, }, history: { kicker: 'HISTORY', @@ -769,6 +810,7 @@ export const en: typeof zhCN = { disable: 'Disable', confirmHint: 'Click ✓ on the capsule', notSupported: 'Not yet supported', + androidReadOnly: 'Global shortcuts are not available on Android. Use the record button on the overview page.', }, permissions: { title: 'Permissions', @@ -792,6 +834,7 @@ export const en: typeof zhCN = { indeterminate: 'Undetermined', openSystem: 'Open System Settings', grant: 'Grant', + rerunAndroidSetup: 'Run setup again', hotkeyInstalled: 'Installed', hotkeyStarting: 'Installing…', hotkeyFailed: 'Listener failed', @@ -799,6 +842,65 @@ export const en: typeof zhCN = { windowsImeDesc: 'Temporarily switches to the OpenLess TSF IME during voice sessions to avoid clipboard insertion limits.', windowsImeInstalled: 'Installed', windowsImeUnavailable: 'Unavailable', + androidImeLabel: 'Input method (IME)', + androidImeSelected: 'Selected', + androidImeEnabled: 'Enabled', + androidImeDisabled: 'Not enabled', + androidOverlayLabel: 'Floating overlay', + androidAccessibilityLabel: 'Accessibility service', + androidAccessibilityImpact: 'Enable it to output results to the current input field without switching keyboards. If disabled, results are copied to the clipboard for manual paste.', + androidInsertStrategyLabel: 'Text insertion strategy', + androidOverlayTriggerLabel: 'Overlay visibility', + androidOverlayActivationModeLabel: 'Overlay activation', + androidOverlayLeftSwipeActionLabel: 'Left swipe action', + androidOverlayCancelSwipeDirectionLabel: 'Cancel swipe direction', + androidOverlaySizeLabel: 'Overlay size', + androidOverlaySizeHint: 'Adjusts the floating button diameter and keeps its current position.', + androidInsertStrategy: { + accessibility: 'Auto output to input field', + clipboard: 'Clipboard only', + }, + androidInsertStrategyHint: { + accessibility: 'Requires accessibility; falls back to clipboard when unavailable.', + clipboard: 'No accessibility permission required; copies only for manual paste.', + }, + androidOverlayTrigger: { + background: 'When app is backgrounded', + keyboard: 'When keyboard appears', + always: 'Always visible', + }, + androidOverlayTriggerHint: { + background: 'Simple and battery-friendly; no overlay while typing in other apps.', + keyboard: 'This mode is shelved. Existing settings are moved back to background.', + always: 'Always available, but permanently on screen.', + }, + androidOverlayTriggerDisabled: { + keyboard: 'Keyboard-triggered display is shelved. Overlay gestures will replace keyboard detection.', + }, + androidOverlayActivationMode: { + tap: 'Tap to arm', + long_press: 'Long press to arm', + }, + androidOverlayActivationModeHint: { + tap: 'First tap arms the overlay; second tap starts normal dictation.', + long_press: 'Hold to arm the overlay; release stops the current recording or QA turn.', + }, + androidOverlayLeftSwipeAction: { + translation: 'Translation dictation', + style_pack: 'Switch style pack', + }, + androidOverlayLeftSwipeActionHint: { + translation: 'Left swipe while armed starts translation dictation.', + style_pack: 'Left swipe while armed switches to the previous style pack.', + }, + androidOverlayCancelSwipeDirection: { + up: 'Swipe up', + down: 'Swipe down', + }, + androidOverlayCancelSwipeDirectionHint: { + up: 'Swipe up while recording to cancel without transcription or insertion.', + down: 'Swipe down while recording to cancel without transcription or insertion.', + }, windowsIme: { installed: 'Installed. Voice input temporarily switches to the OpenLess IME.', notInstalled: 'Not installed. OpenLess is using the clipboard/WM_PASTE fallback.', @@ -982,6 +1084,7 @@ export const en: typeof zhCN = { macEventTap: 'macOS Event Tap', windowsLowLevel: 'Windows low-level keyboard hook', fcitx5: 'fcitx5 input method plugin', + unavailable: 'Unavailable', }, }, localAsr: { diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 5b839ed4..2561ea7e 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -51,6 +51,11 @@ export const ja: typeof zhCN = { emptyTitle: '{{recordHotkey}} を押して質問を開始', emptyDesc: '任意のアプリでテキストを選択した後、{{recordHotkey}} を 1 回押して録音を開始し、もう 1 回押して送信します。回答はここに表示され、続けて追加質問が可能です。', recordingHint: '録音中… {{recordHotkey}} をもう一度押して終了し、質問します', + mobileRecordLabel: '録音ボタン', + mobileRecordStart: '録音を開始', + mobileRecordStop: '終了して送信', + composerPlaceholder: '質問を入力。Enter で送信、Shift+Enter で改行', + composerSend: '送信', statusIdle: '{{recordHotkey}} で質問', statusRecording: '録音中', statusThinking: '思考中', @@ -225,6 +230,28 @@ export const ja: typeof zhCN = { actionRequestMic: '許可ダイアログを表示', accessibilityHint: '許可後は **OpenLess を完全に終了** してから再起動してください(macOS TCC の仕様)。', footerHint: 'すべての権限が揃うとこのガイドは自動で閉じます。閉じない場合はメニューバーの OpenLess → 終了 から再起動してください。', + androidContinue: 'アプリに進む', + androidFooterHint: '音声入力にはマイク権限が必要です。上の「許可ダイアログを表示」をタップするか、先にアプリへ進み、概要ページで後から許可してください。', + androidTitle: 'OpenLess を設定', + androidIntro: 'モバイル権限とサービス設定を順番に完了します。', + androidStepCounter: '{{current}} / {{total}}', + androidBack: '戻る', + androidNext: '次へ', + androidFinish: '完了して開始', + androidSteps: { + microphoneTitle: 'マイク権限', + microphoneDesc: 'Android のシステム権限カードを表示し、OpenLess の録音を許可します。', + accessibilityTitle: 'アクセシビリティサービス', + accessibilityDesc: '認識結果を現在の入力欄へ貼り付け、入力環境の検出を補助します。', + overlayPermissionTitle: 'フローティングウィンドウ権限', + overlayPermissionDesc: '他のアプリ上に録音コントロールを表示できるようにします。', + overlayConfigTitle: 'フローティングウィンドウ設定', + overlayConfigDesc: '表示タイミング、起動方法、スワイプ操作、ボタンサイズを設定します。', + asrTitle: 'ASR クラウドサービス', + asrDesc: '音声認識サービスのプロバイダー、キー、エンドポイント、モデルを設定します。', + llmTitle: 'LLM サービス', + llmDesc: '整文、翻訳、Q&A に使う言語モデルサービスを設定します。', + }, }, overview: { kicker: 'DASHBOARD', @@ -260,6 +287,20 @@ export const ja: typeof zhCN = { recentLoadFailed: '最近の認識を読み込めません。再試行してください。', historyRetry: '再試行', weekDays: ['日', '月', '火', '水', '木', '金', '土'], + inAppDictation: { + title: 'アプリ内音声入力', + start: '録音開始', + stop: '録音停止', + idle: 'タップして録音開始', + recording: '録音中…', + processing: '処理中…', + }, + androidMicBanner: { + title: 'マイク権限が必要です', + desc: 'マイクを許可すると、アプリ内音声入力が使えます。', + grant: '許可ダイアログを表示', + openSettings: '設定を開く', + }, }, history: { kicker: 'HISTORY', @@ -771,6 +812,7 @@ export const ja: typeof zhCN = { disable: '無効化', confirmHint: '右側の ✓ をクリック', notSupported: '未対応', + androidReadOnly: 'Android ではグローバルショートカットは使えません。概要ページの録音ボタンを使ってください。', }, permissions: { title: '権限', @@ -794,6 +836,7 @@ export const ja: typeof zhCN = { indeterminate: '未確定', openSystem: 'システム設定を開く', grant: '許可する', + rerunAndroidSetup: 'セットアップを再実行', hotkeyInstalled: 'インストール済み', hotkeyStarting: 'インストール中…', hotkeyFailed: '監視失敗', @@ -801,6 +844,31 @@ export const ja: typeof zhCN = { windowsImeDesc: '音声セッション中に OpenLess TSF IME へ一時的に切り替え、クリップボード入力の制限を回避します。', windowsImeInstalled: 'インストール済み', windowsImeUnavailable: '利用不可', + androidImeLabel: '入力メソッド (IME)', + androidImeSelected: '選択中', + androidImeEnabled: '有効', + androidImeDisabled: '無効', + androidOverlayLabel: 'フローティングオーバーレイ', + androidAccessibilityLabel: 'アクセシビリティ', + androidAccessibilityImpact: '有効にすると、キーボードを切り替えずに現在の入力欄へ結果を出力します。無効の場合はクリップボードへコピーし、手動で貼り付けます。', + androidInsertStrategyLabel: '挿入方式', + androidOverlayTriggerLabel: '表示タイミング', + androidOverlayActivationModeLabel: '起動方法', + androidOverlayLeftSwipeActionLabel: '左スワイプ動作', + androidOverlayCancelSwipeDirectionLabel: 'キャンセル方向', + androidOverlaySizeLabel: 'オーバーレイサイズ', + androidOverlaySizeHint: 'フローティングボタンの直径を調整し、現在位置を保持します。', + androidInsertStrategy: { accessibility: '入力欄へ自動出力', clipboard: 'クリップボードのみ' }, + androidInsertStrategyHint: { accessibility: 'アクセシビリティが必要です。使えない場合はクリップボードにコピーします。', clipboard: 'アクセシビリティ権限は不要です。コピー後に手動で貼り付けます。' }, + androidOverlayTrigger: { background: 'バックグラウンド', keyboard: 'キーボード表示時', always: '常時' }, + androidOverlayTriggerHint: { background: 'シンプル', keyboard: 'このモードは保留中です。既存設定はバックグラウンドに戻します。', always: '常に表示' }, + androidOverlayTriggerDisabled: { keyboard: 'キーボード表示時の表示は保留中です。今後はフローティングウィンドウのジェスチャーで置き換えます。' }, + androidOverlayActivationMode: { tap: 'タップで起動', long_press: '長押しで起動' }, + androidOverlayActivationModeHint: { tap: '1回目のタップで待機状態に入り、2回目のタップで通常の音声入力を開始します。', long_press: '押している間だけ待機状態に入り、離すと現在の録音またはQAターンを終了します。' }, + androidOverlayLeftSwipeAction: { translation: '翻訳入力', style_pack: 'スタイルパック切替' }, + androidOverlayLeftSwipeActionHint: { translation: '待機状態で左スワイプすると翻訳入力を開始します。', style_pack: '待機状態で左スワイプすると前のスタイルパックへ切り替えます。' }, + androidOverlayCancelSwipeDirection: { up: '上へスワイプ', down: '下へスワイプ' }, + androidOverlayCancelSwipeDirectionHint: { up: '録音中に上へスワイプすると、文字起こしや挿入をせずにキャンセルします。', down: '録音中に下へスワイプすると、文字起こしや挿入をせずにキャンセルします。' }, windowsIme: { installed: 'インストール済み。音声入力時に OpenLess IME へ一時的に切り替えます。', notInstalled: '未インストール。OpenLess は現在クリップボード / WM_PASTE フォールバックを使用しています。', @@ -984,6 +1052,7 @@ export const ja: typeof zhCN = { macEventTap: 'macOS Event Tap', windowsLowLevel: 'Windows 低レベルキーボードフック', fcitx5: 'fcitx5 インプットメソッドプラグイン', + unavailable: '利用不可', }, }, localAsr: { diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 0ecbf448..77e73e46 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -51,6 +51,11 @@ export const ko: typeof zhCN = { emptyTitle: '{{recordHotkey}} 를 눌러 질문 시작', emptyDesc: '아무 앱에서 텍스트를 선택한 후 {{recordHotkey}} 를 한 번 눌러 녹음을 시작하고, 다시 한 번 눌러 종료 후 제출합니다. 답변이 여기에 표시되며 연속해서 후속 질문이 가능합니다.', recordingHint: '녹음 중… {{recordHotkey}} 를 다시 눌러 종료하고 질문', + mobileRecordLabel: '녹음 버튼', + mobileRecordStart: '녹음 시작', + mobileRecordStop: '종료하고 제출', + composerPlaceholder: '질문을 입력하세요. Enter로 보내고 Shift+Enter로 줄바꿈', + composerSend: '보내기', statusIdle: '{{recordHotkey}} 로 질문', statusRecording: '녹음 중', statusThinking: '생각 중', @@ -225,6 +230,28 @@ export const ko: typeof zhCN = { actionRequestMic: '권한 대화상자 표시', accessibilityHint: '허용 후에는 **OpenLess 를 완전히 종료** 한 다음 다시 실행해야 합니다(macOS TCC 규칙).', footerHint: '모든 권한이 부여되면 이 가이드는 자동으로 닫힙니다. 닫히지 않으면 메뉴 막대의 OpenLess → 종료 후 앱을 다시 실행해 주세요.', + androidContinue: '앱으로 계속', + androidFooterHint: '받아쓰기에는 마이크 권한이 필요합니다. 위의 권한 요청을 탭하거나, 앱으로 먼저 들어가 개요 페이지에서 나중에 허용할 수 있습니다.', + androidTitle: 'OpenLess 설정', + androidIntro: '모바일 권한과 서비스 설정을 단계별로 완료합니다.', + androidStepCounter: '{{current}} / {{total}} 단계', + androidBack: '이전', + androidNext: '다음', + androidFinish: '완료하고 시작', + androidSteps: { + microphoneTitle: '마이크 권한', + microphoneDesc: 'Android 시스템 권한 카드를 표시하고 OpenLess 녹음을 허용합니다.', + accessibilityTitle: '접근성 서비스', + accessibilityDesc: '인식 결과를 현재 입력란에 붙여넣고 입력 환경 감지를 보조합니다.', + overlayPermissionTitle: '플로팅 창 권한', + overlayPermissionDesc: '다른 앱 위에 녹음 제어 버튼을 표시할 수 있게 합니다.', + overlayConfigTitle: '플로팅 창 설정', + overlayConfigDesc: '표시 시점, 활성화 방식, 스와이프 동작, 버튼 크기를 설정합니다.', + asrTitle: 'ASR 클라우드 서비스', + asrDesc: '음성 인식 서비스의 공급자, 키, 엔드포인트, 모델을 설정합니다.', + llmTitle: 'LLM 서비스', + llmDesc: '문장 다듬기, 번역, Q&A에 사용할 언어 모델 서비스를 설정합니다.', + }, }, overview: { kicker: 'DASHBOARD', @@ -260,6 +287,20 @@ export const ko: typeof zhCN = { recentLoadFailed: '최근 인식 기록을 불러올 수 없습니다. 다시 시도해 주세요.', historyRetry: '다시 시도', weekDays: ['일', '월', '화', '수', '목', '금', '토'], + inAppDictation: { + title: '앱 내 받아쓰기', + start: '녹음 시작', + stop: '녹음 중지', + idle: '탭하여 녹음 시작', + recording: '녹음 중…', + processing: '처리 중…', + }, + androidMicBanner: { + title: '마이크 권한이 필요합니다', + desc: '마이크를 허용하면 앱 내 받아쓰기와 음성 입력을 사용할 수 있습니다.', + grant: '권한 요청', + openSettings: '설정 열기', + }, }, history: { kicker: 'HISTORY', @@ -771,6 +812,7 @@ export const ko: typeof zhCN = { disable: '비활성화', confirmHint: '오른쪽 ✓ 클릭', notSupported: '지원되지 않음', + androidReadOnly: 'Android에서는 전역 단축키를 사용할 수 없습니다. 개요 페이지의 녹음 버튼을 사용하세요.', }, permissions: { title: '권한', @@ -794,6 +836,7 @@ export const ko: typeof zhCN = { indeterminate: '미결정', openSystem: '시스템 설정 열기', grant: '허용', + rerunAndroidSetup: '설정 마법사 다시 실행', hotkeyInstalled: '설치됨', hotkeyStarting: '설치 중…', hotkeyFailed: '감지 실패', @@ -801,6 +844,31 @@ export const ko: typeof zhCN = { windowsImeDesc: '음성 세션 동안 OpenLess TSF 입력기로 일시적으로 전환하여 클립보드 입력 제한을 회피하기 위해 사용.', windowsImeInstalled: '설치됨', windowsImeUnavailable: '사용 불가', + androidImeLabel: '입력기 (IME)', + androidImeSelected: '선택됨', + androidImeEnabled: '활성화됨', + androidImeDisabled: '비활성', + androidOverlayLabel: '플로팅 오버레이', + androidAccessibilityLabel: '접근성 서비스', + androidAccessibilityImpact: '켜면 키보드를 전환하지 않고 현재 입력칸에 결과를 출력합니다. 끄면 클립보드에 복사되며 직접 붙여넣어야 합니다.', + androidInsertStrategyLabel: '텍스트 삽입 방식', + androidOverlayTriggerLabel: '오버레이 표시', + androidOverlayActivationModeLabel: '오버레이 활성화', + androidOverlayLeftSwipeActionLabel: '왼쪽 스와이프 동작', + androidOverlayCancelSwipeDirectionLabel: '취소 스와이프 방향', + androidOverlaySizeLabel: '오버레이 크기', + androidOverlaySizeHint: '플로팅 버튼 지름을 조정하고 현재 위치를 유지합니다.', + androidInsertStrategy: { accessibility: '입력칸에 자동 출력', clipboard: '클립보드만' }, + androidInsertStrategyHint: { accessibility: '접근성 서비스가 필요합니다. 사용할 수 없으면 클립보드에 복사합니다.', clipboard: '접근성 권한이 필요 없으며 직접 붙여넣습니다.' }, + androidOverlayTrigger: { background: '백그라운드', keyboard: '키보드 표시 시', always: '항상' }, + androidOverlayTriggerHint: { background: '단순', keyboard: '이 모드는 보류되었습니다. 기존 설정은 백그라운드로 되돌립니다.', always: '항상 표시' }, + androidOverlayTriggerDisabled: { keyboard: '키보드 표시 감지는 보류되었습니다. 이후 오버레이 제스처로 대체합니다.' }, + androidOverlayActivationMode: { tap: '탭으로 활성화', long_press: '길게 눌러 활성화' }, + androidOverlayActivationModeHint: { tap: '첫 탭은 대기 상태로 전환하고, 두 번째 탭은 일반 받아쓰기를 시작합니다.', long_press: '누르고 있는 동안 대기 상태가 되며, 손을 떼면 현재 녹음 또는 QA 턴을 종료합니다.' }, + androidOverlayLeftSwipeAction: { translation: '번역 받아쓰기', style_pack: '스타일 팩 전환' }, + androidOverlayLeftSwipeActionHint: { translation: '대기 상태에서 왼쪽으로 밀면 번역 받아쓰기를 시작합니다.', style_pack: '대기 상태에서 왼쪽으로 밀면 이전 스타일 팩으로 전환합니다.' }, + androidOverlayCancelSwipeDirection: { up: '위로 스와이프', down: '아래로 스와이프' }, + androidOverlayCancelSwipeDirectionHint: { up: '녹음 중 위로 밀면 전사와 삽입 없이 취소합니다.', down: '녹음 중 아래로 밀면 전사와 삽입 없이 취소합니다.' }, windowsIme: { installed: '설치됨. 음성 입력 시 OpenLess 입력기로 일시 전환됩니다.', notInstalled: '설치되지 않음. OpenLess 는 현재 클립보드 / WM_PASTE 폴백을 사용합니다.', @@ -984,6 +1052,7 @@ export const ko: typeof zhCN = { macEventTap: 'macOS Event Tap', windowsLowLevel: 'Windows 저수준 키보드 후크', fcitx5: 'fcitx5 입력기 플러그인', + unavailable: '사용 불가', }, }, localAsr: { diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 38a1bc08..50a73278 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -47,6 +47,11 @@ export const zhCN = { emptyTitle: '按 {{recordHotkey}} 开始提问', emptyDesc: '在任意 app 选中一段文字后,按一次 {{recordHotkey}} 开始录音,再按一次结束并提交。回答会显示在这里,可以连续多轮追问。', recordingHint: '录音中…再按一次 {{recordHotkey}} 结束并提问', + mobileRecordLabel: '录音按钮', + mobileRecordStart: '开始录音', + mobileRecordStop: '结束并提交', + composerPlaceholder: '输入问题,Enter 发送,Shift+Enter 换行', + composerSend: '发送', statusIdle: '按 {{recordHotkey}} 提问', statusRecording: '录音中', statusThinking: '思考中', @@ -221,6 +226,28 @@ export const zhCN = { actionRequestMic: '弹出授权', accessibilityHint: '授权后必须**完全退出 OpenLess** 再重新打开(macOS TCC 规则)。', footerHint: '授权全部完成后此引导自动关闭。如果一直不消失,从菜单栏 OpenLess → 退出,重新打开 App。', + androidContinue: '先进入应用', + androidFooterHint: '听写需要麦克风权限。可点击上方「弹出授权」,或先进入应用后在概览页继续授权。', + androidTitle: '配置 OpenLess', + androidIntro: '按步骤完成移动端权限和服务配置。', + androidStepCounter: '第 {{current}} / {{total}} 项', + androidBack: '上一步', + androidNext: '下一步', + androidFinish: '完成并进入', + androidSteps: { + microphoneTitle: '麦克风权限', + microphoneDesc: '调用 Android 系统授权卡片,允许 OpenLess 录制语音。', + accessibilityTitle: '无障碍服务', + accessibilityDesc: '用于把识别结果粘贴回当前输入框,并辅助检测输入环境。', + overlayPermissionTitle: '悬浮窗权限', + overlayPermissionDesc: '允许 OpenLess 在其他应用上显示录音控制按钮。', + overlayConfigTitle: '悬浮窗配置', + overlayConfigDesc: '设置悬浮窗显示时机、触发方式、滑动动作和按钮大小。', + asrTitle: 'ASR 云服务', + asrDesc: '配置语音转文字服务的供应商、密钥、接口地址和模型。', + llmTitle: 'LLM 服务', + llmDesc: '配置文本润色、翻译和问答使用的语言模型服务。', + }, }, overview: { kicker: 'DASHBOARD', @@ -256,6 +283,20 @@ export const zhCN = { recentLoadFailed: '无法读取最近识别,请重试。', historyRetry: '重试', weekDays: ['日', '一', '二', '三', '四', '五', '六'], + inAppDictation: { + title: '应用内录音', + start: '开始录音', + stop: '停止录音', + idle: '点击开始录音', + recording: '录音中…', + processing: '处理中…', + }, + androidMicBanner: { + title: '需要麦克风权限', + desc: '授权麦克风后可使用应用内录音与语音输入。', + grant: '弹出授权', + openSettings: '打开系统设置', + }, }, history: { kicker: 'HISTORY', @@ -767,6 +808,7 @@ export const zhCN = { disable: '停用', confirmHint: '点击右侧 ✓', notSupported: '暂未支持', + androidReadOnly: 'Android 不支持全局快捷键,请在概览页使用录音按钮。', }, permissions: { title: '权限', @@ -790,6 +832,7 @@ export const zhCN = { indeterminate: '未确定', openSystem: '打开系统设置', grant: '授权', + rerunAndroidSetup: '重新运行设置向导', hotkeyInstalled: '已安装', hotkeyStarting: '安装中…', hotkeyFailed: '监听失败', @@ -797,6 +840,65 @@ export const zhCN = { windowsImeDesc: '语音输入时临时切到 OpenLess TSF,绕过剪贴板限制。', windowsImeInstalled: '已安装', windowsImeUnavailable: '不可用', + androidImeLabel: '输入法 (IME)', + androidImeSelected: '已选中', + androidImeEnabled: '已启用', + androidImeDisabled: '未启用', + androidOverlayLabel: '悬浮窗', + androidAccessibilityLabel: '无障碍服务', + androidAccessibilityImpact: '开启后可在不切换键盘的情况下把结果输出到当前输入框;不开启时仍会复制到剪贴板,需要手动粘贴。', + androidInsertStrategyLabel: '文本插入策略', + androidOverlayTriggerLabel: '悬浮窗显示时机', + androidOverlayActivationModeLabel: '悬浮窗激活方式', + androidOverlayLeftSwipeActionLabel: '左滑动作', + androidOverlayCancelSwipeDirectionLabel: '取消录音滑向', + androidOverlaySizeLabel: '悬浮窗大小', + androidOverlaySizeHint: '调整悬浮按钮直径,保存后在当前悬浮窗上生效并保留位置。', + androidInsertStrategy: { + accessibility: '自动输出到输入框', + clipboard: '仅剪贴板', + }, + androidInsertStrategyHint: { + accessibility: '需要开启无障碍服务;不可用时会复制到剪贴板。', + clipboard: '不需要无障碍权限,结果只复制到剪贴板,由你手动粘贴。', + }, + androidOverlayTrigger: { + background: '应用退到后台', + keyboard: '弹出键盘时', + always: '始终显示', + }, + androidOverlayTriggerHint: { + background: '省电、实现简单;其他 App 输入时不会自动出现。', + keyboard: '该模式已暂缓,历史配置会自动改为“应用退到后台”。', + always: '入口始终可见,但会一直占屏。', + }, + androidOverlayTriggerDisabled: { + keyboard: '“弹出键盘时”暂缓开放,后续将以悬浮窗手势替代键盘检测。', + }, + androidOverlayActivationMode: { + tap: '点按激活', + long_press: '长按激活', + }, + androidOverlayActivationModeHint: { + tap: '第一次点按进入激活态,第二次点按开始普通听写。', + long_press: '按住进入激活态;松开时结束当前录音或问答轮次。', + }, + androidOverlayLeftSwipeAction: { + translation: '翻译听写', + style_pack: '切换风格包', + }, + androidOverlayLeftSwipeActionHint: { + translation: '激活态左滑后按翻译模式录音。', + style_pack: '激活态左滑后切换到上一个风格包。', + }, + androidOverlayCancelSwipeDirection: { + up: '向上滑', + down: '向下滑', + }, + androidOverlayCancelSwipeDirectionHint: { + up: '录音中向上滑取消本次听写,不转写、不插入。', + down: '录音中向下滑取消本次听写,不转写、不插入。', + }, windowsIme: { installed: '已安装,按需切到 OpenLess 输入法。', notInstalled: '未安装,走剪贴板 / WM_PASTE 兜底。', @@ -980,6 +1082,7 @@ export const zhCN = { macEventTap: 'macOS Event Tap', windowsLowLevel: 'Windows 低层键盘 hook', fcitx5: 'fcitx5 输入法插件', + unavailable: '不可用', }, }, localAsr: { diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index b98030e4..80f14ef2 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -49,6 +49,11 @@ export const zhTW: typeof zhCN = { emptyTitle: '按 {{recordHotkey}} 開始提問', emptyDesc: '在任意 app 選中一段文字後,按一次 {{recordHotkey}} 開始錄音,再按一次結束並提交。回答會顯示在這裏,可以連續多輪追問。', recordingHint: '錄音中…再按一次 {{recordHotkey}} 結束並提問', + mobileRecordLabel: '錄音按鈕', + mobileRecordStart: '開始錄音', + mobileRecordStop: '結束並提交', + composerPlaceholder: '輸入問題,Enter 發送,Shift+Enter 換行', + composerSend: '發送', statusIdle: '按 {{recordHotkey}} 提問', statusRecording: '錄音中', statusThinking: '思考中', @@ -223,6 +228,28 @@ export const zhTW: typeof zhCN = { actionRequestMic: '彈出授權', accessibilityHint: '授權後必須**完全退出 OpenLess** 再重新打開(macOS TCC 規則)。', footerHint: '授權全部完成後此引導自動關閉。如果一直不消失,從菜單欄 OpenLess → 退出,重新打開 App。', + androidContinue: '先進入應用', + androidFooterHint: '聽寫需要麥克風權限。可點擊上方「彈出授權」,或先進入應用後在概覽頁繼續授權。', + androidTitle: '配置 OpenLess', + androidIntro: '按步驟完成移動端權限和服務配置。', + androidStepCounter: '第 {{current}} / {{total}} 項', + androidBack: '上一步', + androidNext: '下一步', + androidFinish: '完成並進入', + androidSteps: { + microphoneTitle: '麥克風權限', + microphoneDesc: '調用 Android 系統授權卡片,允許 OpenLess 錄製語音。', + accessibilityTitle: '無障礙服務', + accessibilityDesc: '用於把識別結果貼回當前輸入框,並輔助檢測輸入環境。', + overlayPermissionTitle: '懸浮窗權限', + overlayPermissionDesc: '允許 OpenLess 在其他應用上顯示錄音控制按鈕。', + overlayConfigTitle: '懸浮窗配置', + overlayConfigDesc: '設置懸浮窗顯示時機、觸發方式、滑動動作和按鈕大小。', + asrTitle: 'ASR 雲服務', + asrDesc: '配置語音轉文字服務的供應商、密鑰、接口地址和模型。', + llmTitle: 'LLM 服務', + llmDesc: '配置文本潤色、翻譯和問答使用的語言模型服務。', + }, }, overview: { kicker: 'DASHBOARD', @@ -258,6 +285,20 @@ export const zhTW: typeof zhCN = { recentLoadFailed: '無法讀取最近識別,請重試。', historyRetry: '重試', weekDays: ['日', '一', '二', '三', '四', '五', '六'], + inAppDictation: { + title: '應用內錄音', + start: '開始錄音', + stop: '停止錄音', + idle: '點擊開始錄音', + recording: '錄音中…', + processing: '處理中…', + }, + androidMicBanner: { + title: '需要麥克風權限', + desc: '授權麥克風後可使用應用內錄音與語音輸入。', + grant: '彈出授權', + openSettings: '打開系統設置', + }, }, history: { kicker: 'HISTORY', @@ -769,6 +810,7 @@ export const zhTW: typeof zhCN = { disable: '停用', confirmHint: '點擊右側 ✓', notSupported: '暫未支持', + androidReadOnly: 'Android 不支援全域快捷鍵,請在概覽頁使用錄音按鈕。', }, permissions: { title: '權限', @@ -792,6 +834,7 @@ export const zhTW: typeof zhCN = { indeterminate: '未確定', openSystem: '打開系統設置', grant: '授權', + rerunAndroidSetup: '重新執行設定向導', hotkeyInstalled: '已安裝', hotkeyStarting: '安裝中…', hotkeyFailed: '監聽失敗', @@ -799,6 +842,31 @@ export const zhTW: typeof zhCN = { windowsImeDesc: '用於在語音會話期間臨時切換到 OpenLess TSF 輸入法,避免剪貼板插入限制。', windowsImeInstalled: '已安裝', windowsImeUnavailable: '不可用', + androidImeLabel: '輸入法 (IME)', + androidImeSelected: '已選中', + androidImeEnabled: '已啟用', + androidImeDisabled: '未啟用', + androidOverlayLabel: '懸浮窗', + androidAccessibilityLabel: '無障礙服務', + androidAccessibilityImpact: '開啟後可在不切換鍵盤的情況下把結果輸出到目前輸入框;未開啟時仍會複製到剪貼簿,需要手動貼上。', + androidInsertStrategyLabel: '文字插入策略', + androidOverlayTriggerLabel: '懸浮窗顯示時機', + androidOverlayActivationModeLabel: '懸浮窗啟用方式', + androidOverlayLeftSwipeActionLabel: '左滑動作', + androidOverlayCancelSwipeDirectionLabel: '取消錄音滑向', + androidOverlaySizeLabel: '懸浮窗大小', + androidOverlaySizeHint: '調整懸浮按鈕直徑,儲存後在目前懸浮窗上生效並保留位置。', + androidInsertStrategy: { accessibility: '自動輸出到輸入框', clipboard: '僅剪貼簿' }, + androidInsertStrategyHint: { accessibility: '需要開啟無障礙服務;不可用時會複製到剪貼簿。', clipboard: '不需要無障礙權限,只複製到剪貼簿,由你手動貼上。' }, + androidOverlayTrigger: { background: '退到背景', keyboard: '鍵盤彈出時', always: '常駐' }, + androidOverlayTriggerHint: { background: '省電', keyboard: '此模式已暫緩,既有設定會改回退到背景。', always: '一直佔屏' }, + androidOverlayTriggerDisabled: { keyboard: '「鍵盤彈出時」暫緩開放,後續將以懸浮窗手勢取代鍵盤偵測。' }, + androidOverlayActivationMode: { tap: '點按啟用', long_press: '長按啟用' }, + androidOverlayActivationModeHint: { tap: '第一次點按進入啟用狀態,第二次點按開始普通聽寫。', long_press: '按住進入啟用狀態;放開時結束目前錄音或問答輪次。' }, + androidOverlayLeftSwipeAction: { translation: '翻譯聽寫', style_pack: '切換風格包' }, + androidOverlayLeftSwipeActionHint: { translation: '啟用狀態左滑後按翻譯模式錄音。', style_pack: '啟用狀態左滑後切換到上一個風格包。' }, + androidOverlayCancelSwipeDirection: { up: '向上滑', down: '向下滑' }, + androidOverlayCancelSwipeDirectionHint: { up: '錄音中向上滑取消本次聽寫,不轉寫、不插入。', down: '錄音中向下滑取消本次聽寫,不轉寫、不插入。' }, windowsIme: { installed: '已安裝。語音輸入時會臨時切換到 OpenLess 輸入法。', notInstalled: '未安裝。OpenLess 正在使用剪貼板 / WM_PASTE 兜底。', @@ -982,6 +1050,7 @@ export const zhTW: typeof zhCN = { macEventTap: 'macOS Event Tap', windowsLowLevel: 'Windows 低層鍵盤 hook', fcitx5: 'fcitx5 輸入法插件', + unavailable: '不可用', }, }, localAsr: { diff --git a/openless-all/app/src/lib/androidMicrophonePermission.ts b/openless-all/app/src/lib/androidMicrophonePermission.ts new file mode 100644 index 00000000..74490bd6 --- /dev/null +++ b/openless-all/app/src/lib/androidMicrophonePermission.ts @@ -0,0 +1 @@ +export * from '@android/lib/androidMicrophonePermission'; diff --git a/openless-all/app/src/lib/ipc.ts b/openless-all/app/src/lib/ipc.ts index 8f8d246b..84202664 100644 --- a/openless-all/app/src/lib/ipc.ts +++ b/openless-all/app/src/lib/ipc.ts @@ -15,6 +15,7 @@ import type { HotkeyStatus, MicrophoneDevice, PermissionStatus, + PlatformCapabilities, CodingAgentPermissionMode, PolishMode, QaHotkeyBinding, @@ -29,13 +30,16 @@ import type { VocabPresetStore, WindowsImeStatus, } from "./types" -export type { UpdateChannel } from "./types" +export type { UpdateChannel, PlatformCapabilities } from "./types" import { OL_DATA } from "./mockData" import { defaultAppShortcutModifiers, defaultQaShortcut, formatComboLabel, } from "./hotkey" +import { + getPlatformCapabilities as loadPlatformCapabilities, +} from "./platform" declare global { interface Window { @@ -47,6 +51,56 @@ const isTauri = globalThis.window !== undefined && "__TAURI_INTERNALS__" in globalThis.window +let platformCapsPromise: Promise | null = null + +async function platformCapabilities(): Promise { + platformCapsPromise ??= loadPlatformCapabilities() + return platformCapsPromise +} + +export async function getPlatformCapabilities(): Promise { + return platformCapabilities() +} + +export { + getAndroidOverlayStatus, + requestAndroidOverlayPermission, + showAndroidOverlay, + hideAndroidOverlay, + getAndroidAccessibilityStatus, + requestAndroidAccessibilityPermission, +} from '../../android/frontend/lib/androidIpc'; + +export { isAndroid, isDesktop, isMobile } from "./platform" + +const androidHotkeyCapability: HotkeyCapability = { + adapter: "unavailable", + availableTriggers: [], + requiresAccessibilityPermission: false, + supportsModifierOnlyTrigger: false, + supportsSideSpecificModifiers: false, + explicitFallbackAvailable: false, + statusHint: + "移动端不支持全局热键;请使用应用内录音按钮或悬浮窗(需授权)。", +} + +const androidHotkeyStatus: HotkeyStatus = { + adapter: "unavailable", + state: "failed", + message: "移动端不支持全局热键", + lastError: { + code: "unavailable", + message: "Global hotkeys are not available on mobile", + }, +} + +const androidWindowsImeStatus: WindowsImeStatus = { + state: "notWindows", + usingTsfBackend: false, + message: "Not available on Android", + dllPath: null, +} + export async function invokeOrMock( cmd: string, args: Record | undefined, @@ -132,6 +186,12 @@ let mockSettings: UserPreferences = { audioRecordingMaxEntries: null, marketplaceBaseUrl: "https://apic.openless.top", marketplaceDevLogin: "", + androidInsertStrategy: "accessibility", + androidOverlayTrigger: "background", + androidOverlayActivationMode: "tap", + androidOverlayLeftSwipeAction: "translation", + androidOverlayCancelSwipeDirection: "up", + androidOverlaySizeDp: 72, } const mockFullStylePrompts: StyleSystemPrompts = { @@ -545,40 +605,95 @@ export interface LatestBetaRelease { publishedAt: string } +export interface AppUpdateMetadata { + rid: number + currentVersion: string + version: string + date?: string | null + body?: string | null + rawJson: Record +} + export function getUpdateChannel(): Promise { - return invokeOrMock( - "get_update_channel", - undefined, - () => "stable" as UpdateChannel, - ) + return platformCapabilities().then((caps) => { + if (!caps.supportsAutoUpdate) { + return "stable" as UpdateChannel + } + return invokeOrMock( + "get_update_channel", + undefined, + () => "stable" as UpdateChannel, + ) + }) } export function setUpdateChannel(channel: UpdateChannel): Promise { - return invokeOrMock("set_update_channel", { channel }, () => undefined) + return platformCapabilities().then((caps) => { + if (!caps.supportsAutoUpdate) { + return undefined + } + return invokeOrMock("set_update_channel", { channel }, () => undefined) + }) } export function fetchLatestBetaRelease(): Promise { - return invokeOrMock("fetch_latest_beta_release", undefined, () => null) + return platformCapabilities().then((caps) => { + if (!caps.supportsAutoUpdate) { + return null + } + return invokeOrMock("fetch_latest_beta_release", undefined, () => null) + }) +} + +export function appCheckUpdateWithChannel( + timeoutMs: number, + channel?: UpdateChannel | null, +): Promise { + return platformCapabilities().then((caps) => { + if (!caps.supportsAutoUpdate) { + return null + } + return invokeOrMock( + "app_check_update_with_channel", + { timeoutMs, channel: channel ?? null }, + () => null, + ) + }) } export function getHotkeyStatus(): Promise { - return invokeOrMock("get_hotkey_status", undefined, () => mockHotkeyStatus) + return platformCapabilities().then((caps) => { + if (!caps.supportsDesktopHotkey) { + return androidHotkeyStatus + } + return invokeOrMock("get_hotkey_status", undefined, () => mockHotkeyStatus) + }) } export function getHotkeyCapability(): Promise { - return invokeOrMock( - "get_hotkey_capability", - undefined, - () => mockHotkeyCapability, - ) + return platformCapabilities().then((caps) => { + if (!caps.supportsDesktopHotkey) { + return androidHotkeyCapability + } + return invokeOrMock( + "get_hotkey_capability", + undefined, + () => mockHotkeyCapability, + ) + }) } export function getWindowsImeStatus(): Promise { - return invokeOrMock( - "get_windows_ime_status", - undefined, - () => mockWindowsImeStatus, - ) + return platformCapabilities().then((caps) => { + if (caps.platform === "android") { + return androidWindowsImeStatus + } + return invokeOrMock( + "get_windows_ime_status", + undefined, + () => mockWindowsImeStatus, + ) + }) } export interface NetworkCheckResult { @@ -804,11 +919,16 @@ export function handleWindowHotkeyEvent( code: string, repeat: boolean, ): Promise { - return invokeOrMock( - "handle_window_hotkey_event", - { event_type: eventType, key, code, repeat }, - () => undefined, - ) + return platformCapabilities().then((caps) => { + if (!caps.supportsDesktopHotkey) { + return undefined + } + return invokeOrMock( + "handle_window_hotkey_event", + { event_type: eventType, key, code, repeat }, + () => undefined, + ) + }) } // ── Polish ───────────────────────────────────────────────────────────── @@ -1048,7 +1168,7 @@ export function requestMicrophonePermission(): Promise { } export function openSystemSettings( - pane: "accessibility" | "microphone", + pane: "accessibility" | "microphone" | "overlay", ): Promise { return invokeOrMock("open_system_settings", { pane }, () => undefined) } @@ -1082,6 +1202,14 @@ export function qaWindowPin(pinned: boolean): Promise { return invokeOrMock("qa_window_pin", { pinned }, () => undefined) } +export function qaToggleRecording(): Promise { + return invokeOrMock("qa_toggle_recording", undefined, () => undefined) +} + +export function qaSubmitText(text: string): Promise { + return invokeOrMock("qa_submit_text", { text }, () => undefined) +} + // ── Less Computer 浮窗 ──────────────────────────────────────────────── /** 用户点 ✕ / 按 Esc 关闭 Less Computer 浮窗(隐藏窗口)。 */ export function lessComputerWindowDismiss(): Promise { @@ -1158,8 +1286,21 @@ export async function openExternal(url: string): Promise { window.open(url, "_blank", "noopener,noreferrer") return } - const { open } = await import("@tauri-apps/plugin-shell") - await open(url) + try { + const { open } = await import("@tauri-apps/plugin-shell") + await open(url) + return + } catch (error) { + console.warn("[external-open] shell plugin failed", error) + } + try { + const { invoke } = await import("@tauri-apps/api/core") + await invoke("open_external_url", { url }) + return + } catch (error) { + console.warn("[external-open] native fallback failed", error) + } + window.open(url, "_blank", "noopener,noreferrer") } /** diff --git a/openless-all/app/src/lib/platform.ts b/openless-all/app/src/lib/platform.ts new file mode 100644 index 00000000..03badcdb --- /dev/null +++ b/openless-all/app/src/lib/platform.ts @@ -0,0 +1,122 @@ +// Platform capability detection for desktop vs Android APK targets. +// Prefers Tauri `get_platform_capabilities`; falls back to UA / OS heuristics. + +import { detectOS } from '../components/WindowChrome'; +import type { PlatformCapabilities, PlatformKind } from './types'; + +export type { PlatformCapabilities, PlatformKind }; + +let cachedCapabilities: PlatformCapabilities | null = null; + +function detectAndroidFromUa(): boolean { + if (typeof navigator === 'undefined') return false; + const uaDataPlatform = + (navigator as Navigator & { userAgentData?: { platform?: string } }) + .userAgentData?.platform ?? ''; + const hints = `${navigator.userAgent || ''} ${navigator.platform || ''} ${uaDataPlatform}`; + return /Android/i.test(hints); +} + +function detectIosFromUa(): boolean { + if (typeof navigator === 'undefined') return false; + const uaDataPlatform = + (navigator as Navigator & { userAgentData?: { platform?: string } }) + .userAgentData?.platform ?? ''; + const hints = `${navigator.userAgent || ''} ${navigator.platform || ''} ${uaDataPlatform}`; + return /iPhone|iPad|iPod/i.test(hints); +} + +/** Unavailable capability flags for iOS / other non-Android mobile targets. */ +const MOBILE_UNAVAILABLE: PlatformCapabilities = { + platform: 'mobile', + supportsDesktopHotkey: false, + supportsTray: false, + supportsOverlay: false, + supportsImeInput: false, + supportsLocalAsr: false, + supportsInAppDictation: false, + supportsAutoUpdate: false, +}; + +export function isAndroid(): boolean { + if (cachedCapabilities) return cachedCapabilities.platform === 'android'; + return detectOS() === 'android' || detectAndroidFromUa(); +} + +export function isMobile(): boolean { + if (cachedCapabilities) { + return ( + cachedCapabilities.platform === 'mobile' || + cachedCapabilities.platform === 'android' + ); + } + return isAndroid() || detectIosFromUa(); +} + +export function isDesktop(): boolean { + if (cachedCapabilities) return cachedCapabilities.platform === 'desktop'; + return !isMobile(); +} + +export function inferPlatformCapabilities(): PlatformCapabilities { + if (isAndroid()) { + return { + platform: 'android', + supportsDesktopHotkey: false, + supportsTray: false, + supportsOverlay: true, + supportsImeInput: false, + supportsLocalAsr: false, + supportsInAppDictation: true, + supportsAutoUpdate: false, + }; + } + + if (detectIosFromUa()) { + return MOBILE_UNAVAILABLE; + } + + const os = detectOS(); + return { + platform: 'desktop', + supportsDesktopHotkey: true, + supportsTray: true, + supportsOverlay: true, + supportsImeInput: os === 'win', + supportsLocalAsr: os === 'mac' || os === 'win', + supportsInAppDictation: false, + supportsAutoUpdate: true, + }; +} + +export async function getPlatformCapabilities(): Promise { + if (cachedCapabilities) return cachedCapabilities; + + const isTauri = + globalThis.window !== undefined && + '__TAURI_INTERNALS__' in globalThis.window; + + if (!isTauri) { + cachedCapabilities = inferPlatformCapabilities(); + return cachedCapabilities; + } + + try { + const { invoke } = await import('@tauri-apps/api/core'); + cachedCapabilities = await invoke( + 'get_platform_capabilities', + ); + return cachedCapabilities; + } catch (err) { + console.warn( + '[platform] get_platform_capabilities unavailable; using inferred defaults', + err, + ); + cachedCapabilities = inferPlatformCapabilities(); + return cachedCapabilities; + } +} + +export function getCachedPlatformCapabilities(): PlatformCapabilities | null { + return cachedCapabilities; +} diff --git a/openless-all/app/src/lib/stylePrefs.test.ts b/openless-all/app/src/lib/stylePrefs.test.ts index dc4d43f1..122871e5 100644 --- a/openless-all/app/src/lib/stylePrefs.test.ts +++ b/openless-all/app/src/lib/stylePrefs.test.ts @@ -80,6 +80,12 @@ const previousPrefs: UserPreferences = { audioRecordingMaxEntries: null, marketplaceBaseUrl: '', marketplaceDevLogin: '', + androidInsertStrategy: 'accessibility', + androidOverlayTrigger: 'background', + androidOverlayActivationMode: 'tap', + androidOverlayLeftSwipeAction: 'translation', + androidOverlayCancelSwipeDirection: 'up', + androidOverlaySizeDp: 72, }; const nextPrefs: UserPreferences = { diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index 343c54de..0b67a230 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -2,6 +2,26 @@ // All keys are camelCase (Rust serializes with #[serde(rename_all = "camelCase")]). // PolishMode is an exception — Rust uses lowercase serialization. +import type { + AndroidAccessibilityStatus, + AndroidInsertStrategy, + AndroidOverlayActivationMode, + AndroidOverlayCancelSwipeDirection, + AndroidOverlayLeftSwipeAction, + AndroidOverlayStatus, + AndroidOverlayTrigger, +} from '../../android/frontend/lib/androidTypes'; + +export type { + AndroidAccessibilityStatus, + AndroidInsertStrategy, + AndroidOverlayActivationMode, + AndroidOverlayCancelSwipeDirection, + AndroidOverlayLeftSwipeAction, + AndroidOverlayStatus, + AndroidOverlayTrigger, +}; + export type PolishMode = 'raw' | 'light' | 'structured' | 'formal'; export type InsertStatus = 'inserted' | 'pasteSent' | 'copiedFallback' | 'failed'; @@ -78,7 +98,7 @@ export interface HotkeyBinding { keys?: HotkeyKey[] | null; } -export type HotkeyAdapterKind = 'macEventTap' | 'windowsLowLevel' | 'fcitx5'; +export type HotkeyAdapterKind = 'macEventTap' | 'windowsLowLevel' | 'fcitx5' | 'unavailable'; export interface HotkeyCapability { adapter: HotkeyAdapterKind; @@ -338,6 +358,18 @@ export interface UserPreferences { marketplaceBaseUrl: string; /** Marketplace dev-mode 模拟登录用户名(GitHub login 风格)。生产换 OAuth token 后此字段废弃。 */ marketplaceDevLogin: string; + /** Android: cross-app dictation insert strategy. */ + androidInsertStrategy: AndroidInsertStrategy; + /** Android: floating overlay visibility trigger mode. */ + androidOverlayTrigger: AndroidOverlayTrigger; + /** Android: how the floating overlay enters the armed interaction state. */ + androidOverlayActivationMode: AndroidOverlayActivationMode; + /** Android: action performed by left swiping while the overlay is armed. */ + androidOverlayLeftSwipeAction: AndroidOverlayLeftSwipeAction; + /** Android: vertical swipe direction that cancels recording. */ + androidOverlayCancelSwipeDirection: AndroidOverlayCancelSwipeDirection; + /** Android: floating overlay control diameter in dp. */ + androidOverlaySizeDp: number; } export interface MarketplaceListItem { @@ -487,3 +519,18 @@ export type PermissionStatus = | 'notDetermined' | 'restricted' | 'notApplicable'; + +/** Runtime platform kind returned by `get_platform_capabilities`. */ +export type PlatformKind = 'desktop' | 'android' | 'mobile'; + +/** Feature flags for desktop vs Android APK UI gating. Mirrors src-tauri PlatformCapabilities. */ +export interface PlatformCapabilities { + platform: PlatformKind; + supportsDesktopHotkey: boolean; + supportsTray: boolean; + supportsOverlay: boolean; + supportsImeInput: boolean; + supportsLocalAsr: boolean; + supportsInAppDictation: boolean; + supportsAutoUpdate: boolean; +} diff --git a/openless-all/app/src/lib/useMobileLayout.ts b/openless-all/app/src/lib/useMobileLayout.ts new file mode 100644 index 00000000..77a00995 --- /dev/null +++ b/openless-all/app/src/lib/useMobileLayout.ts @@ -0,0 +1,25 @@ +import { useEffect, useState } from 'react'; +import { detectOS } from '../components/WindowChrome'; + +function shouldUseMobileLayout(breakpoint: number): boolean { + if (typeof window === 'undefined') return false; + const osQuery = new URLSearchParams(window.location.search).get('os'); + return osQuery === 'android' || detectOS() === 'android' || window.innerWidth < breakpoint; +} + +export function useMobileLayout(breakpoint = 720): boolean { + const [mobile, setMobile] = useState(() => shouldUseMobileLayout(breakpoint)); + + useEffect(() => { + const sync = () => setMobile(shouldUseMobileLayout(breakpoint)); + sync(); + window.addEventListener('resize', sync); + window.addEventListener('orientationchange', sync); + return () => { + window.removeEventListener('resize', sync); + window.removeEventListener('orientationchange', sync); + }; + }, [breakpoint]); + + return mobile; +} diff --git a/openless-all/app/src/pages/History.tsx b/openless-all/app/src/pages/History.tsx index f030fe9b..de0074a2 100644 --- a/openless-all/app/src/pages/History.tsx +++ b/openless-all/app/src/pages/History.tsx @@ -10,6 +10,7 @@ import { clearHistory, deleteHistoryEntry, listHistory, readAudioRecording } fro import type { DictationSession, PolishMode } from '../lib/types'; import { useHotkeySettings } from '../state/HotkeySettingsContext'; import { Btn, Card, PageHeader, Pill } from './_atoms'; +import { useMobileLayout } from '../lib/useMobileLayout'; function useFilters(): Array<{ id: 'all' | PolishMode; label: string }> { const { t } = useTranslation(); @@ -35,6 +36,7 @@ function useModeLabel(): Record { export function History() { const { t } = useTranslation(); const os = detectOS(); + const mobile = useMobileLayout(); const FILTERS = useFilters(); const MODE_LABEL = useModeLabel(); const [filter, setFilter] = useState<'all' | PolishMode>('all'); @@ -176,8 +178,16 @@ export function History() {
} /> -
- +
+
setFilter(f.id)} style={{ padding: '3px 9px', fontSize: 11, borderRadius: 999, - border: '0.5px solid ' + (filter === f.id ? 'var(--ol-ink)' : 'var(--ol-line-strong)'), - background: filter === f.id ? 'var(--ol-ink)' : 'transparent', - color: filter === f.id ? '#fff' : 'var(--ol-ink-3)', + border: '0.5px solid ' + (filter === f.id ? 'var(--ol-pill-selected-border)' : 'var(--ol-line-strong)'), + background: filter === f.id ? 'var(--ol-pill-selected-bg)' : 'transparent', + color: filter === f.id ? 'var(--ol-pill-selected-ink)' : 'var(--ol-ink-3)', cursor: 'default', fontFamily: 'inherit', fontWeight: 500, transition: 'background 0.16s var(--ol-motion-quick), color 0.16s var(--ol-motion-quick), border-color 0.16s var(--ol-motion-quick)', }} @@ -254,16 +264,16 @@ export function History() {
- + {item ? ( <> -
-
+
+
{formatTime(item.createdAt)} {MODE_LABEL[item.mode]} {formatDuration(item.durationMs, t)}
-
+
void onCopy()}>{justCopied ? t('common.copied') : t('common.copy')} {item.hasAudioRecording && !audioMissingIds.has(item.id) && ( void onExportAudio()}>{t('history.exportRecording')} @@ -278,7 +288,7 @@ export function History() { key={item.id} /> )} -
+
{t('history.rawLabel')}

diff --git a/openless-all/app/src/pages/Overview.tsx b/openless-all/app/src/pages/Overview.tsx index 457523aa..8dadbc82 100644 --- a/openless-all/app/src/pages/Overview.tsx +++ b/openless-all/app/src/pages/Overview.tsx @@ -4,10 +4,27 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Icon } from '../components/Icon'; import { formatComboLabel } from '../lib/hotkey'; -import { getCredentials, listHistory } from '../lib/ipc'; -import type { CredentialsStatus, DictationSession, PolishMode } from '../lib/types'; +import { + checkMicrophonePermission, + getCredentials, + getPlatformCapabilities, + listHistory, + startDictation, + stopDictation, +} from '../lib/ipc'; +import type { + CapsulePayload, + CapsuleState, + CredentialsStatus, + DictationSession, + PermissionStatus, + PlatformCapabilities, + PolishMode, +} from '../lib/types'; import { useHotkeySettings } from '../state/HotkeySettingsContext'; import { Btn, Card, PageHeader, Pill } from './_atoms'; +import { useMobileLayout } from '../lib/useMobileLayout'; +import { checkAndroidMicrophoneAccess, requestAndroidMicrophoneAccess } from '../lib/androidMicrophonePermission'; function useModeLabels(): Record { const { t } = useTranslation(); @@ -53,6 +70,7 @@ const LLM_NAME_KEY_BY_ID: Record = { export function Overview({ onOpenHistory }: OverviewProps) { const { t } = useTranslation(); + const mobile = useMobileLayout(); const modeLabel = useModeLabels(); const [history, setHistory] = useState([]); const [historyError, setHistoryError] = useState(false); @@ -176,9 +194,12 @@ export function Overview({ onOpenHistory }: OverviewProps) { desc={t('overview.metricTotalTrend')} /> + + +

-
+
0 ? t('overview.metricAvgTrend') : t('overview.metricNoData')} /> @@ -204,8 +225,8 @@ export function Overview({ onOpenHistory }: OverviewProps) { {/* 底部一行 = flex:1 撑满剩余高度(父 wrapper 是 display:flex/column)。 只有「最近识别」内部允许滚动;其他卡片按内容自然高度,不破裂底部圆角。 issue #243 follow-up:去掉外层 overflow 后底部圆角被裁的视觉问题。 */} -
- +
+
{t('overview.weekTitle')} {t('overview.weekUnit')} @@ -222,12 +243,12 @@ export function Overview({ onOpenHistory }: OverviewProps) {
- +
{t('overview.recentTitle')} {t('overview.recentAll')}
-
+
{historyError ? (
{t('overview.recentLoadFailed')} @@ -261,10 +282,11 @@ interface ProviderCardProps { function ProviderCard({ kind, name, subname, status }: ProviderCardProps) { const { t } = useTranslation(); + const mobile = useMobileLayout(); // ASR 卡用 mic 图标,其他用 sparkle —— 通过比较译文判断会随语言改变,故改用本地化无关的字面量比较。 const isAsr = kind === t('overview.asrKind'); return ( - +
-
+
{kind} {status === 'configured' && ( @@ -309,13 +331,14 @@ interface MetricProps { } function Metric({ icon, label, value, trend, accent }: MetricProps) { + const mobile = useMobileLayout(); return ( - +
{label}
-
{value}
+
{value}
{trend || ' '}
); @@ -350,9 +373,10 @@ function WeekChart({ data }: { data: number[] }) { function RecentRow({ session, modeLabel }: { session: DictationSession; modeLabel: Record }) { const { t } = useTranslation(); + const mobile = useMobileLayout(); return ( -
-
+
+
{formatTime(session.createdAt)} @@ -393,3 +417,189 @@ function weekDayLabels(names: string[]): string[] { } return out; } + +function AndroidMicGrantBanner() { + const { t } = useTranslation(); + const mobile = useMobileLayout(); + const [platformCaps, setPlatformCaps] = useState(null); + const [microphone, setMicrophone] = useState('loading'); + const [busy, setBusy] = useState(false); + + useEffect(() => { + void getPlatformCapabilities().then(setPlatformCaps); + }, []); + + const refreshMic = useCallback(async () => { + if (platformCaps?.platform === 'android') { + setMicrophone(await checkAndroidMicrophoneAccess()); + return; + } + setMicrophone(await checkMicrophonePermission()); + }, [platformCaps?.platform]); + + useEffect(() => { + if (platformCaps?.platform !== 'android') return; + void refreshMic(); + const onFocus = () => { void refreshMic(); }; + window.addEventListener('focus', onFocus); + return () => window.removeEventListener('focus', onFocus); + }, [platformCaps?.platform, refreshMic]); + + if (platformCaps?.platform !== 'android') return null; + if (microphone === 'loading' || microphone === 'granted' || microphone === 'notApplicable') { + return null; + } + + const onGrant = async () => { + setBusy(true); + try { + setMicrophone(await requestAndroidMicrophoneAccess()); + } finally { + setBusy(false); + void refreshMic(); + } + }; + + return ( + +
+ +
+
+ {t('overview.androidMicBanner.title')} +
+
+ {t('overview.androidMicBanner.desc')} +
+
+
+ void onGrant()} disabled={busy} style={{ justifyContent: 'center', width: mobile ? '100%' : undefined }}> + {t('overview.androidMicBanner.grant')} + +
+ ); +} + +function InAppDictationControl() { + const { t } = useTranslation(); + const mobile = useMobileLayout(); + const [platformCaps, setPlatformCaps] = useState(null); + const [capsuleState, setCapsuleState] = useState('idle'); + const [busy, setBusy] = useState(false); + + useEffect(() => { + void getPlatformCapabilities().then(setPlatformCaps); + }, []); + + useEffect(() => { + if (!platformCaps?.supportsInAppDictation) return; + let cancelled = false; + let unlisten: (() => void) | undefined; + void (async () => { + try { + const { listen } = await import('@tauri-apps/api/event'); + const handle = await listen('capsule:state', (event) => { + if (cancelled) return; + setCapsuleState(event.payload.state); + }); + if (cancelled) { + handle(); + } else { + unlisten = handle; + } + } catch { + // browser dev mock + } + })(); + return () => { + cancelled = true; + unlisten?.(); + }; + }, [platformCaps?.supportsInAppDictation]); + + if (!platformCaps?.supportsInAppDictation) { + return null; + } + + const recording = capsuleState === 'recording'; + const processing = capsuleState === 'transcribing' || capsuleState === 'polishing'; + const statusLabel = recording + ? t('overview.inAppDictation.recording') + : processing + ? t('overview.inAppDictation.processing') + : t('overview.inAppDictation.idle'); + + const onToggle = async () => { + if (busy || processing) return; + setBusy(true); + try { + if (recording) { + await stopDictation(); + } else { + if (platformCaps?.platform === 'android') { + const current = await checkAndroidMicrophoneAccess(); + const status = current === 'granted' + ? current + : await requestAndroidMicrophoneAccess(); + if (status !== 'granted') return; + } + await startDictation(); + } + } catch (error) { + console.error('[overview] in-app dictation toggle failed', error); + } finally { + setBusy(false); + } + }; + + return ( + + +
+
+ {t('overview.inAppDictation.title')} +
+
+ {statusLabel} +
+
+ {!mobile && {statusLabel}} +
+ ); +} diff --git a/openless-all/app/src/pages/QaPanel.tsx b/openless-all/app/src/pages/QaPanel.tsx index 184e2fc3..847581ac 100644 --- a/openless-all/app/src/pages/QaPanel.tsx +++ b/openless-all/app/src/pages/QaPanel.tsx @@ -12,8 +12,16 @@ import { useEffect, useMemo, useRef, useState, type CSSProperties } from 'react'; import { useTranslation } from 'react-i18next'; -import { getSettings, isTauri, qaWindowDismiss, qaWindowPin } from '../lib/ipc'; -import type { QaChatMessage, QaStatePayload, UserPreferences } from '../lib/types'; +import { + getPlatformCapabilities, + getSettings, + isTauri, + qaSubmitText, + qaToggleRecording, + qaWindowDismiss, + qaWindowPin, +} from '../lib/ipc'; +import type { PlatformCapabilities, QaChatMessage, QaStatePayload, UserPreferences } from '../lib/types'; import { getHotkeyBindingLabel } from '../lib/hotkey'; import { renderQaMarkdown, renderQaPlainText } from '../lib/qaMarkdown'; @@ -21,17 +29,24 @@ const SELECTION_PREVIEW_MAX = 60; type Status = 'idle' | 'recording' | 'thinking' | 'error'; -export function QaPanel() { +interface QaPanelProps { + embedded?: boolean; + onRequestClose?: () => void; +} + +export function QaPanel({ embedded = false, onRequestClose }: QaPanelProps = {}) { const { t, i18n } = useTranslation(); const [messages, setMessages] = useState([]); const [status, setStatus] = useState('idle'); const [errorMsg, setErrorMsg] = useState(''); const [selectionPreview, setSelectionPreview] = useState(''); + const [composerText, setComposerText] = useState(''); const [pinned, setPinned] = useState(false); /** 流式 LLM 答案:answer_delta 累积、answer 事件来时清空(最终内容已落到 messages)。 */ const [streamingAnswer, setStreamingAnswer] = useState(''); /** 录音电平:0..1。后端每帧 33ms 通过 qa:level emit。详见 issue #162。 */ const [level, setLevel] = useState(0); + const [platformCaps, setPlatformCaps] = useState(null); /** 用户当前的录音热键 label(如 "右 Option" / "Right Alt")。issue #205: * 原版硬编码 "Option",Windows 用户没这个键,文案失真。读 prefs 后由 i18n * 插值动态显示,平台与用户配置都能跟上。 */ @@ -40,6 +55,10 @@ export function QaPanel() { ); const tRef = useRef(t); tRef.current = t; + const embeddedRef = useRef(embedded); + embeddedRef.current = embedded; + const onRequestCloseRef = useRef(onRequestClose); + onRequestCloseRef.current = onRequestClose; // ── 后端事件订阅(mount 时订阅一次,永不重订阅)────────────────── useEffect(() => { @@ -75,14 +94,18 @@ export function QaPanel() { // ASR 在 finalize、user message 还没 push 的过渡帧。提前切到 thinking // 视图避免 UI 卡 recording 几百 ms 反馈缺失。详见 issue #161。 setStatus('thinking'); - setSelectionPreview(''); + if (payload.selection_preview != null) { + setSelectionPreview(payload.selection_preview); + } setErrorMsg(''); setStreamingAnswer(''); setLevel(0); break; case 'thinking': setStatus('thinking'); - setSelectionPreview(''); + if (payload.selection_preview != null) { + setSelectionPreview(payload.selection_preview); + } setErrorMsg(''); setStreamingAnswer(''); setLevel(0); @@ -95,7 +118,6 @@ export function QaPanel() { break; case 'answer': setStatus('idle'); - setSelectionPreview(''); setErrorMsg(''); // messages 已被上面的 setMessages 落定,清掉流式 buffer 避免和最终气泡重影。 setStreamingAnswer(''); @@ -111,7 +133,13 @@ export function QaPanel() { }); const dismissHandle = await listen('qa:dismiss', () => { setPinned(false); - void qaWindowDismiss(); + setSelectionPreview(''); + setComposerText(''); + if (embeddedRef.current) { + onRequestCloseRef.current?.(); + } else { + void qaWindowDismiss(); + } }); // qa:level — 录音电平,节流 ~33ms/帧。详见 issue #162。 const levelHandle = await listen<{ level: number }>('qa:level', event => { @@ -153,11 +181,12 @@ export function QaPanel() { if (event.key === 'Escape') { event.preventDefault(); void qaWindowDismiss(); + onRequestClose?.(); } }; window.addEventListener('keydown', onKey, true); return () => window.removeEventListener('keydown', onKey, true); - }, []); + }, [onRequestClose]); // ── 读取用户当前的录音热键 label,给 i18n 插值用(issue #205)。 // QaPanel 跑在独立 webview(label="qa"),没有 HotkeySettingsContext @@ -182,6 +211,25 @@ export function QaPanel() { }; }, [i18n.language]); + useEffect(() => { + let cancelled = false; + void getPlatformCapabilities() + .then(caps => { + if (!cancelled) setPlatformCaps(caps); + }) + .catch(err => { + console.warn('[QaPanel] load platform capabilities failed', err); + }); + return () => { + cancelled = true; + }; + }, []); + + const mobileRecordButton = platformCaps?.supportsDesktopHotkey === false; + const recordControlLabel = mobileRecordButton + ? t('qa.mobileRecordLabel') + : recordHotkeyLabel; + const onTogglePin = () => { const next = !pinned; setPinned(next); @@ -190,6 +238,18 @@ export function QaPanel() { const onClose = () => { void qaWindowDismiss(); + onRequestClose?.(); + }; + + const onSubmitText = () => { + const text = composerText.trim(); + if (!text || status === 'thinking' || status === 'recording') return; + setComposerText(''); + void qaSubmitText(text).catch(error => { + console.error('[QaPanel] qa_submit_text failed', error); + setErrorMsg(error instanceof Error ? error.message : String(error)); + setStatus('error'); + }); }; // ── 自动滚动到底(新消息进来时)──────────────────────────────────── @@ -201,18 +261,18 @@ export function QaPanel() { }, [messages, status]); return ( -
- +
+
{messages.length === 0 && status === 'idle' && ( - + )} {messages.length === 0 && status === 'recording' && ( )} @@ -222,7 +282,7 @@ export function QaPanel() { t={t} preview={selectionPreview} level={level} - recordHotkey={recordHotkeyLabel} + recordHotkey={recordControlLabel} /> )} {streamingAnswer && ( @@ -232,10 +292,27 @@ export function QaPanel() { )} {status === 'error' && ( - + )}
- + {mobileRecordButton && ( + { + if (status === 'thinking') return; + void qaToggleRecording(); + }} + /> + )} + +
); } @@ -246,9 +323,10 @@ interface ToolbarProps { pinned: boolean; onTogglePin: () => void; onClose: () => void; + embedded?: boolean; } -function Toolbar({ pinned, onTogglePin, onClose }: ToolbarProps) { +function Toolbar({ pinned, onTogglePin, onClose, embedded = false }: ToolbarProps) { const { t } = useTranslation(); // 拖动 (issue #205): // - macOS: lib.rs::make_qa_window_draggable_macos 在 NSWindow 层把整窗口设 @@ -258,7 +336,7 @@ function Toolbar({ pinned, onTogglePin, onClose }: ToolbarProps) { // 两条路径并存不冲突;data-tauri-drag-region 放在 toolbar 的空白 spacer 上,IconBtn // 作为 button 子元素仍然正常 click。 return ( -
+
['t']; + onClick: () => void; +}) { + const disabled = status === 'thinking'; + const recording = status === 'recording'; + return ( +
+ +
+ ); +} + +function QaComposer({ + value, + disabled, + t, + onChange, + onSubmit, +}: { + value: string; + disabled: boolean; + t: ReturnType['t']; + onChange: (value: string) => void; + onSubmit: () => void; +}) { + const canSubmit = value.trim().length > 0 && !disabled; + return ( +
+