diff --git a/.flocks/plugins/agents/asset-survey/agent.yaml b/.flocks/plugins/agents/asset-survey/agent.yaml index 6349cb670..affe426df 100644 --- a/.flocks/plugins/agents/asset-survey/agent.yaml +++ b/.flocks/plugins/agents/asset-survey/agent.yaml @@ -1,4 +1,5 @@ name: asset-survey +name_cn: 互联网资产测绘智能体 description: >- Internet asset survey and reconnaissance agent. Translates natural language queries into structured asset mapping searches. Discovers and analyzes internet-facing assets including @@ -23,8 +24,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - websearch - webfetch - threatbook_mcp_hrti_query diff --git a/.flocks/plugins/agents/device-inspector/agent.yaml b/.flocks/plugins/agents/device-inspector/agent.yaml index 52dff1030..e5acd07ed 100644 --- a/.flocks/plugins/agents/device-inspector/agent.yaml +++ b/.flocks/plugins/agents/device-inspector/agent.yaml @@ -1,4 +1,5 @@ name: device-inspector +name_cn: 设备巡检智能体 description: >- Generic device inspection agent for connected security devices. Discovers enabled devices through device_context, dynamically finds the right device tools, and @@ -21,8 +22,7 @@ tools: - glob - grep - bash - - todoread - - todowrite + - todo - tool_search - device_context - run_slash_command diff --git a/.flocks/plugins/agents/host-forensics-fast/agent.yaml b/.flocks/plugins/agents/host-forensics-fast/agent.yaml index 787820b64..13ed2dc9e 100644 --- a/.flocks/plugins/agents/host-forensics-fast/agent.yaml +++ b/.flocks/plugins/agents/host-forensics-fast/agent.yaml @@ -1,4 +1,5 @@ name: host-forensics-fast +name_cn: 主机快速排查智能体 description: >- Fast Linux host compromise triage subagent for first-pass investigation. Use when the user wants a quick, concise, and accurate host security check, rapid cryptomining triage, or an diff --git a/.flocks/plugins/agents/host-forensics/agent.yaml b/.flocks/plugins/agents/host-forensics/agent.yaml index 4424ef1a7..2e171dbf0 100644 --- a/.flocks/plugins/agents/host-forensics/agent.yaml +++ b/.flocks/plugins/agents/host-forensics/agent.yaml @@ -1,4 +1,5 @@ name: host-forensics +name_cn: 主机取证智能体 description: >- Linux host compromise detection and forensics subagent. Use when the user asks to inspect, analyze, or investigate whether a host is compromised, cryptomining, backdoors, webshells, @@ -29,8 +30,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - tool_search - ssh_run_script - ssh_host_cmd diff --git a/.flocks/plugins/agents/hrti_threat_intelligence/agent.yaml b/.flocks/plugins/agents/hrti_threat_intelligence/agent.yaml index 82d6a828f..4e59354be 100644 --- a/.flocks/plugins/agents/hrti_threat_intelligence/agent.yaml +++ b/.flocks/plugins/agents/hrti_threat_intelligence/agent.yaml @@ -1,4 +1,5 @@ name: hrti-threat-intelligence +name_cn: 热点威胁情报智能体 description: >- Situational threat intelligence subagent for querying and analyzing trending cybersecurity events. Understands natural language queries, retrieves hot threat intelligence reports via HRTI_list_query @@ -23,8 +24,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - tool_search - threatbook_mcp_hrti_list_query - threatbook_mcp_hrti_query @@ -37,4 +37,4 @@ tools: - virustotal_file_query - threatbook_mcp_ip_query - threatbook_mcp_domain_query - - threatbook_mcp_hash_query \ No newline at end of file + - threatbook_mcp_hash_query diff --git a/.flocks/plugins/agents/ndr-analyst/agent.yaml b/.flocks/plugins/agents/ndr-analyst/agent.yaml index 2aa559723..fa3e24226 100644 --- a/.flocks/plugins/agents/ndr-analyst/agent.yaml +++ b/.flocks/plugins/agents/ndr-analyst/agent.yaml @@ -1,4 +1,5 @@ name: ndr-analyst +name_cn: NDR 分析智能体 description: >- NDR network traffic analyst: analyzes flow logs and alerts, identifies attack techniques, and determines whether attacks succeeded. @@ -22,8 +23,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - tool_search - websearch - webfetch diff --git a/.flocks/plugins/agents/phishing-detector/agent.yaml b/.flocks/plugins/agents/phishing-detector/agent.yaml index 8d027089c..386de253f 100644 --- a/.flocks/plugins/agents/phishing-detector/agent.yaml +++ b/.flocks/plugins/agents/phishing-detector/agent.yaml @@ -1,4 +1,5 @@ name: phishing-detector +name_cn: 钓鱼邮件检测智能体 description: > Specialized agent for detecting and analyzing phishing emails, extracting IOCs, and assessing threat level. @@ -20,8 +21,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - tool_search - websearch - webfetch diff --git a/.flocks/plugins/agents/ti-analyst/agent.yaml b/.flocks/plugins/agents/ti-analyst/agent.yaml index ef897787d..39913ee4f 100644 --- a/.flocks/plugins/agents/ti-analyst/agent.yaml +++ b/.flocks/plugins/agents/ti-analyst/agent.yaml @@ -1,4 +1,5 @@ name: ti-analyst +name_cn: 威胁情报智能体 description: >- Threat intelligence analyst subagent for IOC analysis, attribution, and threat actor research. Analyzes IPs, domains, and file hashes to assess threat levels, identify attack origins, @@ -24,8 +25,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - tool_search - threatbook_mcp_ip_query - threatbook_mcp_ip_attribution @@ -42,4 +42,4 @@ tools: - virustotal_url_query - virustotal_file_query - websearch - - webfetch \ No newline at end of file + - webfetch diff --git a/.flocks/plugins/agents/vul_threat_intelligence/agent.yaml b/.flocks/plugins/agents/vul_threat_intelligence/agent.yaml index ac1d33579..649e37440 100644 --- a/.flocks/plugins/agents/vul_threat_intelligence/agent.yaml +++ b/.flocks/plugins/agents/vul_threat_intelligence/agent.yaml @@ -1,4 +1,5 @@ name: vul-threat-intelligence +name_cn: 漏洞情报智能体 description: >- Vulnerability threat intelligence subagent for querying and analyzing vulnerability data. Understands natural language queries, extracts structured parameters, and retrieves accurate @@ -23,8 +24,7 @@ tools: - edit - write - bash - - todoread - - todowrite + - todo - tool_search - threatbook_mcp_vulnlist_query - threatbook_mcp_vuln_query @@ -43,4 +43,4 @@ tools: - virustotal_ip_query - virustotal_domain_query - virustotal_url_query - - virustotal_file_query \ No newline at end of file + - virustotal_file_query diff --git a/.flocks/plugins/skills/browser-use/SKILL.md b/.flocks/plugins/skills/browser-use/SKILL.md index 6bfa70dea..f409db0ab 100644 --- a/.flocks/plugins/skills/browser-use/SKILL.md +++ b/.flocks/plugins/skills/browser-use/SKILL.md @@ -1,6 +1,6 @@ --- name: browser-use -description: 统一处理浏览器使用任务，支持 CDP 直连/无头模式 使用用户本机 Chromium 系浏览器。Use when the user asks to browse websites, interact with pages, fill forms, capture screenshots, reuse an existing Chrome/Chromium/Edge login session, access internal/login-only pages, handle access-restricted content, when websearch/webfetch are unavailable, or automate browser actions. +description: 统一处理浏览器使用任务，支持可见浏览器 CDP 直连、专用 headless CDP、agent-browser。Use when the user asks to browse websites, interact with pages, fill forms, capture screenshots, reuse an existing Chrome/Chromium/Edge login session, work with an already-open browser/sidebar browser, access login-only/internal/dynamic pages, or automate browser actions. --- # Browser Use @@ -22,71 +22,56 @@ description: 统一处理浏览器使用任务，支持 CDP 直连/无头模式 | `cdp-direct` | 复用本机 Chromium 系浏览器的 CDP 直连模式 | 用户明确说用 CDP 模式| | `cdp-headless` | 通过 `BU_CDP_WS` / `BU_CDP_URL` 连接独立 headless Chromium 实例 | 只有用户明确要求 headless，或任务本身是后台任务/定时任务，或系统不支持可视化 | - - 用户明确指出模式后，直接阅读执行规则部分 - - 当用户没有明确指出使用模式时，进入下一步自动判定 - - 不要默认切到 `cdp-headless`；能用用户正常浏览器完成的任务，优先保持可见浏览器流程 +- 用户明确指出模式后，直接阅读执行规则部分 +- 当用户没有明确指出使用模式时，进入下一步自动判定 -## 自动判定模式 +## 自动判定与失败处理 -当用户没有明确指出使用模式时，按下面两步自动判定： +当用户没有明确指出使用模式时,按以下 4 步自动判定 + 失败处理: -1. 先判断本次 `cdp-direct` 是否需要 headless 浏览器实例 - 满足以下任一条件时，判定为“使用 headless 浏览器实例的 `cdp-direct` 流程”： - - 任务天然是后台执行 - - 任务属于定时任务、`CI` / `cron` - - 用户明确要求本次使用 headless 浏览器 - - 系统不支持可视化，如服务器 +### Step 1: 是否需要 headless -2. 如果判定需要 headless，则按以下顺序执行 - - 先读取 `references/cdp-headless.md` - - 优先使用显式提供的 `BU_CDP_WS` / `BU_CDP_URL` - - 不要引导用户去操作日常浏览器 profile 的 inspect 授权页 - - 如果没有显式 CDP endpoint，再按 `references/cdp-headless.md` 中当前平台对应的后台启动方式启动专用 Chromium 实例；必须让浏览器进程脱离当前 shell 独立存活，并为它分配未被占用的专用 remote debugging 端口，优先复用安装脚本设置的 `AGENT_BROWSER_EXECUTABLE_PATH` - - 连通后读取 `references/cdp-direct.md`，后续页面操作统一按 `cdp-direct` 工作流执行 +满足以下任一条件时,判定为“使用 headless 浏览器实例的 `cdp-direct` 流程”: -### 第一步：跑 CDP 可用性检测 +- 任务天然是后台执行 +- 任务属于定时任务 / `CI` / `cron` +- 用户明确要求本次使用 headless 浏览器 +- 系统不支持可视化,如服务器 -先执行： +如果判定需要 headless,则按以下顺序执行: -```bash -flocks browser --doctor -``` - -该命令会检查 `flocks browser` 的 daemon 是否可用、Chrome/Chromium/Edge 是否运行，以及当前是否有可用的浏览器连接。 - -### 第二步：根据检测结果决定模式 - -#### 结果 A：doctor 通过 +- 先读取 `references/cdp-headless.md` +- 优先使用显式提供的 `BU_CDP_WS` / `BU_CDP_URL` +- 不要引导用户去操作日常浏览器 profile 的 inspect 授权页 +- 如果没有显式 CDP endpoint,再按 `references/cdp-headless.md` 中当前平台对应的后台启动方式启动专用 Chromium 实例;必须让浏览器进程脱离当前 shell 独立存活,并为它分配未被占用的专用 remote debugging 端口,优先复用安装脚本设置的 `AGENT_BROWSER_EXECUTABLE_PATH` +- 连通后读取 `references/cdp-direct.md`,后续页面操作统一按 `cdp-direct` 工作流执行 -这时立即确定使用 `CDP 直连`，然后马上阅读： +### Step 2: 跑 CDP 可用性检测 -- `references/cdp-direct.md` +先执行: -之后只按 CDP 流程执行，不再切到 `agent-browser`。 - -#### 结果 B：浏览器已运行，但 daemon 或 active browser connection 不可用 - -必须直接提示用户： - -```text -browser: not connected — 请确保 Chrome / Chromium / Edge 已打开，然后访问对应浏览器的 inspect 页面（例如 chrome://inspect/#remote-debugging 或 edge://inspect/#remote-debugging）并勾选 Allow remote debugging +```bash +flocks browser --doctor ``` -然后等待用户进一步指示，不要直接操作。 +该命令会检查 `flocks browser` 的 daemon 是否可用、Chrome/Chromium/Edge 是否运行,以及当前是否有可用的浏览器连接。不要只看命令退出码；必须优先读取 `next action` 行，再结合 `browser running` / `daemon alive` / `active browser connections` 三行判断。 -当用户确认已开启remote-debugging后: -1. 执行 `flocks browser --setup` 触发交互式 attach，不要用短超时包装该命令 -2. 再运行 `flocks browser --doctor` 做只读确认。 -3. 如果还失败，先执行 `flocks browser --reload` 清理旧 daemon，再重新执行 `flocks browser --setup`，避免因为残留 daemon 造成干扰。 +### Step 3: 根据 doctor 输出决定模式 -如果用户在 `Windows PowerShell` 中执行 `flocks browser -c`，优先使用单行代码并用分号分隔；多行单引号字符串容易因为换行/转义处理差异而触发假失败。 +| 结果 | 触发条件 | 一线修复 | 仍失败兜底 | +|---|---|---|---| +| **A** | `next action` 以 `ready` 开头 | 立即确定 `CDP 直连`,阅读 `references/cdp-direct.md`,之后不再切到 `agent-browser` | — | +| **B** | `next action` 以 `attach` 开头 | 不要先反复 `--setup`；按输出执行 `flocks browser -c 'print(page_info())'` 或 `flocks browser -c 'print(list_tabs(include_chrome=False))'` 触发一次实际连接/观察 | 如果 `-c` 失败或仍无连接，执行 `flocks browser --reload` 清理旧 daemon，再执行 `flocks browser --setup`；setup 可能需要多次，直到用户完成浏览器 Allow/inspect 授权或错误信息稳定 | +| **C** | `next action` 以 `setup` 开头 | 先执行 `flocks browser --setup`（不包短超时），再运行 `--doctor` 确认 | 如提示 remote debugging 未启用、`DevToolsActivePort` 缺失、403 handshake 或 not live yet，再提示用户打开对应 inspect 页面并 Allow；用户确认后可多次 `--setup` | +| **D** | `next action` 提示启动浏览器或提供 endpoint | 明确告知需要先启动 Chrome/Chromium/Edge 或提供 CDP endpoint | **不**擅自降级到 curl/webfetch；坚持告知 skill 边界 | -- 如果 `--setup` 成功，随后 `--doctor` 通过：立即使用 `CDP 直连`，并立刻阅读 `references/cdp-direct.md` +### Step 4: 跨模式通用失败 -#### 结果 C：`flocks browser --doctor` 失败，或当前机器没有可用 Chrome/Chromium/Edge +| 触发条件 | 一线修复 | 仍失败兜底 | +|---|---|---| +| `cdp-headless` 启动了专用 Chromium 实例 | 记 PID + 日志 + 专用 profile 路径 | 任务结束或明确放弃才清理;**不**关闭用户提供的远程浏览器 | +| 模式已确定后用户改主意 | 重新跑 `--doctor` 走 Result A/B/C 判定 | 避免同时加载 `cdp-direct.md` + `agent-browser.md` | -说明当前环境不适合 `CDP 直连`。此时要： -- 明确告诉用户是哪一项不满足，提示需要做什么操作才能达到要求 ## 执行规则 @@ -94,7 +79,8 @@ browser: not connected — 请确保 Chrome / Chromium / Edge 已打开，然后 2. `cdp-headless` 是唯一例外：先读取 `references/cdp-headless.md` 完成浏览器启动与连接，再读取 `references/cdp-direct.md` 执行通用页面操作。 3. 在 `cdp-headless` 中，如果当前任务自己启动了专用浏览器实例，必须记录 PID / 日志 / 专用 profile，并只在任务结束或明确放弃后清理自己启动的实例；不要关闭用户提供的远程浏览器。 4. 不要同时加载 `references/cdp-direct.md` 和 `references/agent-browser.md`。 -5. `flocks browser` 的 daemon 文件固定放在 `~/.flocks/browser/`，例如 `bu.sock`、`bu.log`、`bu.pid`、`bu.port`。 +5. `flocks browser` 的 daemon 文件固定放在 `~/.flocks/browser/`,例如 `bu.sock`、`bu.log`、`bu.pid`、`bu.port`。 +6. 基础操作能力（打开、观察、点击、输入、滚动、截图、提取、等待、关闭）优先按 `references/cdp-direct.md` 的“基础操作速查”执行 ## 产品经验Skill diff --git a/.flocks/plugins/skills/browser-use/references/cdp-direct.md b/.flocks/plugins/skills/browser-use/references/cdp-direct.md index 51c58f13f..0ab0de7ec 100644 --- a/.flocks/plugins/skills/browser-use/references/cdp-direct.md +++ b/.flocks/plugins/skills/browser-use/references/cdp-direct.md @@ -32,141 +32,245 @@ flocks browser --doctor 通过`flocks browser -c '...'` 操作浏览器 -## 语法说明 +## 基础操作速查 -- `flocks browser -c '...'` 执行的是一段 Python 代码，不是交互式 REPL；如果希望看到结果，必须显式 `print(...)`。 -- 多行代码请直接写成真正的多行 shell 字符串或 heredoc；不要把 `\n` 当字面量塞进单引号参数里。 -- 在 `Windows PowerShell` 中，默认写法是：把整段 Python 代码放进一对外层双引号里，并尽量压成单行，用分号分隔语句。 -- 在 `Windows PowerShell` 中，内层字符串尽量统一改用单引号，例如 `js('document.body.innerText.slice(0, 5000)')`；这样可以减少外层双引号、内层双引号互相打架导致的截断或变形。 -- 如果代码里本身包含很多引号、反引号、`$`，或者已经长到不适合单行，先写到临时 `.py` 文件，再用 `Get-Content -Raw` 读出后传给 `-c`；不要硬拼多行单引号字符串。 +这些操作对应本地 `browser-use` skill 的常用能力。这里仅提供命令模板；执行顺序和选择原则见“主流程”。 -Windows PowerShell 推荐示例： +### 打开页面或复用目标 tab -```powershell -flocks browser -c "r = js('document.body.innerText.slice(0, 5000)'); print(r)" +```bash +flocks browser -c ' +tid = open_or_attach_tab("https://example.com", activate=True) +wait_for_load() +print({"targetId": tid, "page": page_info()}) +' ``` -Windows PowerShell 多行代码推荐写法： +如果用户说页面已经在浏览器或侧边栏打开，先列出非内部页并选择目标 tab；不要直接新开 headless： -```powershell -@' -tid = new_tab("https://example.com", activate=True) -wait_for_load() -print(page_info()) -'@ | Set-Content "$env:TEMP\flocks-browser-cmd.py" +```bash +flocks browser -c ' +for tab in list_tabs(include_chrome=False): + print(tab) +' +``` -flocks browser -c (Get-Content -Raw "$env:TEMP\flocks-browser-cmd.py") +确认目标后： + +```bash +flocks browser -c ' +attach_tab("") +print(page_info()) +' ``` -## 核心操作循环 -> 打开页面 -> 观察当前状态 -> 执行动作 -> 再观察验证 +### 观察页面状态 -1. 新建或打开已打开 tab -2. 用 `page_info()` / `js(...)` 观察当前页面、URL、文本、结构和阻塞状态 -3. 选择当前最稳妥的动作方式 -4. 动作后重新观察，确认页面是否真的变化 -5. 如果未达成目标，先解释当前卡点，再换一种方式继续 +```bash +flocks browser -c ' +print(page_info()) +print(js("document.body.innerText.slice(0, 4000)")) +' +``` +查找可点击元素： -## 标准工作流 +```bash +flocks browser -c ' +items = js(""" +Array.from(document.querySelectorAll("a,button,input,textarea,select,[role=button],[onclick]")) + .slice(0, 80) + .map((el, i) => ({ + i, + tag: el.tagName, + text: (el.innerText || el.value || el.getAttribute("aria-label") || el.title || "").trim().slice(0, 120), + selector: el.id ? "#" + el.id : el.name ? el.tagName.toLowerCase() + "[name=" + JSON.stringify(el.name) + "]" : null, + disabled: !!el.disabled, + rect: (() => { const r = el.getBoundingClientRect(); return {x:r.x,y:r.y,w:r.width,h:r.height}; })() + })) +""") +print(items) +' +``` + +### 点击、输入与按键 -1. 如果是具体网站/产品任务，先搜索已存在的skill，看 `<产品>-use` skill 下是否存在浏览器相关操作；如果没有，再自己探索。 -2. 创建自己的 tab，保存 `targetId`，等待加载，并先读取页面基础状态： +优先用稳定 DOM 操作： ```bash flocks browser -c ' -tid = new_tab("https://example.com", activate=True) -wait_for_load() +js("document.querySelector(\"button[type=submit]\")?.click()") +wait(0.5) print(page_info()) ' ``` -如需进一步观察页面结构、文本或候选交互元素，再按当前站点实际情况用 `js(...)` 做针对性提取； - -后续步骤继续使用同一个 tab 时，先恢复它： +输入字段必须触发 `input` / `change` 事件： ```bash flocks browser -c ' -attach_tab("") +js(""" +const el = document.querySelector("input[name=q], textarea[name=q]"); +el.value = "search text"; +el.dispatchEvent(new Event("input", {bubbles: true})); +el.dispatchEvent(new Event("change", {bubbles: true})); +""") +press_key("Enter") +wait(0.5) print(page_info()) ' ``` -3. 执行动作并验证。能稳定定位 DOM 时，优先用 `js(...)`；必须操作可见但 DOM 难以稳定定位的控件时，再使用 `click_at_xy(...)`、`type_text(...)`、`press_key(...)`： +只有 DOM 难以稳定定位时，才退到坐标： ```bash flocks browser -c ' -js("document.querySelector(\"button[type=submit]\")?.click()") +click_at_xy(420, 315) +type_text("text") +press_key("Enter") wait(0.5) print(page_info()) ' ``` -4. 需要提取数据时用 `js(...)` 或 `http_get(...)`。静态页面/接口批量抓取优先 `http_get`，不要浪费浏览器。 -5. 结束时只关闭自己创建的 tab。若不确定 tab 是否属于自己，保留它并说明；如果当前任务还临时拉起了专用 headless 浏览器实例，tab 清理完后再按 `references/cdp-headless.md` 的约定决定是否关闭整个浏览器进程。 +### 滚动、等待与截图 -注意： +```bash +flocks browser -c ' +scroll(500, 500, dy=-800) +wait(0.5) +print(page_info()) +' +``` -- 每次点击、输入、提交、弹窗处理、导航、滚动或重渲染后，都重新调用 `page_info()` 或 `js(...)`。 -- 页面变化后，之前读取到的元素状态、坐标和文本都可能过期，必须重新观察。 -- 提取大量结构化数据时，优先在页面内用 `js(...)` 组装 JSON 后返回。 -- 判断内容是否已在 DOM 中，不要只依赖当前可见区域；懒加载或虚拟列表再配合 `scroll(...)` 分段读取。 +等待指定文本或选择器时，用短轮询，避免盲等： +```bash +flocks browser -c ' +import time +deadline = time.time() + 10 +while time.time() < deadline: + if js("document.body.innerText.includes(\"Success\")"): + print("found") + break + wait(0.5) +else: + print("not found") +' +``` -## Tab 与可见性 +截图只在需要视觉证据或调试时保存： -- `new_tab(url, activate=True)` 会创建并 attach 到新 tab，默认同时让 tab 在浏览器中可见；这是需要用户登录或观察页面时的默认入口。 -- `open_or_attach_tab(url, activate=True)` 只会复用当前任务自己创建过的同 URL tab；不会按 URL 复用用户已有 tab。 -- `new_tab(url, activate=False)` 会创建后台 tab 并 attach，不主动抢当前可见 tab。 -- `attach_tab(target_id)` 只 attach 到目标 tab，不激活浏览器 UI；后续读取页面状态、导出数据、保存认证状态等命令优先使用它。 -- `switch_tab(target_id)` 会 attach 到目标 tab 并执行 `Target.activateTarget`，让目标 tab 在浏览器中可见；只在需要用户看到或手动操作时使用。 -- `managed_tabs()` 只列出当前任务创建并仍然存在的 tab，可用于调试和确认复用目标。 -- `close_tab(target_id, activate_next=False)` 默认只允许关闭自己创建的 tab，且不自动切到其他已打开 tab；确实需要关闭用户 tab 时必须显式传 `allow_unmanaged=True`。 -- `list_tabs()` 默认会包含 `chrome://`、`about:` 等内部页面；要面向用户页面时用 `list_tabs(include_chrome=False)`。 -- 忽略 `chrome://omnibox-popup.top-chrome/` 这类假 page target。页面 `w=0 h=0` 时通常是 attach 到了错误 target。 -- 当当前 session stale、内部页或不可见，并且确实要恢复到某个用户页面时，先 `ensure_real_tab()`；它会先 attach 到非内部页而不是主动激活浏览器 UI。 +```bash +flocks browser -c ' +print(capture_screenshot("/tmp/browser-use-shot.png", full=False, max_dim=1800)) +' +``` -## 读取、定位与执行 +### 提取数据 -本节的核心不是罗列所有操作方式，而是确定优先级：先读清楚，再选最稳的执行方式。 +提取文本、HTML、链接或结构化数据时，优先在页面内组装 JSON： -默认先用 `page_info()` 与 `js(...)` 观察页面，不依赖截图： +```bash +flocks browser -c ' +print(js("document.title")) +print(js("location.href")) +print(js("document.body.innerText.slice(0, 8000)")) +' +``` -```python -print(page_info()) -print(js("document.body.innerText.slice(0, 2000)")) +```bash +flocks browser -c ' +rows = js(""" +Array.from(document.querySelectorAll("a[href]")).map(a => ({ + text: a.innerText.trim(), + href: a.href +})).filter(x => x.text || x.href).slice(0, 200) +""") +print(rows) +' ``` -推荐顺序： +静态资源或接口可直接访问时，优先 `http_get(url)`，不要浪费浏览器上下文。 -1. 先用 `page_info()` 看 URL、标题、滚动位置、页面尺寸，以及是否被原生对话框阻塞 -2. 再用 `js(...)` 读取文本、DOM 结构、元素状态、业务字段 -3. 能稳定定位 DOM 时，优先直接在页面内执行 JS 或 raw CDP -4. 只有 DOM 难以稳定定位，或需要操作 compositor 级控件时，才退到坐标点击 -5. iframe / 特殊 target 场景，再考虑 `iframe_target(...)` 配合 `js(..., target_id=...)` +### 关闭当前任务资源 -能稳定定位 DOM 时，优先通过页面内 JS 或 selector 相关能力操作；需要穿过 iframe、shadow DOM、cross-origin 或自定义控件时，再使用 compositor 级坐标点击： +只关闭自己创建或确认属于本任务的 tab： -```python -click_at_xy(x, y) +```bash +flocks browser -c ' +close_tab("", activate_next=False) +' ``` -坐标点击前尽量用 DOM 布局信息确认目标位置，例如 `getBoundingClientRect()`。坐标点击不稳定、目标不可见或需要隐藏 input 时，改用 DOM、iframe target 或 raw CDP。 +不确定是否是用户原有 tab 时，保留 tab 并说明原因。 -截图仅作为可选调试产物： +## 语法说明 -```python -capture_screenshot("/tmp/shot.png", max_dim=1800) +- `flocks browser -c '...'` 执行的是一段 Python 代码，不是交互式 REPL；如果希望看到结果，必须显式 `print(...)`。 +- 多行代码请直接写成真正的多行 shell 字符串或 heredoc；不要把 `\n` 当字面量塞进单引号参数里。 +- 在 `Windows PowerShell` 中，默认写法是：把整段 Python 代码放进一对外层双引号里，并尽量压成单行，用分号分隔语句。 +- 在 `Windows PowerShell` 中，内层字符串尽量统一改用单引号，例如 `js('document.body.innerText.slice(0, 5000)')`；这样可以减少外层双引号、内层双引号互相打架导致的截断或变形。 +- 如果代码里本身包含很多引号、反引号、`$`，或者已经长到不适合单行，先写到临时 `.py` 文件，再用 `Get-Content -Raw` 读出后传给 `-c`；不要硬拼多行单引号字符串。 + +Windows PowerShell 推荐示例： + +```powershell +flocks browser -c "r = js('document.body.innerText.slice(0, 5000)'); print(r)" ``` -如果启用点击调试： +Windows PowerShell 多行代码推荐写法： -```bash -flocks browser --debug-clicks -c ' -click_at_xy(420, 315) -' +```powershell +@' +tid = new_tab("https://example.com", activate=True) +wait_for_load() +print(page_info()) +'@ | Set-Content "$env:TEMP\flocks-browser-cmd.py" + +flocks browser -c (Get-Content -Raw "$env:TEMP\flocks-browser-cmd.py") ``` +## 主流程 + +按同一个循环执行：准备目标 tab -> 观察页面 -> 选择动作 -> 执行 -> 再观察验证。不要凭旧状态继续操作。 + +1. 如果是具体网站/产品任务，先搜索已存在的 skill，看 `<产品>-use` skill 下是否存在浏览器相关操作；如果没有，再自己探索。 +2. 创建自己的 tab，或在用户明确要求继续当前页面时先列出并 attach 目标 tab。保存 `targetId`，等待加载，并读取页面基础状态。后续步骤继续使用同一个 tab 时，优先 `attach_tab(target_id)`，不要反复 `switch_tab(...)` 抢用户焦点。 + +3. 先用 `page_info()` 看 URL、标题、滚动位置、页面尺寸和对话框阻塞状态，再用 `js(...)` 读取文本、DOM 结构、元素状态或业务字段。 +4. 能稳定定位 DOM 时，优先直接在页面内执行 JS 或 raw CDP；只有 DOM 难以稳定定位，或需要操作 compositor 级控件时，才退到坐标点击。iframe / 特殊 target 场景，再考虑 `iframe_target(...)` 配合 `js(..., target_id=...)`。 + +5. 坐标点击前用 DOM 布局信息确认目标位置，例如 `getBoundingClientRect()`。坐标点击不稳定、目标不可见或需要隐藏 input 时，改用 DOM、iframe target 或 raw CDP。 +6. 需要提取数据时用 `js(...)` 或 `http_get(...)`。提取大量结构化数据时，优先在页面内用 `js(...)` 组装 JSON 后返回；静态页面/接口批量抓取优先 `http_get`，不要浪费浏览器。 +7. 判断内容是否已在 DOM 中，不要只依赖当前可见区域；懒加载或虚拟列表再配合 `scroll(...)` 分段读取。 +8. 结束时只关闭自己创建的 tab。若不确定 tab 是否属于自己，保留它并说明；如果当前任务还临时拉起了专用 headless 浏览器实例，tab 清理完后再按 `references/cdp-headless.md` 的约定决定是否关闭整个浏览器进程。 + + +## Tab 与可见性 + +- `new_tab(url, activate=True)` 会创建并 attach 到新 tab，默认同时让 tab 在浏览器中可见；这是需要用户登录或观察页面时的默认入口。 +- `open_or_attach_tab(url, activate=True)` 只会复用当前任务自己创建过的同 URL tab；不会按 URL 复用用户已有 tab。 +- `new_tab(url, activate=False)` 会创建后台 tab 并 attach，不主动抢当前可见 tab。 +- `attach_tab(target_id)` 只 attach 到目标 tab，不激活浏览器 UI；后续读取页面状态、导出数据、保存认证状态等命令优先使用它。 +- `switch_tab(target_id)` 会 attach 到目标 tab 并执行 `Target.activateTarget`，让目标 tab 在浏览器中可见；只在需要用户看到或手动操作时使用。 +- `managed_tabs()` 只列出当前任务创建并仍然存在的 tab，可用于调试和确认复用目标。 +- `close_tab(target_id, activate_next=False)` 默认只允许关闭自己创建的 tab，且不自动切到其他已打开 tab；确实需要关闭用户 tab 时必须显式传 `allow_unmanaged=True`。 +- `list_tabs()` 默认会包含 `chrome://`、`about:` 等内部页面；要面向用户页面时用 `list_tabs(include_chrome=False)`。 +- 忽略 `chrome://omnibox-popup.top-chrome/` 这类假 page target。页面 `w=0 h=0` 时通常是 attach 到了错误 target。 +- 当当前 session stale、内部页或不可见，并且确实要恢复到某个用户页面时，先 `ensure_real_tab()`；它会先 attach 到非内部页而不是主动激活浏览器 UI。 + +## 表单填充与点击操作模式 + +### 字段类型（js 优先） + +| 字段 | 推荐 | 备注 | +|---|---|---| +| text / email / tel / password / textarea | `js("el=document.querySelector('[name=x]');el.value='...';el.dispatchEvent(new Event('input'))")` | **必须** dispatchEvent | +| radio / checkbox | `js("document.querySelector('[name=size][value=small]').click()")` | `click_at_xy` 不可靠 | +| select | `js("el=document.querySelector('[name=size]');el.value='small';el.dispatchEvent(new Event('change'))")` | 必须 dispatch change | +| file | `upload_file('input[type=file]', '/abs/path')` | 唯一方式 | + ## 对话框与阻塞状态 浏览器原生 `alert`、`confirm`、`prompt`、`beforeunload` 会冻结 JS 线程。动作后如果 `page_info()` 返回 `{"dialog": ...}`，先处理对话框： @@ -244,19 +348,21 @@ print({"cookies": [c["name"] for c in cookies], "localStorage": js("Object.keys( 如果 `flocks browser` 不可用或连接失败： -1. 先运行 `flocks browser --doctor` 看版本、安装模式、daemon 和浏览器状态。 -2. 首次安装或冷启动优先运行 `flocks browser --setup`。 -3. Chrome / Chromium / Edge 未运行时只启动浏览器，再重试；不要直接让用户改设置。 -4. 只有在明确提示 remote debugging 未启用或 `DevToolsActivePort` 缺失时，才让用户打开对应浏览器的 inspect 页面（例如 `chrome://inspect/#remote-debugging` 或 `edge://inspect/#remote-debugging`）并勾选 Allow remote debugging。 -5. 用户刚开启 remote debugging 时，不要立刻再次运行 `flocks browser --doctor`；先执行一次 `flocks browser --setup`，或直接执行 `flocks browser -c 'print(page_info())'` 触发 daemon attach，再用 `--doctor` 做只读确认。 -6. `connection refused`、`DevTools not live yet`、`/json/version` 404 通常是浏览器正在启动，轮询等待，不要重启。 -7. stale websocket / stale socket 时执行一次： +1. 先运行 `flocks browser --doctor` 看版本、安装模式、daemon 和浏览器状态；不要只看退出码，优先读 `next action`，再看 `browser running`、`daemon alive`、`active browser connections`。 +2. `next action` 为 `attach`，或 `daemon alive` ok 但 `active browser connections` 为 0 时，不要先反复 `--setup`。先用一次实际命令触发连接/观察：`flocks browser -c 'print(page_info())'` 或 `flocks browser -c 'print(list_tabs(include_chrome=False))'`。 +3. 如果上一步失败或仍无连接，再执行 `flocks browser --reload` 清旧 daemon，然后执行 `flocks browser --setup`；setup 可能需要多次，因为用户可能需要完成浏览器 inspect/Allow 授权，或浏览器需要时间写入 remote debugging 状态。 +4. 首次安装、冷启动、daemon 不存在/不通，且浏览器已经运行或配置了 `BU_CDP_URL` / `BU_CDP_WS` 时，优先运行 `flocks browser --setup`。 +5. Chrome / Chromium / Edge 未运行且没有显式 CDP endpoint 时，只提示用户启动浏览器或提供 endpoint；不要直接让用户改设置。 +6. 只有在明确提示 remote debugging 未启用、`DevToolsActivePort` 缺失、403 handshake、remote-debugging page 或 not live yet 时，才让用户打开对应浏览器的 inspect 页面（例如 `chrome://inspect/#remote-debugging` 或 `edge://inspect/#remote-debugging`）并勾选 Allow remote debugging。 +7. 用户刚开启 remote debugging 时，不要立刻再次运行 `flocks browser --doctor`；先执行一次 `flocks browser --setup`，或直接执行 `flocks browser -c 'print(page_info())'` 触发 daemon attach，再用 `--doctor` 做只读确认。 +8. `connection refused`、`DevTools not live yet`、`/json/version` 404 通常是浏览器正在启动，轮询等待，不要重启。 +9. stale websocket / stale socket 时执行一次： ```bash flocks browser -c 'restart_daemon()' ``` -8. 页面操作异常但连接本身正常时，优先回到上面的“页面操作排障”，不要把所有问题都归因为浏览器没连上。 +10. 页面操作异常但连接本身正常时，优先回到上面的“页面操作排障”，不要把所有问题都归因为浏览器没连上。 ## The self-heal loop 自修复/扩展 循环 diff --git a/.flocks/plugins/skills/browser-use/references/cdp-setup.md b/.flocks/plugins/skills/browser-use/references/cdp-setup.md index d3b0f3ed8..3f72f5825 100644 --- a/.flocks/plugins/skills/browser-use/references/cdp-setup.md +++ b/.flocks/plugins/skills/browser-use/references/cdp-setup.md @@ -1,8 +1,17 @@ # Flocks browser setup -浏览器已运行，但 daemon 或 active browser connection 不可用 +浏览器已运行，但 daemon 不存在/不通，或 active browser connection 不可用 -必须直接提示用户： +先区分两种情况： + +1. `daemon alive` ok 但 `active browser connections` 为 0： + - 不要先反复执行 `flocks browser --setup`，因为 setup 在 daemon 已运行且协议正常时可能直接输出 nothing to do。 + - 先执行 `flocks browser -c 'print(page_info())'` 或 `flocks browser -c 'print(list_tabs(include_chrome=False))'` 触发一次实际连接/观察。 + - 如果仍失败，再执行 `flocks browser --reload` 清旧 daemon，然后执行 `flocks browser --setup`。 +2. daemon 不存在/不通，且浏览器已运行或配置了 `BU_CDP_URL` / `BU_CDP_WS`： + - 执行 `flocks browser --setup` 触发 attach，不要用短超时包装该命令。 + +只有在错误明确指向 remote debugging 未启用、`DevToolsActivePort` 缺失、403 handshake 或 not live yet 时，才提示用户： ```text browser: not connected — 请确保 Chrome / Chromium / Edge 已打开，然后访问对应浏览器的 inspect 页面（例如 chrome://inspect/#remote-debugging 或 edge://inspect/#remote-debugging）并勾选 Allow remote debugging @@ -13,4 +22,4 @@ browser: not connected — 请确保 Chrome / Chromium / Edge 已打开，然后 当用户确认已开启remote-debugging后: 1. 执行 `flocks browser --setup` 触发交互式 attach，不要用短超时包装该命令 2. 再运行 `flocks browser --doctor` 做只读确认。 -3. 如果还失败，先执行 `flocks browser --reload` 清理旧 daemon，再重新执行 `flocks browser --setup`，避免因为残留 daemon 造成干扰。 \ No newline at end of file +3. 如果还失败，先执行 `flocks browser --reload` 清理旧 daemon，再重新执行 `flocks browser --setup`，避免因为残留 daemon 造成干扰。setup 可能需要多次，直到用户完成浏览器 Allow/inspect 授权或错误信息稳定。 diff --git a/.flocks/plugins/skills/find-skills/SKILL.md b/.flocks/plugins/skills/find-skills/SKILL.md deleted file mode 100644 index e9ff76030..000000000 --- a/.flocks/plugins/skills/find-skills/SKILL.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -name: find-skills -description: Helps users discover and install agent skills when they ask questions like "how do I do X", "find a skill for X", "is there a skill that can...", or express interest in extending capabilities. This skill should be used when the user is looking for functionality that might exist as an installable skill. -category: system ---- - -# Find Skills - -This skill helps you discover and install skills from the open agent skills ecosystem. - -## When to Use This Skill - -Use this skill when the user: - -- Asks "how do I do X" where X might be a common task with an existing skill -- Says "find a skill for X" or "is there a skill for X" -- Asks "can you do X" where X is a specialized capability -- Expresses interest in extending agent capabilities -- Wants to search for tools, templates, or workflows -- Mentions they wish they had help with a specific domain (design, testing, deployment, etc.) - -## What is the Skills CLI? - -The Skills CLI (`npx skills`) is the package manager for the open agent skills ecosystem. Skills are modular packages that extend agent capabilities with specialized knowledge, workflows, and tools. - -**Key commands:** - -- `npx skills find [query]` - Search for skills interactively or by keyword -- `npx skills add ` - Install a skill from GitHub or other sources -- `npx skills check` - Check for skill updates -- `npx skills update` - Update all installed skills - -**Browse skills at:** https://skills.sh/ - -## How to Help Users Find Skills - -### Step 1: Understand What They Need - -When a user asks for help with something, identify: - -1. The domain (e.g., React, testing, design, deployment) -2. The specific task (e.g., writing tests, creating animations, reviewing PRs) -3. Whether this is a common enough task that a skill likely exists - -### Step 2: Check the Leaderboard First - -Before running a CLI search, check the [skills.sh leaderboard](https://skills.sh/) to see if a well-known skill already exists for the domain. The leaderboard ranks skills by total installs, surfacing the most popular and battle-tested options. - -For example, top skills for web development include: -- `vercel-labs/agent-skills` — React, Next.js, web design (100K+ installs each) -- `anthropics/skills` — Frontend design, document processing (100K+ installs) - -### Step 3: Search for Skills - -If the leaderboard doesn't cover the user's need, run the find command: - -```bash -npx skills find [query] -``` - -For example: - -- User asks "how do I make my React app faster?" → `npx skills find react performance` -- User asks "can you help me with PR reviews?" → `npx skills find pr review` -- User asks "I need to create a changelog" → `npx skills find changelog` - -### Step 4: Verify Quality Before Recommending - -**Do not recommend a skill based solely on search results.** Always verify: - -1. **Install count** — Prefer skills with 1K+ installs. Be cautious with anything under 100. -2. **Source reputation** — Official sources (`vercel-labs`, `anthropics`, `microsoft`) are more trustworthy than unknown authors. -3. **GitHub stars** — Check the source repository. A skill from a repo with <100 stars should be treated with skepticism. - -### Step 5: Present Options to the User - -When you find relevant skills, present them to the user with: - -1. The skill name and what it does -2. The install count and source -3. The install command they can run -4. A link to learn more at skills.sh - -Example response: - -``` -I found a skill that might help! The "react-best-practices" skill provides -React and Next.js performance optimization guidelines from Vercel Engineering. -(185K installs) - -To install it: -npx skills add vercel-labs/agent-skills@react-best-practices - -Learn more: https://skills.sh/vercel-labs/agent-skills/react-best-practices -``` - -### Step 6: Offer to Install - -If the user wants to proceed, you can install the skill for them: - -```bash -npx skills add -g -y -``` - -The `-g` flag installs globally (user-level) and `-y` skips confirmation prompts. - -## Common Skill Categories - -When searching, consider these common categories: - -| Category | Example Queries | -| --------------- | ---------------------------------------- | -| Web Development | react, nextjs, typescript, css, tailwind | -| Testing | testing, jest, playwright, e2e | -| DevOps | deploy, docker, kubernetes, ci-cd | -| Documentation | docs, readme, changelog, api-docs | -| Code Quality | review, lint, refactor, best-practices | -| Design | ui, ux, design-system, accessibility | -| Productivity | workflow, automation, git | - -## Tips for Effective Searches - -1. **Use specific keywords**: "react testing" is better than just "testing" -2. **Try alternative terms**: If "deploy" doesn't work, try "deployment" or "ci-cd" -3. **Check popular sources**: Many skills come from `vercel-labs/agent-skills` or `ComposioHQ/awesome-claude-skills` - -## When No Skills Are Found - -If no relevant skills exist: - -1. Acknowledge that no existing skill was found -2. Offer to help with the task directly using your general capabilities -3. Suggest the user could create their own skill with `npx skills init` - -Example: - -``` -I searched for skills related to "xyz" but didn't find any matches. -I can still help you with this task directly! Would you like me to proceed? - -If this is something you do often, you could create your own skill: -npx skills init my-xyz-skill -``` diff --git a/.flocks/plugins/skills/user-defined-page-builder/SKILL.md b/.flocks/plugins/skills/user-defined-page-builder/SKILL.md new file mode 100644 index 000000000..d780e404f --- /dev/null +++ b/.flocks/plugins/skills/user-defined-page-builder/SKILL.md @@ -0,0 +1,523 @@ +--- +name: user-defined-page-builder +category: system +description: Guide users to create, develop, hide, or delete user-defined custom pages that appear in the WebUI left navigation under Home, with live preview and no restart required. Also guide development of page-scoped backend APIs through the User Defined Page Backend API Runtime when built-in APIs are insufficient. Trigger when the user asks to create, remove, or delete a custom page, user-defined page, dashboard, navigation tab, integrate custom APIs for a page, or sends messages such as "create a custom page", "delete custom page", "remove user-defined page", "创建自定义页面", "删除自定义页面", "用户自定义页面", "自定义页面", "左侧导航页面", "首页下面的页面", "页面数据来源", "自定义 API", or wants help understanding how custom pages work in Flocks. +--- + +# User Defined Page Builder + +When the user wants to create **user-defined custom pages** (shown in the WebUI left navigation under **Home**), first explain the feature clearly, then guide them through creation and development. + +## Core Principles + +- **Language**: Detect the user's language from their messages or UI locale. Conduct the **entire conversation in the user's language** (Chinese or English). Do not switch languages mid-session. +- **Admin-required notice**: Creating, editing, hiding, deleting, importing, or exporting user-defined pages requires administrator privileges. Before starting any write workflow, remind the user that the operation must be performed by an admin. This skill does **not** verify the user's role; WebUI visibility and backend APIs enforce authorization. +- **Explain before acting**: If the user only asks what the feature is, explain fully before creating anything. +- **Confirm once**: Before creating, confirm `pageId` (lowercase English + hyphens), `title` (navigation label in the user's language), and optional `icon` (Lucide icon name). +- **User space only**: Read and write only under `~/.flocks/plugins/user_defined_pages/`. +- **Final location check**: After finishing any page development, verify that all user-defined page files are stored under `~/.flocks/plugins/user_defined_pages//`. They must **not** remain in the project code directories such as `webui/`, `flocks/`, `tests/`, or `docs/`. +- **SDK only**: Page code may import only `react` and `@flocks/user-defined-page-sdk` (`Card`, `api`, `useCurrentUser`). +- **Never write `dist/`**: Build artifacts are generated automatically. +- **Auth-aware**: All `/api/user-defined-pages/*` routes require authentication. Prefer **direct file writes** for Rex; use API Token only when calling HTTP from non-browser clients. Never embed tokens in page source. +- **Page-scoped backend**: When built-in `/api/*` is insufficient, use the User Defined Page Backend API Runtime design: page APIs live under the page directory and are exposed only at `/api/user-defined-pages//api/*`. + +## Authentication + +Flocks protects **all HTTP API paths by default** (including `/api/user-defined-pages/*`). Only bootstrap, static assets, and a few public endpoints are exempt. Understand who needs what credential: + +### WebUI (browser) + +- User must be **logged in** (session cookie `flocks_session`). +- The WebUI axios client sends cookies automatically (`withCredentials: true`). +- Navigation, page host, bundle loading, and in-page `api` calls all reuse this session — **no extra token setup** for end users. +- If the user is not logged in or the session expired, user-defined pages and related APIs return **401**. + +### Rex / Agent (recommended: file writes, no HTTP auth) + +When creating or editing pages, first remind the user that the operation requires admin privileges, then **write files directly** under `~/.flocks/plugins/user_defined_pages//`: + +- No HTTP request → no API Token needed. +- The file watcher detects changes, rebuilds, and publishes SSE events automatically. +- This is the **preferred path** for Rex in chat sessions. +- This skill does not perform role verification; WebUI and backend API paths are responsible for enforcing admin-only management. + +### Rex / Agent (optional: HTTP API) + +Use the REST API only when file-editing access is unavailable or you need an explicit build trigger. Page management APIs require admin privileges. `curl`, Python `httpx`/`requests`, and other **non-browser** clients **must** carry an API Token — even on `127.0.0.1`. + +**Token location**: `~/.flocks/config/.secret.json`, secret id `server_api_token`. + +**Generate or rotate** (on the Flocks server): + +```bash +flocks admin generate-api-token +``` + +**Configure on a remote client** (same token value): + +```bash +flocks admin set-api-token --token +``` + +**Read token in Python** (when Rex runs a script inside Flocks): + +```python +from flocks.security import get_secret_manager +from flocks.server.auth import API_TOKEN_SECRET_ID + +token = get_secret_manager().get(API_TOKEN_SECRET_ID) +``` + +**Request headers** (either works): + +```text +Authorization: Bearer +X-Flocks-API-Token: +``` + +All `curl` examples in this skill use `Authorization: Bearer ` — substitute the real token from the secret store. Do **not** ask the user to paste the token into chat; read it from the secret file or use file writes instead. + +API Token authenticates as a synthetic **admin service identity** (`api-token-service`). It is for automation, not for end-user page rendering. + +### Custom page code (`@flocks/user-defined-page-sdk` `api`) + +- The SDK `api` helper is the WebUI axios client — it sends the **logged-in user's session cookie**, not an API Token. +- Page code may call other `/api/*` endpoints (alerts, sessions, etc.) while the user is logged in. +- **Never** hardcode `server_api_token` or any secret inside `src/Page.tsx` or other page source; tokens would be exposed in the bundle. + +### Explain to users (first reply / when asked) + +**Chinese example**: + +> 自定义页面相关接口都需要登录鉴权。普通用户可以查看和使用已发布页面，但创建、修改、隐藏、删除、导入或导出页面需要管理员权限。我（Rex）在开始这类写操作前会提醒需要管理员操作，通常直接读写 `~/.flocks/plugins/user_defined_pages/` 目录，不经过 HTTP。若用脚本调管理 API，需在服务端配置 `server_api_token` 并在请求头携带 Bearer Token。 + +**English example**: + +> User-defined page APIs require authentication. Regular users can view and use published pages, but creating, editing, hiding, deleting, importing, or exporting pages requires admin privileges. I (Rex) remind the user before starting these write operations and usually read/write `~/.flocks/plugins/user_defined_pages/` directly without HTTP. Non-browser management API clients must send a Bearer API Token from `server_api_token` in `~/.flocks/config/.secret.json`. + +## First Reply Must Cover + +Explain these points in the user's language: + +1. **What it is**: Custom React pages under the Home section of the left navigation — for alert dashboards, asset views, duty screens, etc. +2. **Where files live**: `~/.flocks/plugins/user_defined_pages//` in the user space, **not** in the project code directory. +3. **How it appears**: After creation, a nav item shows under Home; route is `/user-defined-pages/`. +4. **How to develop**: Describe requirements in chat; you write `src/Page.tsx`; saving triggers auto-build; **no restart** required. +5. **Live updates**: Source changes rebuild automatically; open pages and navigation refresh via SSE. +6. **How to remove**: Tell the user both options below — hiding from nav (reversible) and permanently deleting the page directory. +7. **Authentication and authorization**: WebUI uses login session automatically. Regular users can use published pages. Creating/modifying pages requires admin privileges; Rex should remind the user before write operations but does not verify roles in this skill; scripts calling management APIs need `server_api_token` (see **Authentication** above). +8. **Data sources**: Built-in `/api/*` endpoints, page-scoped backend APIs (`/api/user-defined-pages//api/*`), or workflows (`/api/workflow/{id}/run`) (see **Backend Data & API Extension** below). +9. **Backup + restart/upgrade continuity**: Back up the full page directory and explain that Flocks scans/rebuilds pages from `~/.flocks/plugins/user_defined_pages/` after restart or upgrade. + +Then ask whether the user already has a page idea. If they have an idea, remind them that creation requires admin privileges, then start creation. If they do not have an idea, offer 2–3 example scenarios. + +## Page ID Rules + +- Allowed: `a-z`, `0-9`, `-` +- Examples: `alert-dashboard`, `threat-overview`, `duty-screen` +- Disallowed: uppercase, spaces, CJK characters, underscores + +## Directory Layout + +```text +~/.flocks/plugins/user_defined_pages// + manifest.json + src/index.tsx + src/Page.tsx + dist/page.js # auto-generated + dist/meta.json # auto-generated + assets/ # optional +``` + +## Creation Options + +### Option A — API (when HTTP is needed) + +Requires a valid `server_api_token` (see **Authentication**). Rex should prefer Option B unless API is explicitly required. + +```bash +curl -s -X POST http://127.0.0.1:8000/api/user-defined-pages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{"id":"alert-dashboard","title":"Alert Dashboard","icon":"BarChart3","order":100}' +``` + +Chinese example title: `"title":"告警看板"`. + +### Option B — Write files directly (preferred for Rex) + +Create under `~/.flocks/plugins/user_defined_pages//`: + +**manifest.json** + +```json +{ + "id": "alert-dashboard", + "title": "Alert Dashboard", + "route": "/user-defined-pages/alert-dashboard", + "icon": "BarChart3", + "order": 100, + "enabled": true, + "placement": "home.after", + "entry": "src/index.tsx", + "updatedAt": 0 +} +``` + +**src/index.tsx** + +```tsx +import Page from './Page'; +export default Page; +``` + +**src/Page.tsx** — start from the template below. + +## Page Template + +Use the manifest `title` for the card heading. Keep in-page status text in the user's language. + +```tsx +import { useEffect, useState } from 'react'; +import { Card } from '@flocks/user-defined-page-sdk'; + +export default function Page() { + const [ready, setReady] = useState(false); + + useEffect(() => { + setReady(true); + }, []); + + return ( + + {ready ? 'Ready' : 'Loading...'} + + ); +} +``` + +For Chinese pages, use Chinese copy inside the component, e.g. `{ready ? '页面已就绪' : '加载中...'}`. + +## Development Flow + +1. After scaffold creation, tell the user the nav label, route, and directory path. +2. Identify data sources — built-in `/api/*`, page-scoped backend APIs, workflows, or external systems that need server-side proxying. +3. Edit `src/Page.tsx` based on requirements (add more files under `src/` if needed). +4. On save, the system rebuilds automatically. If build fails, read `dist/meta.json` → `error` and fix. +5. Manual rebuild: `POST /api/user-defined-pages//build` +6. Before wrapping up, run a final location check: every page file created for the user must be under `~/.flocks/plugins/user_defined_pages//`; do not leave page source, API handlers, assets, or drafts in the repository code directories. + +## Backup and Restore + +Always provide this backup command in the first explanation and in the final wrap-up: + +```bash +cp -a ~/.flocks/plugins/user_defined_pages/ ~/.flocks/workspace/outputs//-backup +``` + +Restore by copying the backup directory back to `~/.flocks/plugins/user_defined_pages//`. After restart (or immediately if watcher is active), the page will be scanned and available again. + +## Backend Data & API Extension + +When a custom page needs backend logic or external data that built-in APIs do not provide, use a **page-scoped backend API runtime**. + +### Design Principle + +Do **not** register arbitrary global FastAPI routes such as `/api/my-dashboard/stats`. + +The page backend should be scoped to the page namespace: + +```text +/api/user-defined-pages//api/{path:path} +``` + +This keeps page APIs tied to page lifecycle, permissions, logs, hot reload, deletion, and future UI management. + +### Architecture + +```text +User Defined Page (src/Page.tsx) + └─ SDK api ──► /api/user-defined-pages//api/* (page-scoped backend) + ├─► /api/workflow/{id}/run (multi-step workflows) + └─► /api/* (built-in Flocks APIs) +``` + +### Target Directory Layout + +When a page needs backend code, add an `api/` directory inside that page: + +```text +~/.flocks/plugins/user_defined_pages// + manifest.json + src/Page.tsx + api/ + routes.yaml + handlers.py + dist/ + page.js + meta.json +``` + +### Route Manifest + +Use `api/routes.yaml` to declare the page API surface: + +```yaml +routes: + - method: GET + path: /stats + handler: handlers.get_stats + timeoutMs: 5000 + + - method: POST + path: /ack + handler: handlers.ack_alert + timeoutMs: 10000 +``` + +Rules: + +- `path` must start with `/` and is always scoped under `/api/user-defined-pages//api`. +- `handler` points to a callable in `api/handlers.py`. +- Keep route count small and page-specific. +- Prefer read-only `GET` for dashboards; use `POST` for actions. +- Do not expose global admin operations from page APIs. + +### Handler Code + +Use `api/handlers.py` for server-side page logic: + +```python +async def get_stats(ctx, request): + # ctx exposes trusted server-side helpers such as: + # ctx.user, ctx.page_id, ctx.secrets, ctx.logger + return { + "open": 12, + "critical": 3, + } + +async def ack_alert(ctx, request): + body = await request.json() + alert_id = body.get("id") + if not alert_id: + return {"ok": False, "error": "missing alert id"} + return {"ok": True, "id": alert_id} +``` + +Implementation expectations for the runtime: + +- Route module loading is controlled by Flocks, not by arbitrary `include_router`. +- Handlers run as trusted local plugins, not as a security sandbox. +- Runtime enforces auth, page ID validation, route validation, request/response size limits, timeout, and structured error reporting. +- Secrets are read server-side through `ctx.secrets` or `get_secret_manager()`; never return secrets to the page. +- Watcher should monitor `api/routes.yaml` and `api/*.py`; API changes should hot-reload without restarting Flocks. +- API runtime errors should be visible in page diagnostics (for example `dist/meta.json` or a dedicated API meta file). + +### Call from Page Code + +The SDK `api` helper sends the logged-in user's session cookie: + +```tsx +const res = await api.get('/api/user-defined-pages/alert-dashboard/api/stats'); +``` + +If the SDK later provides a page helper, prefer: + +```tsx +const res = await api.page.get('/stats'); +``` + +Until `api.page` exists, use explicit `/api/user-defined-pages//api/*` paths. + +### Other extension paths + +| Need | Approach | Call from page | +|------|----------|----------------| +| Page-specific backend data | `api/routes.yaml` + `api/handlers.py` | `/api/user-defined-pages//api/*` | +| External REST API needed by one page | Page handler proxies it server-side | same | +| Local compute / file transform for one page | Page handler | same | +| Multi-step orchestration | Workflow under `~/.flocks/plugins/workflows//` | `POST /api/workflow//run` | +| Existing Flocks data | Built-in routes | `api.get/post('/api/...')` | +| Fully custom standalone server | Separate process | Prefer a page API proxy to avoid browser CORS and secret exposure | + +### Management APIs (for Rex / scripts) + +| Action | Method | +|--------|--------| +| List page API routes | `GET /api/user-defined-pages//api` | +| Call page API | `GET/POST/... /api/user-defined-pages//api/` | +| Reload page API | `POST /api/user-defined-pages//api/reload` | +| Read page detail/build info | `GET /api/user-defined-pages/` | + +If these endpoints are not implemented yet, treat this section as the target design and implement the backend runtime before promising the feature as available. + +### Limitations (tell users when relevant) + +- Do not register user routes globally under `/api/custom/...`. +- Page API code is trusted local plugin code, not sandboxed untrusted code. +- Page code may only import `react` and `@flocks/user-defined-page-sdk` — backend logic lives in `api/handlers.py`, not in page TSX. +- Page API routes are page-scoped; deleting the page should remove its backend routes as well. + +### Explain to users (when page needs custom data) + +**Chinese example**: + +> 如果内置 API 不够用，我们按页面专属后端 API 的设计来做：在 `~/.flocks/plugins/user_defined_pages//api/` 下定义 `routes.yaml` 和 `handlers.py`，后端统一暴露到 `/api/user-defined-pages//api/*`。密钥只在服务端读取，不会写进页面代码。 + +**English example**: + +> When built-in APIs are insufficient, use the page-scoped backend API design: define `api/routes.yaml` and `api/handlers.py` under `~/.flocks/plugins/user_defined_pages//`, then expose them through `/api/user-defined-pages//api/*`. Secrets stay server-side. + +## Useful APIs + +| Action | Method | +|--------|--------| +| List | `GET /api/user-defined-pages?enabledOnly=true` | +| Detail | `GET /api/user-defined-pages/` | +| Save source | `PUT /api/user-defined-pages/` with `{"sourcePath":"src/Page.tsx","sourceContent":"..."}` | +| Update manifest | `PUT /api/user-defined-pages/` with `{"manifest":{"title":"New Title","order":50}}` | +| Rebuild | `POST /api/user-defined-pages//build` | +| Hide from nav | `PUT /api/user-defined-pages/` with `{"manifest":{"enabled":false}}` | +| Delete permanently | Remove `~/.flocks/plugins/user_defined_pages//` (see below) | + +## Remove or Delete a Page + +Always explain both approaches when the user asks how to delete, or proactively mention this in the first introduction and wrap-up. + +### Option 1 — Hide from navigation (soft delete, reversible) + +Update manifest so the page no longer appears in the left nav, but files are kept: + +```bash +curl -s -X PUT http://127.0.0.1:8000/api/user-defined-pages/ \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{"manifest":{"enabled":false}}' +``` + +Or edit `manifest.json` and set `"enabled": false`. Navigation updates automatically via SSE; **no restart** required. + +To restore later, set `"enabled": true` again. + +### Option 2 — Delete permanently (hard delete) + +Remove the entire page directory under user space: + +```bash +rm -rf ~/.flocks/plugins/user_defined_pages/ +``` + +Only do this **after confirming** with the user — this cannot be undone (unless they have backups). After deletion, the nav item disappears automatically; no restart required. + +**Chinese phrasing example** (adapt to user's language): + +> 如果不再需要这个页面，有两种方式： +> 1. **从导航隐藏**：把 `manifest.json` 里的 `enabled` 设为 `false`，页面文件仍保留，以后可恢复； +> 2. **彻底删除**：删除目录 `~/.flocks/plugins/user_defined_pages//`，导航标签会消失且无法恢复，请先确认再操作。 + +**English phrasing example**: + +> To remove a page you have two options: +> 1. **Hide from navigation** — set `"enabled": false` in the manifest (files kept, reversible); +> 2. **Delete permanently** — remove `~/.flocks/plugins/user_defined_pages//` (irreversible; confirm with the user first). + +When the user explicitly asks to delete a page, confirm which option they want before acting. + +## Conversation Steps + +### Step 1 — Understand needs + +Ask about: +- Purpose (dashboard / list / form / screen) +- Navigation title (user's language) +- Suggested `pageId` +- Whether multiple nav pages are needed +- **Data sources** — built-in APIs, page-scoped backend APIs, workflows, or external systems that need server-side proxying + +### Step 2 — Create scaffold + +After confirming `pageId` and `title`, create the page and report nav name, route, and directory. + +### Step 3 — Implement + +Iterate on `src/Page.tsx`: +- Match WebUI Tailwind styling +- Use `api` for built-in `/api/*` data when available +- If built-in APIs are insufficient, design or implement page-scoped APIs under `api/routes.yaml` + `api/handlers.py`, then call `/api/user-defined-pages//api/*` from the page +- Tell the user to wait for hot reload after each save + +### Step 4 — Wrap up + +Before responding, perform a final location check and explicitly confirm that the page files are under `~/.flocks/plugins/user_defined_pages//`, not in the project code directory. + +Summarize page ID, nav label, route, data sources used (built-in API / page API / workflow), the verified storage directory, how to keep editing via chat, how to **hide** (`"enabled": false`) or **permanently delete** (remove `~/.flocks/plugins/user_defined_pages//`), and how to add another page or extend data with page-scoped backend APIs. +Also include one concrete backup command and remind the user that restart/upgrade keeps pages because files are stored in user home and startup reconciliation rebuilds missing/old bundles. + +## Rex-User Collaboration Loop + +Use this loop to ensure Rex and the user co-develop the page with clear ownership and fast feedback. + +### Responsibilities + +- **User provides**: business goal, fields/metrics, expected interactions, visual style, and acceptance criteria. +- **Rex provides**: page scaffold/files, frontend implementation, optional page-scoped backend API (`api/routes.yaml` + `api/handlers.py`), build/runtime troubleshooting, and final storage verification. + +### Iteration cadence (must follow) + +For each iteration, Rex should: + +1. Restate the current task in one sentence (what will change in this round). +2. Implement only the agreed slice (small increment; avoid large unreviewed rewrites). +3. Tell the user exactly what to verify in WebUI (route, interaction, expected data/result). +4. Wait for user feedback, then continue to the next slice. + +### Creation completion checklist + +Do not declare "done" until all are true: + +- Nav item visible under Home with expected title/icon/order. +- Route works: `/user-defined-pages/`. +- Frontend behavior matches user requirements. +- If custom backend is used, page API routes work under `/api/user-defined-pages//api/*`. +- Backup command provided. +- Hide/delete options provided. +- Final location check explicitly confirmed (`~/.flocks/plugins/user_defined_pages//` only). + +### If the user is unsure what to provide + +Rex should ask only the minimum needed in this order: + +1. page purpose +2. title + pageId +3. data sources +4. key interactions +5. visual preference + +## Troubleshooting + +| Symptom | Action | +|---------|--------| +| Nav item missing | Check `manifest.enabled`; wait for build | +| Blank / error page | Check `GET /api/user-defined-pages/` → `build.error` | +| Build failed | Fix TSX syntax; only import `react` and `@flocks/user-defined-page-sdk` | +| Changes not visible | Confirm file saved under `src/`; try `POST .../build` | +| 401 Unauthorized (WebUI) | User not logged in or session expired — re-login | +| 401 Unauthorized (curl/script) | Add `Authorization: Bearer `; run `flocks admin generate-api-token` if missing | +| Page `api` calls fail | User must stay logged in; do not put API Token in page source | +| Page API 404 | Confirm `api/routes.yaml` path and page ID; reload page API runtime | +| Page API 500 | Check handler traceback / page API diagnostics; validate handler return shape | +| External API from page fails | Do not call third-party URLs directly from page code — proxy through page-scoped backend | +| Need custom `/api/foo` route | Do not add global routes — use `/api/user-defined-pages//api/foo` | + +## Do Not + +- Write pages into `webui/` or `flocks/` code directories +- Leave generated user page source, API handlers, assets, or drafts anywhere outside `~/.flocks/plugins/user_defined_pages//` +- Modify files under `dist/` +- Import non-whitelisted npm packages into page code +- Skip `pageId` format validation +- Hardcode `server_api_token` or other secrets in page source (`src/*.tsx`) +- Ask users to paste API tokens into chat +- Register global custom FastAPI routes or write backend logic into `src/Page.tsx` +- Call third-party APIs directly from page code with embedded secrets +- Write page backend files outside `~/.flocks/plugins/user_defined_pages//api/` diff --git a/.flocks/plugins/skills/web2cli/SKILL.md b/.flocks/plugins/skills/web2cli/SKILL.md index 27fe5685f..b5fdc0fc4 100644 --- a/.flocks/plugins/skills/web2cli/SKILL.md +++ b/.flocks/plugins/skills/web2cli/SKILL.md @@ -51,16 +51,32 @@ mkdir -p "$CAPTURE_ROOT/captures" - 浏览器内存中的原始捕获数据：`window.__capturedRequests` - 导出的接口抓包 JSON：`$CAPTURE_ROOT/captures/${CAPTURE_NAME}_api.json` - 浏览器认证状态：`$CAPTURE_ROOT/auth-state.json` -- 操作适配规格：`$CAPTURE_ROOT/web2cli-spec.json` - 站点自适应 Hook（仅当 base 失败时创建）：`$CAPTURE_ROOT/hook.js` -- 生成的 CLI 工具：`$CAPTURE_ROOT/_cli.py`，`generate-cli.py` 会把 `-` 等非 Python 模块名字符替换为 `_` +- 生成的 CLI 工具：`$CAPTURE_ROOT/_cli.py`，文件名中的 `-` 等非 Python 模块名字符需替换为 `_` - 生成的验证材料：`$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json` -- 生成的接口文档：`$CAPTURE_ROOT/${CAPTURE_NAME}_api.md` +- 生成的接口文档：`$CAPTURE_ROOT/cli-reference.md` - 生成的 Postman 集合：`$CAPTURE_ROOT/${CAPTURE_NAME}_postman.json` ## 标准流程 -> 按照以下 1-12 的操作流程完成任务 +> 按照以下 1-11 的操作流程完成任务 + +Copy this checklist and check off items as you complete them: + +```text +Task Progress: +- [ ] Step 1: Confirm target site/tab and prepare capture directory +- [ ] Step 2: Check browser availability and open or attach target tab +- [ ] Step 3: Wait for required manual login or authorization +- [ ] Step 4: Inject Web2CLI capture hook and verify it is installed +- [ ] Step 5: Perform the target page operation and confirm requests are captured +- [ ] Step 6: Export captured API data and save browser auth state +- [ ] Step 7: Analyze captured APIs and identify the CLI request chain +- [ ] Step 8: Generate CLI, verify.json, and cli-reference.md +- [ ] Step 9: Validate the generated CLI against live captured/authenticated data +- [ ] Step 10: Integrate the WebCLI capability into a maintainable skill or device asset +- [ ] Step 11: Summarize generated capability and close only the Web2CLI tab +``` ### 1. 打开浏览器或创建 Tab @@ -133,8 +149,9 @@ print(js("window.__apiCapture.config.captureMode")) ### 4. 明确需要捕获的功能/操作 -- 要求用户手动操作要捕获的页面动作，例如查询、翻页、筛选、提交表单、点击按钮、导出数据。 -- 或者请求用户描述需要 hook 的操作或功能，便于你直接去页面代替用户执行 +- 方式 1：要求用户手动操作要捕获的页面动作，例如查询、翻页、筛选、提交表单、点击按钮、导出数据。 +- 方式 2：请求用户描述需要 hook 的操作或功能，你直接去页面代替用户执行 +- 方式 3：用户之前已经描述了需要的 CLI功能，你直接去页面代替用户执行 需要确认捕获是否开始时： @@ -255,78 +272,75 @@ jq -r '.[].method' "$CAPTURE_ROOT/captures/${CAPTURE_NAME}_api.json" | sort | un jq '.[] | select(.method == "POST") | {url: .url, body: .requestBody}' "$CAPTURE_ROOT/captures/${CAPTURE_NAME}_api.json" ``` -### 8. 生成 web2cli-spec 规格 +### 8. 判断最终产物落点 -先基于 `"$CAPTURE_ROOT/captures/${CAPTURE_NAME}_api.json"` 生成中间契约层 `web2cli-spec.json`。 +生成任何 CLI、handler 或最终文件前，必须先判断本次 WebCLI 能力最终应该沉淀到哪里。不要先生成一个孤立 CLI，再在后续步骤才决定是否改成 device tool。 -```bash -uv run python .flocks/plugins/skills/web2cli/scripts/generate-spec.py \ - "$CAPTURE_ROOT/captures/${CAPTURE_NAME}_api.json" \ - --base-url "https://example.com" \ - --output "$CAPTURE_ROOT/web2cli-spec.json" -``` +根据用户目标和场景二选一： -`web2cli-spec.json` 是抓包结果到最终 CLI 之间的可编辑契约，包含： +- **通用网站、查询脚本、内部系统操作、非设备页接入**：最终主 CLI 放在 skill 的 `scripts/`，按 `references/cli-in-skill.md` 集成为长期维护的 skill / CLI 资产。 +- **安全设备接入、来自设备接入页、需要出现在设备页配置和调用**：最终主实现放在 `tools/device//` 下，按 `references/cli-in-device.md` 生成 `_provider.yaml`、工具 YAML 和 handler。CLI 只可作为可选调试/回归入口，不作为设备运行时主路径。 -- 目标站点与命令名 -- 鉴权策略（如 `PUBLIC` / `COOKIE` / `HEADER`） -- 主请求的 method、endpoint、query/body/payload 模板 -- CLI 参数定义 -- 固定输出列定义 -- 验证材料初稿 +如果用户目标不清楚，先用 `question` 明确最终落点，再继续生成。 -生成后必须检查并按需修正： +### 9. 按目标落点生成可验证实现 -- `strategy` 是否正确 -- `args` 是否符合实际操作意图 -- `columns` 与字段路径是否对应目标数据 -- `verify` 的最少行数、必填列是否合理 +#### 9.1 通用 CLI / Skill 场景 -### 9. 基于 spec 生成 CLI 工具 +生成前必须读取并遵循： -从 `"$CAPTURE_ROOT/web2cli-spec.json"` 生成最终 CLI。 +- `$WEB2CLI_SKILL/references/cli-requirements.md` +- `$WEB2CLI_SKILL/references/cli-in-skill.md` -```bash -uv run python .flocks/plugins/skills/web2cli/scripts/generate-cli.py \ - --spec "$CAPTURE_ROOT/web2cli-spec.json" \ - --format python \ - --output "$CAPTURE_ROOT/${CAPTURE_NAME}_cli.py" -``` +基于抓包结果、认证状态和用户目标，生成 CLI、验证材料和接口文档。阶段性产物至少包含： -如果 `CAPTURE_NAME` 包含 `-` 等不能作为 Python 模块名的字符，生成器会自动规范化输出文件名，例如 `test-domain_cli.py` 会写为 `test_domain_cli.py`，并在命令输出中打印实际路径。 +- `$CAPTURE_ROOT/_cli.py` +- `$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json` +- `$CAPTURE_ROOT/cli-reference.md` -生成验证文件： +如果 `CAPTURE_NAME` 包含 `-` 等不能作为 Python 模块名的字符，生成 CLI 文件名时必须规范化为 `_`，例如 `test-domain_cli.py` 应写为 `test_domain_cli.py`。 -```bash -uv run python .flocks/plugins/skills/web2cli/scripts/generate-cli.py \ - --spec "$CAPTURE_ROOT/web2cli-spec.json" \ - --format verify \ - --output "$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json" -``` +随后按 `references/cli-in-skill.md` 将主 CLI 集成到 skill 的 `scripts/`，不要把最终 CLI 保留成一次性抓包文件名。 -生成接口文档： +#### 9.2 安全设备接入场景 -```bash -uv run python .flocks/plugins/skills/web2cli/scripts/generate-cli.py \ - --spec "$CAPTURE_ROOT/web2cli-spec.json" \ - --format markdown \ - --title "${CAPTURE_NAME} API Documentation" \ - --output "$CAPTURE_ROOT/${CAPTURE_NAME}_api.md" -``` +生成前必须读取并遵循： + +- `$WEB2CLI_SKILL/references/cli-in-device.md` + +基于抓包结果、认证状态和用户目标，生成 device 插件目录： + +- `$HOME/.flocks/plugins/tools/device//_provider.yaml` +- `$HOME/.flocks/plugins/tools/device//.yaml` +- `$HOME/.flocks/plugins/tools/device//.handler.py` +- `$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json` +- `$CAPTURE_ROOT/cli-reference.md` -### 10. CLI工具验证与修改 +安全设备接入场景不要求先生成 `$CAPTURE_ROOT/_cli.py`。如确实需要 CLI 做调试或回归，可生成可选 CLI，但必须明确它不是设备运行时主路径，且不要和 handler 独立演进出两套认证/请求逻辑。 -根据生成的 CLI ，任意选择一个接口调用测试可用性 -- CLI 工具可用性 +### 10. 验证与修改 + +根据第 8 步确定的目标落点验证可用性： + +- 通用 CLI / Skill 场景：用生成的 CLI 任意选择一个接口调用测试可用性 +- 安全设备接入场景：用生成的 handler/device tool 或可选 CLI 任意选择一个低风险接口调用测试可用性 - 认证状态可用性 - `verify.json` 的输出约束是否满足 -- method、endpoint、query/body/payload 的一致性，必要时根据${CAPTURE_NAME}_api.json调整 +- method、endpoint、query/body/payload 的一致性，必要时根据 `${CAPTURE_NAME}_api.json` 调整 + +推荐先查看 `"$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json"`，再以默认参数执行一次最小验证，确认固定输出列与认证状态都正确。 + +### 11. 将 WebCLI 能力沉淀为最终产物 + +无论主实现放在哪里，都必须保留 skill 级文档入口，供长期维护、认证恢复、重新抓包和排障使用： -推荐先查看 `"$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json"`，再用生成的 CLI 以默认参数执行一次，确认固定输出列与认证状态都正确。 +- `references/browser-workflow.md` 必须记录浏览器连接检查、登录步骤、state 保存位置和认证恢复流程 +- `references/cli-reference.md` 必须记录 CLI 或 device handler 的能力、参数、验证方式和回归方法 +- `SKILL.md` 必须说明当前能力最终落点：`scripts/` 或 `tools/device//` -### 11. CLI 工具集成到skill +注意：skill 文档入口必选，不等于必须把主 CLI 代码也放进 skill 的 `scripts/`。安全设备接入场景下，主实现应以 device handler 为准。 -将 CLI 按 `references/cli-in-skill.md` 集成为 skill； +不要只停留在一次性 CLI 或临时抓包结果；最终都要沉淀成可长期维护的资产。 ### 12. summary并关闭浏览器 tab @@ -383,4 +397,5 @@ else: - 登录状态失效：重新登录后再次执行保存状态命令。 ## Reference +- references/cli-in-device.md 在 skill 集成完成后，将 WebCLI 能力进一步封装为 device 插件 - references/cli-in-skill.md 将生成的 CLI 集成到 skill 中使用 diff --git a/.flocks/plugins/skills/web2cli/references/cli-in-device.md b/.flocks/plugins/skills/web2cli/references/cli-in-device.md new file mode 100644 index 000000000..56a12af07 --- /dev/null +++ b/.flocks/plugins/skills/web2cli/references/cli-in-device.md @@ -0,0 +1,280 @@ +# 生成后的 WebCLI 如何接入 Device 插件 + +> 本文说明：`web2cli` 已经抓到页面请求、并整理出可复用调用逻辑后，怎样把它沉淀成可在设备页识别、配置和调用的 device 插件。 + +## 结论 + +`cli-in-device.md` 不是 `cli-in-skill.md` 的替代物，而是安全设备场景下的进一步封装： + +- 所有 `web2cli` 结果都必须先完成 skill 集成 +- 如果目标是安全设备接入，再继续按本文档额外生成 device 插件 +- 最终交付关系是：`skill` 必选，`device 插件` 为安全设备场景下的额外交付 + +## 何时使用 + +在以下场景调用本文档： + +- 当前任务明确来自“设备接入”页面，目标是把某个安全设备或安全产品接入到设备管理体系 +- 最终产物需要出现在设备页，并允许用户填写实例配置、刷新模板、按 `device_id` 调用 +- 当前 WebCLI 抓到的能力属于安全设备能力，而不是单纯给 skill 复用的站点操作脚本 + +不优先使用本文档的场景： + +- 只是想保留一个可复用 CLI 供 agent 在 skill 中调用 +- 目标不是设备接入，而是某个通用网站的操作自动化、查询脚本或内部工具 +- 暂时只需要沉淀浏览器经验、CLI 参数和认证恢复流程，不需要设备页识别 + +如果当前任务来自“设备接入”页面，并且目标是安全设备接入，WebCLI 在完成 skill 集成后，还应当额外生成标准 device 插件： + +```text +$HOME/.flocks/plugins/tools/device// +├── _provider.yaml +├── .yaml +├── .handler.py +├── _cli.py # 可选，仅用于调试/回归 +└── _test.yaml # 可选，最小验证样例 +``` + +其中： + +- `_provider.yaml`：决定设备页是否能识别该模板，以及用户创建实例时需要填写哪些字段 +- `.yaml`：定义可调用工具、参数和 action +- `.handler.py`：设备运行时入口，负责读取配置、认证、发请求、清洗结果 +- `_cli.py`：只作为调试入口保留，不作为设备运行时主路径 + +认证默认规则： + +- 自定义 CLI / WebCLI 默认认证方式为 `cookie/auth-state`：优先复用浏览器保存的 `auth-state.json`，从中按请求域名/path/secure 规则选择 Cookie，并在需要时读取 localStorage +- 默认认证状态文件：`~/.flocks/browser//auth-state.json` +- 优先使用 `auth_state_path` 指向 `~/.flocks/browser//auth-state.json` +- 可以额外暴露可选 `username` / `password`，但它们只用于 cookie 失效后的认证恢复，不替代默认的 `auth_state_path` +- 不要生成或使用 `auth_state_json` / `Legacy Auth State JSON` 这类内联 JSON 字段；设备配置只保存 state 文件路径，不粘贴 state 文件内容 +- 只有在目标站点确实还依赖额外字段时，才补充 `cookie`、`csrf_token`、`access_token` 或特定认证头；这些字段是 `auth_state_path` 之外的补充，不替代默认的 cookie/auth-state +- 不要把 `cookie` 或 `token` 设计成和 `auth-state` 并列的多个默认入口；如果用户提供的是 state 文件路径，必须写入 `auth_state_path` + +## 命名约定 + +- 插件目录：`$HOME/.flocks/plugins/tools/device//` +- `plugin_id`：推荐使用稳定产品名加版本，例如 `_v1_0_0` +- `service_id`：推荐使用稳定能力标识，例如 `_device` +- handler 文件：`.handler.py` +- 可选 CLI 文件：`_cli.py` + +约定说明： + +- `` 用产品或系统的稳定标识，不用一次性任务名 +- 目录名可以带版本；`service_id` 要尽量稳定，避免和临时抓包任务绑定 +- Python 文件名统一用 `_` + +## 最小 `_provider.yaml` + +至少包含以下字段： + +```yaml +name: Acme Portal +vendor: acme_security +service_id: acme_portal_device +version: "1.0.0" +integration_type: device +description: > + Acme Portal WebCLI-backed device integration for alert listing and asset + detail queries. Configure Base URL and the required login state fields + separately in the credentials form. +description_cn: > + Acme Portal 的 WebCLI 设备接入模板，支持告警列表和资产详情查询。 + 请在设备配置中分别填写 Base URL 与所需登录态字段。 +credential_fields: + - key: base_url + label: Base URL + storage: config + config_key: base_url + input_type: url + required: true + - key: auth_state_path + label: Auth State Path + storage: config + config_key: auth_state_path + input_type: text + default: "~/.flocks/browser/acme-portal/auth-state.json" + - key: username + label: Username + storage: config + config_key: username + input_type: text + required: false + description: 仅在 cookie 失效后需要 Agent 辅助登录刷新 state 时填写 + - key: password + label: Password + storage: secret + config_key: password + secret_id: acme_portal_password + input_type: password + required: false + description: 仅在 cookie 失效后需要 Agent 辅助登录刷新 state 时填写 + - key: cookie + label: Cookie + storage: secret + config_key: cookie + secret_id: acme_portal_cookie + input_type: password + - key: csrf_token + label: CSRF Token + storage: secret + config_key: csrf_token + secret_id: acme_portal_csrf_token + input_type: password +defaults: + timeout: 30 + category: custom +notes: | + WebCLI 设备建议优先复用稳定隐藏接口，不建议把浏览器自动化作为默认运行时。 + 若返回 401/403、跳转登录页或 CSRF 失效，应先按认证失效处理。 +``` + +注意： + +- 必须包含 `integration_type: device` +- `description` 用英文，`description_cn` 用中文 +- 只把运行时真正需要用户填写的字段放进 `credential_fields` +- 不要把真实 cookie、token、密码、auth state JSON 写进插件文件 +- 默认先放 `auth_state_path`，并指向 `~/.flocks/browser//auth-state.json`；不要添加 `auth_state_json` / `Legacy Auth State JSON` +- 可以补充可选 `username` / `password`，但必须标注它们仅用于认证恢复或浏览器辅助登录，不得作为默认运行时认证入口 +- `cookie`、`csrf_token`、`access_token` 或特定认证头只有在实际站点需要时再补，并在 handler 中明确说明来源与刷新方式 + +## 最小工具 YAML + +MVP 阶段推荐一个分组工具 + 多个 action： + +```yaml +name: acme_portal_ops +description: > + Acme Portal grouped device tool. Use the action parameter to query alerts, + assets, and other WebCLI-backed operations. +description_cn: > + Acme Portal 分组设备工具。通过 action 参数调用告警、资产和其他 WebCLI 能力。 +category: custom +enabled: true +requires_confirmation: false +provider: acme_portal_device +inputSchema: + type: object + properties: + action: + type: string + enum: [list_alerts, get_asset_detail] + description: 统一业务动作名，不要暴露内部实现来源。 + alert_id: + type: string + description: 查询资产详情时可选使用的关联标识。 + required: [action] +handler: + type: script + script_file: acme_portal.handler.py + function: handle +``` + +规则： + +- `provider` 必须与 `_provider.yaml.service_id` 一致 +- 高风险写操作必须设置 `requires_confirmation: true` +- 对外 action 用统一业务语义，不要命名成 `webcli_get_alerts`、`api_get_alerts` + +## 最小 handler 结构 + +MVP 阶段优先单文件 handler，不强制拆 client 模块： + +```python +from __future__ import annotations + +from typing import Any + +from flocks.config.config_writer import ConfigWriter +from flocks.tool.registry import ToolContext, ToolResult + +SERVICE_ID = "acme_portal_device" + + +def _service_config() -> dict[str, Any]: + raw = ConfigWriter.get_api_service_raw(SERVICE_ID) + return raw if isinstance(raw, dict) else {} + + +async def handle(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + cfg = _service_config() + if action == "list_alerts": + return ToolResult(success=True, output={"items": [], "source": "webcli_api"}) + if action == "get_asset_detail": + return ToolResult(success=True, output={"item": None, "source": "webcli_api"}) + return ToolResult(success=False, error=f"Unsupported action: {action}") +``` + +要求： + +- 通过 `ConfigWriter.get_api_service_raw(SERVICE_ID)` 读取当前设备实例配置 +- handler 内部负责认证头构造、分页、超时、重试和响应归一化 +- handler 默认只读取 `auth_state_path` 指向的 `auth-state.json`；如果文件缺失、不是合法 JSON，或没有匹配当前 Base URL 的 Cookie，应返回明确错误并提示重新登录/保存 state +- handler 不要 fallback 到内联 `auth_state_json`；这会把路径字符串、占位文本或过期内容误当 JSON 解析，导致设备测试报错不清晰 +- 如果模板提供了 `username` / `password`，handler 也不要在普通 tool 调用里静默自动登录；这些字段只用于后续由 Rex 进入浏览器认证恢复流程时辅助填表 +- CLI 可选保留，但不要让设备运行时通过 subprocess 调 CLI + +## 组合 API / WebCLI / 处理逻辑 + +同一设备可以混合多种能力来源，但对外仍然是统一 action： + +- `api`：正式 API，可直接调用 +- `webcli_api`：WebCLI 抓到的隐藏接口 +- `process`：本地字段归一化、过滤、聚合、补全 +- `composed`：先调一种来源，再补另一种来源，最后统一输出 + +推荐选择顺序： + +1. 正式 API 稳定可用时，优先正式 API +2. 正式 API 缺能力但 WebCLI 接口稳定时，用 `webcli_api` +3. 需要字段清洗、补全、排序、聚合时，在 handler 内增加 `process` +4. 需要多个来源补齐同一业务结果时，用 `composed` +5. 必须验证码、强动态页面或人工交互时，只记录为 browser fallback，不放进默认设备运行时 +6. 如果某个隐藏接口依赖 `Authorization`、`Tdp-Authentication`、CSRF 等临时头，只有在 handler 已实现可靠的恢复/刷新逻辑时才暴露为默认 action；否则保留在 CLI 或文档中，不放进设备默认动作 + +示例 action 映射： + +```yaml +list_alerts: webcli_api +get_asset_detail: composed +list_users: api +normalize_alert: process +``` + +这里的映射可以写进 handler 常量、注释、`notes` 或单独的设计文档，但不要把“来源类型”直接暴露给最终用户。 + +## 认证失败处理 + +出现以下情况时，优先按认证失效处理： + +- 返回 `401` 或 `403` +- 返回内容出现 `Unauthorized`、`login`、未登录、无权限 +- Cookie / CSRF / access token 明显过期 +- `auth_state_path` 已存在，但接口仍跳转登录页 + +处理原则： + +1. 不要无限重试 +2. 优先返回明确话术，提示 Rex 使用 `flocks browser` 和对应 skill 的认证失败处理去恢复登录态 +3. 如果设备已配置可选 `username` / `password`，Rex 可以在浏览器恢复流程中读取它们辅助登录；如遇验证码、MFA、短信码或人工确认，立即停下并让用户接管 +4. 登录成功后执行 `flocks browser state save ` 更新 cookie/state 文件 +5. 如仍失败，再提示用户重新登录或更新设备配置中的认证字段 +6. 如果保留了 CLI，可用 CLI 做一次最小验证 +7. 验证通过后，再让用户回到设备页点击“刷新设备模板” + +## `_test.yaml` 建议 + +如果该 WebCLI 设备已经有最小可验证动作，建议补一个 `_test.yaml`，至少覆盖： + +- 一个低风险读操作 +- 最小必填参数 +- 成功时的关键字段断言 + +这样后续更新 handler 或认证逻辑时更容易回归验证。 + +## 一句话原则 + +`web2cli` 生成的 CLI 是中间产物；只有在“安全设备接入”场景下，才把它整理成标准 device 插件，让设备页能识别、配置并调用。 diff --git a/.flocks/plugins/skills/web2cli/references/cli-requirements.md b/.flocks/plugins/skills/web2cli/references/cli-requirements.md new file mode 100644 index 000000000..16ef30d38 --- /dev/null +++ b/.flocks/plugins/skills/web2cli/references/cli-requirements.md @@ -0,0 +1,101 @@ +# Web2CLI CLI 生成要求 + +> 本文是生成 WebCLI 工具时必须遵循的参考要求。应根据抓包结果、认证状态和用户目标直接生成 CLI、验证材料和接口文档。 + +## 输入材料 + +- 抓包 JSON：`$CAPTURE_ROOT/captures/${CAPTURE_NAME}_api.json` +- 浏览器认证状态：`$CAPTURE_ROOT/auth-state.json` +- 页面入口 URL：来自用户目标或当前浏览器页面 +- 用户要复现的操作：来自用户目标和最近页面操作 +- 目标 base URL：从抓包 URL 中归纳，必要时结合用户指定值 + +## 生成目标 + +生成以下文件： + +- CLI 主脚本：`$CAPTURE_ROOT/_cli.py` +- 验证材料：`$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json` +- 接口文档：`$CAPTURE_ROOT/cli-reference.md` + +命名要求： + +- `` 必须是合法 Python 文件名片段 +- 将 `-`、空格等不适合作为 Python 模块名的字符替换为 `_` +- 不要把一次性路径、cookie、token 或用户私密信息硬编码到脚本中 + +## CLI 行为要求 + +CLI 必须说明并实现： + +- 命令名：`` +- 目标能力：`` +- 默认认证策略：`auth-state` / `cookie` / `header` / `public` +- 默认认证输入：优先使用 `--auth-state "$CAPTURE_ROOT/auth-state.json"` 或对应环境变量 +- 必填参数：`` +- 可选参数与默认值：`` +- 输出格式：默认 `table` 或 `json`，必要时同时支持 `--json` +- 退出码：成功为 `0`，认证失败、参数错误、请求失败和验证失败使用非零退出码 + +## 请求链路要求 + +从抓包 JSON 中选出与目标操作直接相关的请求，并写清楚： + +- 请求顺序和依赖关系 +- method、endpoint、query、body/payload 模板 +- 必要 headers，如 `content-type`、`csrf`、`x-requested-with` +- 认证信息从哪里读取，如何注入到请求中 +- 分页、排序、过滤、时间范围等参数如何映射到 CLI 参数 +- 响应字段路径，以及嵌套列表、空值、错误结构如何处理 + +不要把无关埋点、静态资源、日志上报、健康检查或页面渲染请求纳入主链路。 + +## 输出要求 + +固定输出列必须写清楚： + +| column | source_path | required | description | +| --- | --- | --- | --- | +| `` | `` | `` | `` | + +要求： + +- 表格输出列顺序稳定 +- JSON 输出保留原始字段或清洗后的结构 +- 必填列为空时应在验证材料中标记为失败 +- 时间、数量、状态等字段需要说明格式化规则 + +## 验证要求 + +`verify.json` 至少包含： + +- CLI 调用样例 +- 认证输入说明 +- 最少返回行数 +- 必填列列表 +- 预期 HTTP 状态码或业务成功字段 +- 常见失败场景与判定方式 + +示例结构： + +```json +{ + "command": "uv run python _cli.py --auth-state auth-state.json", + "min_rows": 1, + "required_columns": [""], + "success_status": [200], + "failure_hints": ["authentication expired", "missing required argument"] +} +``` + +## 接口文档要求 + +`cli-reference.md` 至少包含： + +- 能力说明和适用场景 +- 认证方式与刷新登录态的方法 +- CLI 参数表 +- 请求链路摘要 +- 输出字段说明 +- 验证方式和常见问题 + diff --git a/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py b/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py index 526e9f5d6..4ae242d42 100644 --- a/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py +++ b/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py @@ -105,6 +105,16 @@ def _resolve_ref(value: Any) -> str | None: return value +def _normalize_base_url(base_url: str) -> str: + """Return the TDP service origin, not a UI/API page path.""" + normalized = base_url.strip().rstrip("/") + for suffix in ("/config/api", "/api/v1"): + if normalized.lower().endswith(suffix): + normalized = normalized[: -len(suffix)].rstrip("/") + break + return normalized + + def _service_config() -> dict[str, Any]: raw = ConfigWriter.get_api_service_raw(SERVICE_ID) return raw if isinstance(raw, dict) else {} @@ -136,7 +146,7 @@ def _resolve_runtime_config() -> RuntimeConfig: or DEFAULT_BASE_URL ) if base_url: - base_url = base_url.strip().rstrip("/") + base_url = _normalize_base_url(base_url) if not base_url.startswith(("http://", "https://")): base_url = f"https://{base_url}" diff --git a/.flocks/plugins/workflows/loop_host_forensics_fast/workflow.json b/.flocks/plugins/workflows/loop_host_forensics_fast/workflow.json index 70a5fa38a..6290bcdd1 100644 --- a/.flocks/plugins/workflows/loop_host_forensics_fast/workflow.json +++ b/.flocks/plugins/workflows/loop_host_forensics_fast/workflow.json @@ -24,7 +24,7 @@ "type": "python", "name": "单台快速巡检", "description": "每台主机先做 SSH 预检；预检通过后再调 task(host-forensics-fast)。超时仅重试一次；循环态保留文件路径、执行状态、verdict 与失败分类。", - "code": "import os\nimport re\nimport time\n\nhosts = inputs.get(\"hosts\", [])\nidx = int(inputs.get(\"host_idx\", 0))\nhost = hosts[idx] if 0 <= idx < len(hosts) else \"\"\nhost = str(host).strip()\n\nsu = inputs.get(\"ssh_user\")\nif isinstance(su, str):\n su = su.strip()\nelse:\n su = \"\"\n\nconnect_host = host\nconnect_user = \"\"\nif \"@\" in host:\n user_part, host_part = host.split(\"@\", 1)\n connect_user = str(user_part).strip()\n connect_host = str(host_part).strip() or host\n ssh_target = host\n user_hint = (\n \"主机列表项已含 `user@host` 形式。SSH 工具调用必须使用：host=`\"\n + connect_host\n + \"`，username=`\"\n + connect_user\n + \"`。\"\n )\nelif su:\n connect_user = su\n connect_host = host\n ssh_target = su + \"@\" + host\n user_hint = (\n \"工作流已指定 `ssh_user`=`\"\n + su\n + \"`。SSH 工具调用必须使用：host=`\"\n + connect_host\n + \"`，username=`\"\n + connect_user\n + \"`。\"\n )\nelse:\n connect_host = host\n ssh_target = host\n user_hint = (\n \"未指定 `ssh_user`。SSH 工具调用请使用 host=`\"\n + connect_host\n + \"`，username 留空使用默认账户（一般为 root）。\"\n )\n\n\ndef _extract_verdict(markdown_text):\n if not markdown_text:\n return \"UNKNOWN\"\n match = re.search(\n r\"(?im)^\\s*\\*{0,2}Verdict\\*{0,2}\\s*:\\s*\"\n r\"(CLEAN|SUSPICIOUS|COMPROMISED|UNKNOWN)\\b\",\n markdown_text,\n )\n if not match:\n return \"UNKNOWN\"\n return str(match.group(1)).upper()\n\n\ndef _is_timeout_error(message):\n text = str(message or \"\").lower()\n return (\n \"timed out\" in text\n or \"timeout\" in text\n or \"超时\" in text\n or \"节点执行超时\" in text\n )\n\n\ndef _classify_error(message):\n text = str(message or \"\")\n lower = text.lower()\n if not text.strip():\n return \"unknown\"\n if \"permission denied\" in lower or \"auth failed\" in lower or \"authentication failed\" in lower:\n return \"auth_failed\"\n if \"host key verification failed\" in lower or \"host key\" in lower:\n return \"host_key_verification_failed\"\n if \"connection refused\" in lower:\n return \"connection_refused\"\n if \"no route to host\" in lower:\n return \"no_route_to_host\"\n if \"network is unreachable\" in lower:\n return \"network_unreachable\"\n if \"name or service not known\" in lower or \"could not resolve\" in lower or \"nodename nor servname provided\" in lower:\n return \"dns_resolution_failed\"\n if \"connection reset\" in lower:\n return \"connection_reset\"\n if \"broken pipe\" in lower or \"connection lost\" in lower or \"disconnect\" in lower:\n return \"connection_lost\"\n if \"kex\" in lower or \"key exchange\" in lower or \"protocol error\" in lower:\n return \"ssh_handshake_failed\"\n if _is_timeout_error(text):\n if \"connect\" in lower or \"ssh connection failed\" in lower:\n return \"connect_timeout\"\n return \"execution_timeout\"\n if \"ssh connection failed\" in lower:\n return \"ssh_connection_failed\"\n return \"unknown\"\n\n\nidx1 = idx + 1\nper_host_dir = str(inputs.get(\"per_host_dir\") or \"\").strip()\nif not per_host_dir:\n od = str(inputs.get(\"output_dir\") or \"\").strip()\n if od:\n per_host_dir = os.path.join(od, \"host_triage\")\n else:\n per_host_dir = os.path.join(os.path.dirname(inputs.get(\"batch_report_path\") or \".\") or \".\", \"host_triage\")\nos.makedirs(per_host_dir, exist_ok=True)\n\n\ndef _slug(s):\n t = re.sub(r\"[^0-9A-Za-z._@-]+\", \"_\", str(s)).strip(\"_\")\n t = t.replace(\"@\", \"_at_\")\n return (t[:56] if t else \"host\")\n\n\nnh = len(hosts)\nbase = \"{:04d}_{}\".format(idx1, _slug(ssh_target))\nper_host_md = os.path.join(per_host_dir, base + \".md\")\n\npreflight_timeout_s = 20\npreflight_res = tool.run_safe(\n \"ssh_host_cmd\",\n host=connect_host,\n username=(connect_user or None),\n command=\"echo FLOCKS_SSH_OK\",\n timeout=preflight_timeout_s,\n)\npreflight_output = preflight_res.get(\"output\") or preflight_res.get(\"text\") or \"\"\npreflight_error = preflight_res.get(\"error\") or \"\"\npreflight_ok = bool(preflight_res.get(\"success\")) and \"FLOCKS_SSH_OK\" in str(preflight_output)\n\ntext = \"\"\nok = False\nerr = \"\"\nverdict = \"UNKNOWN\"\nfailure_category = \"\"\nattempts = 0\n\nif preflight_ok:\n desc = \"Fast triage item \" + str(idx1)\n prompt = (\n \"你是 host-forensics-fast 工作模式：对下列目标执行 Linux 主机快速安全巡检（首轮研判）。\\n\"\n \"请使用 ssh_run_script，script_path 为 `.flocks/plugins/agents/host-forensics-fast/scripts/triage_fast.sh`。\\n\"\n + user_hint\n + \"\\n\\n本次 SSH 工具参数必须使用：\\n\"\n + \"- host: \"\n + connect_host\n + \"\\n\"\n + (\"- username: \" + connect_user + \"\\n\" if connect_user else \"- username: （留空，使用默认账户）\\n\")\n + \"\\n请输出简洁 Markdown：结论、可疑项、风险判断、后续建议。\"\n )\n while attempts < 2:\n attempts += 1\n res = tool.run_safe(\n \"task\",\n description=desc + \" attempt \" + str(attempts),\n prompt=prompt,\n subagent_type=\"host-forensics-fast\",\n run_in_background=False,\n )\n text = res.get(\"text\") or \"\"\n ok = bool(res.get(\"success\"))\n err = res.get(\"error\") or \"\"\n if ok:\n verdict = _extract_verdict(text)\n break\n if not _is_timeout_error(err) or attempts >= 2:\n failure_category = _classify_error(err)\n break\n time.sleep(3)\n if not ok and not failure_category:\n failure_category = _classify_error(err)\nelse:\n err = preflight_error or \"SSH preflight failed\"\n failure_category = _classify_error(err)\n\nlines = [\n \"# 单主机快速巡检结果\",\n \"\",\n \"- 批次内序号: {} / {}\".format(idx1, nh),\n \"- 列表项 host: `{}`\".format(host),\n \"- ssh_target: `{}`\".format(ssh_target),\n \"- ssh_host: `{}`\".format(connect_host),\n \"- ssh_user: `{}`\".format(connect_user or \"(default)\"),\n \"- success: {}\".format(ok),\n \"- verdict: {}\".format(verdict),\n \"- failure_category: {}\".format(failure_category or \"\"),\n \"- inspect_attempts: {}\".format(attempts),\n \"\",\n]\nif not preflight_ok:\n lines.extend(\n [\n \"## SSH 预检失败\",\n \"\",\n \"- 分类: `{}`\".format(failure_category),\n \"- 错误:\",\n \"\",\n \"```\",\n str(err),\n \"```\",\n \"\",\n ]\n )\nelif err:\n lines.extend([\"## 错误\", \"\", \"```\", str(err), \"```\", \"\"])\nelse:\n lines.extend([\"## 子 Agent 输出\", \"\", (text if text else \"_(无正文)_\"), \"\"])\nwith open(per_host_md, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\\n\".join(lines))\n\ntr = inputs.get(\"triage_results\", [])\ntr = list(tr) if isinstance(tr, list) else []\ntr.append(\n {\n \"host\": host,\n \"ssh_user\": connect_user or su,\n \"ssh_target\": ssh_target,\n \"ssh_host\": connect_host,\n \"success\": ok,\n \"verdict\": verdict,\n \"failure_category\": failure_category,\n \"inspect_attempts\": attempts,\n \"error\": err,\n \"per_host_md\": per_host_md,\n }\n)\noutputs[\"triage_results\"] = tr\n\nbatch_report_path = inputs.get(\"batch_report_path\", \"\")\nsection = (\n \"\\n## [{}/{}] `{}`\\n\\n\".format(idx1, nh, ssh_target)\n + \"- 单独报告: `{}`\\n\".format(per_host_md)\n + \"- 执行结果: {}\\n\".format(\"成功\" if ok else \"失败\")\n + \"- 判定结果: `{}`\\n\".format(verdict)\n + \"- 失败分类: `{}`\\n\".format(failure_category or \"\")\n + \"- 尝试次数: {}\\n\\n---\\n\".format(attempts)\n)\nif batch_report_path:\n with open(batch_report_path, \"a\", encoding=\"utf-8\") as f:\n f.write(section)\n\noutputs[\"last_host\"] = host\noutputs[\"last_ssh_target\"] = ssh_target\noutputs[\"last_success\"] = ok\noutputs[\"last_verdict\"] = verdict\noutputs[\"last_failure_category\"] = failure_category\noutputs[\"last_per_host_md\"] = per_host_md" + "code": "import os\nimport re\nimport time\n\nhosts = inputs.get(\"hosts\", [])\nidx = int(inputs.get(\"host_idx\", 0))\nhost = hosts[idx] if 0 <= idx < len(hosts) else \"\"\nhost = str(host).strip()\n\nsu = inputs.get(\"ssh_user\")\nif isinstance(su, str):\n su = su.strip()\nelse:\n su = \"\"\n\nconnect_host = host\nconnect_user = \"\"\nif \"@\" in host:\n user_part, host_part = host.split(\"@\", 1)\n connect_user = str(user_part).strip()\n connect_host = str(host_part).strip() or host\n ssh_target = host\n user_hint = (\n \"主机列表项已含 `user@host` 形式。SSH 工具调用必须使用：host=`\"\n + connect_host\n + \"`，username=`\"\n + connect_user\n + \"`。\"\n )\nelif su:\n connect_user = su\n connect_host = host\n ssh_target = su + \"@\" + host\n user_hint = (\n \"工作流已指定 `ssh_user`=`\"\n + su\n + \"`。SSH 工具调用必须使用：host=`\"\n + connect_host\n + \"`，username=`\"\n + connect_user\n + \"`。\"\n )\nelse:\n connect_host = host\n ssh_target = host\n user_hint = (\n \"未指定 `ssh_user`。SSH 工具调用请使用 host=`\"\n + connect_host\n + \"`，username 留空使用默认账户（一般为 root）。\"\n )\n\n\ndef _extract_verdict(markdown_text):\n if not markdown_text:\n return \"UNKNOWN\"\n match = re.search(\n r\"(?im)^\\s*\\*{0,2}Verdict\\*{0,2}\\s*:\\s*\"\n r\"(CLEAN|SUSPICIOUS|COMPROMISED|UNKNOWN)\\b\",\n markdown_text,\n )\n if not match:\n return \"UNKNOWN\"\n return str(match.group(1)).upper()\n\n\ndef _is_timeout_error(message):\n text = str(message or \"\").lower()\n return (\n \"timed out\" in text\n or \"timeout\" in text\n or \"超时\" in text\n or \"节点执行超时\" in text\n )\n\n\ndef _classify_error(message):\n text = str(message or \"\")\n lower = text.lower()\n if not text.strip():\n return \"unknown\"\n if \"permission denied\" in lower or \"auth failed\" in lower or \"authentication failed\" in lower:\n return \"auth_failed\"\n if \"host key verification failed\" in lower or \"host key\" in lower:\n return \"host_key_verification_failed\"\n if \"connection refused\" in lower:\n return \"connection_refused\"\n if \"no route to host\" in lower:\n return \"no_route_to_host\"\n if \"network is unreachable\" in lower:\n return \"network_unreachable\"\n if \"name or service not known\" in lower or \"could not resolve\" in lower or \"nodename nor servname provided\" in lower:\n return \"dns_resolution_failed\"\n if \"connection reset\" in lower:\n return \"connection_reset\"\n if \"broken pipe\" in lower or \"connection lost\" in lower or \"disconnect\" in lower:\n return \"connection_lost\"\n if \"kex\" in lower or \"key exchange\" in lower or \"protocol error\" in lower:\n return \"ssh_handshake_failed\"\n if _is_timeout_error(text):\n if \"connect\" in lower or \"ssh connection failed\" in lower:\n return \"connect_timeout\"\n return \"execution_timeout\"\n if \"ssh connection failed\" in lower:\n return \"ssh_connection_failed\"\n return \"unknown\"\n\n\nidx1 = idx + 1\nper_host_dir = str(inputs.get(\"per_host_dir\") or \"\").strip()\nif not per_host_dir:\n od = str(inputs.get(\"output_dir\") or \"\").strip()\n if od:\n per_host_dir = os.path.join(od, \"host_triage\")\n else:\n per_host_dir = os.path.join(os.path.dirname(inputs.get(\"batch_report_path\") or \".\") or \".\", \"host_triage\")\nos.makedirs(per_host_dir, exist_ok=True)\n\n\ndef _slug(s):\n t = re.sub(r\"[^0-9A-Za-z._@-]+\", \"_\", str(s)).strip(\"_\")\n t = t.replace(\"@\", \"_at_\")\n return (t[:56] if t else \"host\")\n\n\nnh = len(hosts)\nbase = \"{:04d}_{}\".format(idx1, _slug(ssh_target))\nper_host_md = os.path.join(per_host_dir, base + \".md\")\n\npreflight_timeout_s = 20\npreflight_res = tool.run_safe(\n \"ssh_host_cmd\",\n host=connect_host,\n username=(connect_user or None),\n command=\"echo FLOCKS_SSH_OK\",\n timeout=preflight_timeout_s,\n)\npreflight_output = preflight_res.get(\"output\") or preflight_res.get(\"text\") or \"\"\npreflight_error = preflight_res.get(\"error\") or \"\"\npreflight_ok = bool(preflight_res.get(\"success\")) and \"FLOCKS_SSH_OK\" in str(preflight_output)\n\ntext = \"\"\nok = False\nerr = \"\"\nverdict = \"UNKNOWN\"\nfailure_category = \"\"\nattempts = 0\n\nif preflight_ok:\n desc = \"Fast triage item \" + str(idx1)\n prompt = (\n \"你是 host-forensics-fast 工作模式：对下列目标执行 Linux 主机快速安全巡检（首轮研判）。\\n\"\n \"请使用 ssh_run_script，script_path 为 `.flocks/plugins/agents/host-forensics-fast/scripts/triage_fast.sh`。\\n\"\n + user_hint\n + \"\\n\\n本次 SSH 工具参数必须使用：\\n\"\n + \"- host: \"\n + connect_host\n + \"\\n\"\n + (\"- username: \" + connect_user + \"\\n\" if connect_user else \"- username: （留空，使用默认账户）\\n\")\n + \"\\n请输出简洁 Markdown：结论、可疑项、风险判断、后续建议。\"\n )\n while attempts < 2:\n attempts += 1\n res = tool.run_safe(\n \"task\",\n description=desc + \" attempt \" + str(attempts),\n prompt=prompt,\n subagent_type=\"host-forensics-fast\",\n )\n text = res.get(\"text\") or \"\"\n ok = bool(res.get(\"success\"))\n err = res.get(\"error\") or \"\"\n if ok:\n verdict = _extract_verdict(text)\n break\n if not _is_timeout_error(err) or attempts >= 2:\n failure_category = _classify_error(err)\n break\n time.sleep(3)\n if not ok and not failure_category:\n failure_category = _classify_error(err)\nelse:\n err = preflight_error or \"SSH preflight failed\"\n failure_category = _classify_error(err)\n\nlines = [\n \"# 单主机快速巡检结果\",\n \"\",\n \"- 批次内序号: {} / {}\".format(idx1, nh),\n \"- 列表项 host: `{}`\".format(host),\n \"- ssh_target: `{}`\".format(ssh_target),\n \"- ssh_host: `{}`\".format(connect_host),\n \"- ssh_user: `{}`\".format(connect_user or \"(default)\"),\n \"- success: {}\".format(ok),\n \"- verdict: {}\".format(verdict),\n \"- failure_category: {}\".format(failure_category or \"\"),\n \"- inspect_attempts: {}\".format(attempts),\n \"\",\n]\nif not preflight_ok:\n lines.extend(\n [\n \"## SSH 预检失败\",\n \"\",\n \"- 分类: `{}`\".format(failure_category),\n \"- 错误:\",\n \"\",\n \"```\",\n str(err),\n \"```\",\n \"\",\n ]\n )\nelif err:\n lines.extend([\"## 错误\", \"\", \"```\", str(err), \"```\", \"\"])\nelse:\n lines.extend([\"## 子 Agent 输出\", \"\", (text if text else \"_(无正文)_\"), \"\"])\nwith open(per_host_md, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\\n\".join(lines))\n\ntr = inputs.get(\"triage_results\", [])\ntr = list(tr) if isinstance(tr, list) else []\ntr.append(\n {\n \"host\": host,\n \"ssh_user\": connect_user or su,\n \"ssh_target\": ssh_target,\n \"ssh_host\": connect_host,\n \"success\": ok,\n \"verdict\": verdict,\n \"failure_category\": failure_category,\n \"inspect_attempts\": attempts,\n \"error\": err,\n \"per_host_md\": per_host_md,\n }\n)\noutputs[\"triage_results\"] = tr\n\nbatch_report_path = inputs.get(\"batch_report_path\", \"\")\nsection = (\n \"\\n## [{}/{}] `{}`\\n\\n\".format(idx1, nh, ssh_target)\n + \"- 单独报告: `{}`\\n\".format(per_host_md)\n + \"- 执行结果: {}\\n\".format(\"成功\" if ok else \"失败\")\n + \"- 判定结果: `{}`\\n\".format(verdict)\n + \"- 失败分类: `{}`\\n\".format(failure_category or \"\")\n + \"- 尝试次数: {}\\n\\n---\\n\".format(attempts)\n)\nif batch_report_path:\n with open(batch_report_path, \"a\", encoding=\"utf-8\") as f:\n f.write(section)\n\noutputs[\"last_host\"] = host\noutputs[\"last_ssh_target\"] = ssh_target\noutputs[\"last_success\"] = ok\noutputs[\"last_verdict\"] = verdict\noutputs[\"last_failure_category\"] = failure_category\noutputs[\"last_per_host_md\"] = per_host_md" }, { "id": "advance_index", diff --git a/.gitignore b/.gitignore index 344937d93..150d568c6 100644 --- a/.gitignore +++ b/.gitignore @@ -98,8 +98,8 @@ tmp/ .ipynb_checkpoints *.ipynb -# Documentation -docs/_build/ +# Documentation (local / agent-generated; not versioned) +docs/ site/ # Node.js (TUI) @@ -107,7 +107,6 @@ node_modules/ tui/node_modules/ bun.lockb .bun/ -!docs/CHANGELOG.md # TUI build tui/dist/ @@ -134,3 +133,4 @@ changelog.md # Integration tests with real host credentials (not for commit) tests/integration_ssh_*.py .cursor +.codex/ diff --git a/AGENTS.md b/AGENTS.md index 877b39a35..943b60e81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,29 @@ --- +## PowerShell 脚本编码规范 + +新建或修改 `.ps1` 文件后，必须使用 **UTF-8 with BOM**（文件头 `EF BB BF`）+ **CRLF** 换行符。 + +原因：Windows PowerShell 5.1 读取无 BOM 的 UTF-8 文件时会使用系统代码页（如 GBK）解码，可能导致中文字符字节错位，引号/大括号被吞，产生级联解析错误。 + +验证编码（PowerShell）： + +```powershell +$bytes = [System.IO.File]::ReadAllBytes("script.ps1") +$bytes[0..2] | ForEach-Object { $_.ToString("X2") } +# 正确输出：EF BB BF +``` + +保存为正确编码： + +```powershell +$content = Get-Content "script.ps1" -Raw +[System.IO.File]::WriteAllText("script.ps1", $content, [System.Text.UTF8Encoding]::new($true)) +``` + +--- + ## Capability Gap Resolution Protocol **This protocol is mandatory for Rex and all primary agents.** @@ -150,4 +173,4 @@ Rex has a dedicated `flocks_skills` tool for managing agent skills. ## Important - 涉及 `tdp`、`onesec`、`skyeye`、`qingteng` 的任务时，必须先读取并遵循对应的 skill。 -- 对上述系统，禁止绕过对应 skill 直接调用相关 tools；也不要直接使用 `browser`。 \ No newline at end of file +- 对上述系统，禁止绕过对应 skill 直接调用相关 tools；也不要直接使用 `browser`。 diff --git a/docs/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 65% rename from docs/CONTRIBUTING.md rename to CONTRIBUTING.md index 67b9a581d..9ce3ca5e0 100644 --- a/docs/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,29 +131,90 @@ If you are fixing a bug, prefer adding a regression test that reproduces the iss All contribution PRs for `flocks` should target the `dev` branch. -When opening a PR, make it easy for reviewers to understand: +When opening a PR, structure the description so reviewers can quickly identify the change points, the scope of impact, and the business logic that needs careful review. A good PR description answers three core questions: -1. What problem the change solves. -2. What the scope of the change is. -3. Why the chosen approach is appropriate. -4. How you validated the change. -5. Whether there are compatibility, migration, or configuration impacts. +1. **What changed** — the concrete change points in this PR. +2. **What is affected** — the impact scope (users, APIs, configuration, dependencies, performance). +3. **Where to look closely** — the business logic, invariants, and edge cases that reviewers should focus on. -If the PR changes UI or interaction behavior, include screenshots, recordings, or a clear before/after explanation. +### 1. Key Changes (改动点) + +List the concrete changes grouped by area (backend, frontend, docs, tests, etc.). Each bullet should describe one reviewable change, not a vague summary. + +- What was added, modified, or removed. +- Which modules, files, or APIs are touched. +- Any new public interfaces, configuration options, environment variables, or CLI flags. + +### 2. Impact Scope (影响范围) + +Explain who and what is affected. Cover every category that applies, and state explicitly when a category has no impact (e.g. "No public API change."). + +- **User-visible behavior**: UX, UI, CLI, API responses, log/event output, or output format changes. Include screenshots or recordings for UI changes. +- **Compatibility**: backward-compatibility impact, deprecations, default-value changes, or required migrations. +- **Configuration & environment**: new required settings, environment variables, secrets, or deployment changes. +- **Dependencies**: any newly added or upgraded third-party packages, and why they are needed. +- **Performance & resources**: expected impact on latency, memory, CPU, network, or storage. +- **Security & permissions**: authentication, authorization, data-privacy, or secret-handling touch points. + +### 3. Business Logic to Focus On During Review (需重点 Review 的业务逻辑) + +Call out the parts of the change that deserve extra reviewer attention. This section is the most valuable part of the PR description — it shortens review time and reduces back-and-forth. + +- The specific functions, classes, endpoints, or flows that implement the core logic. +- Assumptions, preconditions, or invariants the code relies on, and why they hold. +- Edge cases and error paths that are easy to miss (empty input, partial failure, retry, timeout, concurrency, ordering). +- Any cross-module contract (e.g. how this change interacts with existing APIs, plugins, or workflows). +- Anything you are uncertain about and would like a second opinion on — say so explicitly. + +### 4. Why This Approach + +Briefly justify the chosen approach over reasonable alternatives, especially for non-trivial changes. + +### 5. Test Plan & Validation + +- Which tests you added or updated, and which suites you ran. +- Any manual verification, reproduction scripts, or staging checks. +- For UI changes, attach before/after screenshots or short recordings. + +### 6. Compatibility, Migration & Rollback + +- Any breaking changes and the migration path. +- New or changed configuration, environment variables, or feature flags. +- Rollback strategy if the change is risky. + +--- Recommended PR description template: ```markdown ## Summary +- One or two sentences stating the goal of this PR. + +## Key Changes - ... -## Why +## Impact Scope +- User-visible behavior: +- Compatibility / migration: +- Configuration / environment: +- Dependencies: +- Performance / resources: +- Security / permissions: + +## Business Logic to Review +- ... +- ... + +## Why This Approach - ... ## Test Plan - [x] uv run pytest ... - [x] npm run lint - [ ] Manual verification + +## Compatibility, Migration & Rollback +- ... ``` Please keep PRs as small and focused as practical. Multiple reviewable PRs are usually easier to merge than one large mixed change. diff --git a/README.md b/README.md index 60541dde7..838ab0058 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ Scan the QR code with **WeChat** to join our official discussion group. ## 6. Contributing -See [`docs/CONTRIBUTING.md`](docs/CONTRIBUTING.md) for development setup, coding standards, testing expectations, and Pull Request guidelines. +See [`CONTRIBUTING.md`](CONTRIBUTING.md) for development setup, coding standards, testing expectations, and Pull Request guidelines. ## 7. License diff --git a/README_zh.md b/README_zh.md index 65566a394..f84f6866e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -287,7 +287,7 @@ flocks start --server-host 0.0.0.0 --webui-host 0.0.0.0 ## 6. 参与贡献 -开发环境、代码规范、测试要求和 Pull Request 流程请参考 [`docs/CONTRIBUTING.md`](docs/CONTRIBUTING.md)。 +开发环境、代码规范、测试要求和 Pull Request 流程请参考 [`CONTRIBUTING.md`](CONTRIBUTING.md)。 ## 7. 开源协议 diff --git a/docs/design/flocks-release-upgrade-technical-design.md b/docs/design/flocks-release-upgrade-technical-design.md deleted file mode 100644 index af74d5146..000000000 --- a/docs/design/flocks-release-upgrade-technical-design.md +++ /dev/null @@ -1,653 +0,0 @@ -# Flocks Release And Upgrade Technical Design - -## 目标 - -本文档描述当前已实现的 Flocks OSS、Flocks Pro、Flocks Console 三仓发布、构建、升级流程。 - -核心设计原则： - -- OSS 版继续使用 GitHub Release 作为版本源。 -- Pro 版客户端只信任 Console 下发的 Pro bundle manifest。 -- Pro bundle 是 Console 侧组合发布物，由 OSS core artifact 和 latest Pro wheel artifact 合成。 -- `flockspro` 私有仓只发布企业组件 wheel，不直接构建客户可升级的 bundle。 -- 从 OSS 升级到 Pro 走 Console 审核、license 激活、Pro bundle 安装和安装回执闭环。 - -## 仓库职责 - -### `flocks` - -`flocks` 仓负责 OSS 代码发布和客户端升级执行框架。 - -当前相关实现： - -- GitHub Release 是 OSS 用户检查升级的版本源。 -- `.github/workflows/trigger-pro-bundle.yml` 在 OSS release 发布后向 Console 上报 core artifact。 -- `flocks/updater/updater.py` 负责版本检查、下载、校验、备份、替换、依赖同步、重启和回滚。 -- `flocks/server/routes/cloud_upgrade.py` 负责 OSS 到 Pro 的升级申请、审核状态同步、license 激活、自动安装和安装回执上报。 - -### `flockspro` - -`flockspro` 仓负责企业功能组件，不再负责 Pro bundle 构建。 - -当前相关实现： - -- `.github/workflows/release-wheel.yml` 在 Pro release 或手动触发时构建 wheel。 -- 构建完成后计算 wheel sha256，并向 Console 上报 Pro wheel artifact。 -- `src/flockspro/updater/manifest.py` 定义 Pro bundle manifest 客户端合约。 -- `src/flockspro/updater/source.py` 提供从 Console `/v1/manifest/latest` 读取 Pro bundle manifest 的 updater source。 - -### `flocks_console` - -`flocks_console` 是 Pro 发布升级控制面。 - -当前相关实现： - -- `src/flocks_console/app/manifest_service.py` 保存 core artifact、Pro wheel artifact、bundle build job、bundle release、安装回执。 -- `src/flocks_console/app/pro_bundle_builder.py` 在 Console 侧合成 Pro bundle。 -- `src/flocks_console/app/main.py` 提供 artifact 上报、build job、latest manifest、冻结、回滚、安装回执 API。 -- `web/app/console/upgrade-requests/page.tsx` 和 `web/app/_components/upgrade-review-modal.tsx` 展示 latest core、latest Pro wheel、latest bundle、build job、安装回执。 - -## 总体流程 - -```mermaid -flowchart TD - ossRelease["OSS GitHub Release"] --> ossUser["OSS User Upgrade From GitHub"] - ossRelease --> coreArtifact["Publish Core Artifact To Console"] - proRelease["FlocksPro Release"] --> proWheel["Publish Pro Wheel Artifact To Console"] - coreArtifact --> consoleStore["Console Artifact Store"] - proWheel --> consoleStore - consoleStore --> buildJob["Console Bundle Build Job"] - buildJob --> proBundle["Pro Bundle"] - proBundle --> latestManifest["Console Latest Manifest"] - latestManifest --> proClient["Flocks Pro Client Upgrade"] - proClient --> installReceipt["Install Receipt"] - installReceipt --> consoleStore -``` - -## 版本规则 - -### OSS 版本 - -OSS release tag 是 OSS 用户和 Pro bundle 的主版本来源，例如： - -```text -v2026.5.18 -``` - -OSS 用户看到的是： - -```text -Flocks v2026.5.18 -``` - -### Pro 组件版本 - -`flockspro` 私有包有独立组件版本，例如： - -```text -pro-v2026-5-10 -``` - -该版本只表示 Pro wheel 组件版本，不直接作为客户升级版本。 - -### Pro 对外版本 - -Pro 用户对外展示版本与 OSS release 保持一致，例如： - -```text -Flocks Pro v2026.5.18 -``` - -Console manifest 中： - -```json -{ - "display_version": "v2026.5.18", - "compare_version": "2026.5.18", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10" -} -``` - -`display_version` 用于 UI 展示，`compare_version` 用于客户端版本比较，`flockspro_component_version` 只作为详情和排障字段。 - -## OSS Release 与升级流程 - -### 发布流程 - -1. `flocks` 仓创建 GitHub Release，例如 `v2026.5.18`。 -2. OSS 用户本地 updater 按原有 Release API 检查 GitHub/Gitee/GitLab sources。 -3. `.github/workflows/trigger-pro-bundle.yml` 同时被 release published 触发。 -4. workflow 下载 OSS release archive，计算 sha256。 -5. workflow 向 Console 调用： - -```http -POST /v1/ops/artifacts/flocks-core -``` - -请求字段： - -```json -{ - "oss_version": "v2026.5.18", - "archive_url": "https://github.com/AgentFlocks/flocks/archive/refs/tags/v2026.5.18.tar.gz", - "archive_sha256": "...", - "release_notes": "...", - "source_repo": "AgentFlocks/flocks", - "github_release_id": "...", - "published_at": "2026-05-18T00:00:00Z" -} -``` - -### OSS 客户端升级 - -OSS 用户不依赖 Console。 - -客户端流程： - -1. `check_update()` 调用 GitHub/Gitee/GitLab release source。 -2. 获取 latest tag、release notes、archive URL。 -3. 比较 latest tag 与当前版本。 -4. 用户确认升级后调用 `perform_update()`。 -5. 下载 archive。 -6. 备份当前安装目录到 `~/.flocks/version/`。 -7. 解压新版本源码。 -8. 构建前端资源。 -9. 替换安装目录。 -10. 运行 `uv sync`。 -11. 写入版本 marker。 -12. 重启服务。 -13. 失败时尽量从备份回滚。 - -## FlocksPro Release 与 Pro Wheel Artifact - -### 发布流程 - -1. `flockspro` 仓创建 release，例如 `pro-v2026-5-10`。 -2. `.github/workflows/release-wheel.yml` 被 release published 触发，或手动 workflow_dispatch 触发。 -3. workflow 使用 `uv build --wheel --out-dir dist` 构建 wheel。 -4. workflow 计算 wheel sha256。 -5. workflow 拼出 wheel URL。 -6. workflow 向 Console 调用： - -```http -POST /v1/ops/artifacts/flockspro-wheel -``` - -请求字段： - -```json -{ - "pro_version": "pro-v2026-5-10", - "wheel_url": "https://cdn.agentflocks.com/flockspro/wheels/pro-v2026-5-10/flockspro-0.1.0-py3-none-any.whl", - "wheel_name": "flockspro-0.1.0-py3-none-any.whl", - "wheel_sha256": "...", - "release_notes": "...", - "source_repo": "AgentFlocks/flockspro", - "github_release_id": "...", - "published_at": "2026-05-10T00:00:00Z" -} -``` - -### 不触发 bundle - -Pro wheel artifact 上报只更新 Console 中 latest Pro 组件，不自动创建客户可升级 bundle。 - -原因： - -- Pro 组件独立迭代不应直接推动客户升级。 -- 客户可升级版本以 OSS release 版本为主版本。 -- 下一个 OSS release 会自动组合当前 latest Pro wheel。 - -## Console Artifact Store - -Console 维护以下数据： - -### `core_artifacts` - -保存 OSS release artifact： - -- `artifact_id` -- `oss_version` -- `archive_url` -- `archive_sha256` -- `release_notes` -- `source_repo` -- `github_release_id` -- `published_at` -- `is_latest` -- `metadata` - -### `pro_wheel_artifacts` - -保存 Pro wheel artifact： - -- `artifact_id` -- `pro_version` -- `wheel_url` -- `wheel_name` -- `wheel_sha256` -- `release_notes` -- `source_repo` -- `github_release_id` -- `published_at` -- `is_latest` -- `metadata` - -### `pro_bundle_build_jobs` - -保存 bundle 构建任务： - -- `job_id` -- `core_artifact_id` -- `pro_artifact_id` -- `release_id` -- `channel` -- `status` -- `reason` -- `error_message` -- `created_at` -- `updated_at` -- `metadata` - -### `pro_bundle_releases` - -保存已成功发布的 Pro bundle： - -- `release_id` -- `channel` -- `display_version` -- `compare_version` -- `oss_version` -- `flockspro_component_version` -- `bundle_url` -- `bundle_sha256` -- `build_id` -- `release_notes` -- `published_at` -- `is_latest` -- `is_frozen` -- `metadata` - -### `pro_bundle_installations` - -保存客户端安装回执： - -- `id` -- `release_id` -- `license_id` -- `passport_uid` -- `fingerprint` -- `install_id` -- `installed_version` -- `oss_version` -- `flockspro_component_version` -- `build_id` -- `install_result` -- `error_message` -- `reported_at` - -## Console Bundle Build 流程 - -### 自动触发 - -当 Console 收到 core artifact 时： - -1. 写入 `core_artifacts`，并设为 latest core。 -2. 查找 latest Pro wheel artifact。 -3. 如果存在 latest Pro wheel，创建 `pro_bundle_build_jobs`。 -4. 当前实现会同步执行 build job。 -5. 构建成功后写入 `pro_bundle_releases`，并设为 latest。 -6. 如果不存在 latest Pro wheel，只保存 core artifact，不创建 bundle。 - -### 手动触发 - -Console 提供手动构建 API： - -```http -POST /v1/ops/pro-bundles/builds -``` - -请求可指定： - -```json -{ - "core_artifact_id": "core_...", - "pro_artifact_id": "prowhl_...", - "channel": "flockspro", - "reason": "manual rebuild" -} -``` - -如果不指定 artifact id，则默认使用 latest core 和 latest Pro wheel。 - -### 构建内容 - -`src/flocks_console/app/pro_bundle_builder.py` 负责生成 bundle。 - -输入： - -- core archive URL -- core archive sha256 -- Pro wheel URL -- Pro wheel sha256 -- OSS version -- Pro component version -- channel -- build id -- release notes - -构建步骤： - -1. 下载或复制 core archive。 -2. 校验 core archive sha256。 -3. 下载或复制 Pro wheel。 -4. 校验 Pro wheel sha256。 -5. 解压 core archive。 -6. 将 core 源码放入 bundle 的 `flocks/`。 -7. 将 Pro wheel 放入 bundle 的 `wheels/`。 -8. 生成 bundle 内部 `manifest.json`。 -9. 生成 `checksums.txt`。 -10. 打包为 `flockspro-bundle-vYYYY.M.D.tar.gz`。 -11. 计算 bundle sha256。 -12. 写入本地 dev storage 或对象存储对应目录。 - -Bundle 结构： - -```text -flockspro-bundle-v2026.5.18.tar.gz -├── flocks/ -├── wheels/ -│ └── flockspro-*.whl -├── manifest.json -└── checksums.txt -``` - -内部 `manifest.json`： - -```json -{ - "schema_version": 1, - "display_version": "v2026.5.18", - "compare_version": "2026.5.18", - "channel": "flockspro", - "edition": "flockspro", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10", - "flockspro_wheel": "wheels/flockspro-0.1.0-py3-none-any.whl", - "build_id": "job_...", - "published_at": "2026-05-18T00:00:00Z", - "requires_license_status": ["trial", "test", "commercial"], - "release_notes": "..." -} -``` - -如果配置了 `FLOCKSPRO_MANIFEST_SIGNING_SECRET`，内部 manifest 会附加 `manifest_signature`。 - -## Console Manifest 下发 - -Pro 客户端检查升级时访问： - -```http -GET /v1/manifest/latest?channel=flockspro -``` - -Console 会： - -1. 校验 cloud session。 -2. 读取 `pro_bundle_releases` 中 latest release。 -3. 如果 release 已 frozen，则拒绝下发。 -4. 返回 bundle manifest。 -5. 如果配置了 `FLOCKSPRO_MANIFEST_SIGNING_SECRET`，返回 manifest_signature。 - -返回示例： - -```json -{ - "schema_version": 1, - "edition": "flockspro", - "display_version": "v2026.5.18", - "compare_version": "2026.5.18", - "channel": "flockspro", - "bundle_url": "https://cdn.agentflocks.com/flockspro/v2026.5.18/flockspro-bundle-v2026.5.18.tar.gz", - "bundle_sha256": "...", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10", - "build_id": "job_...", - "published_at": "2026-05-18T00:00:00Z", - "requires_license_status": ["trial", "test", "commercial"], - "release_notes": "..." -} -``` - -## Pro 客户端升级流程 - -### Source 锁定 - -`flocks/updater/updater.py` 中 `_resolve_sources_for_edition()` 会检测： - -- `FLOCKS_EDITION=flockspro` -- 或本地存在 cloud session - -只要进入 Pro edition，升级源强制为： - -```python -["cloud-manifest"] -``` - -Pro 用户不会回退到 GitHub/Gitee/GitLab OSS source。 - -### 检查更新 - -Pro 客户端调用 `_fetch_cloud_manifest_release()`： - -1. 读取 `FLOCKS_MANIFEST_BASE_URL`。 -2. 默认 channel 为 `flockspro`。 -3. 携带 cloud session token 调用 Console `/v1/manifest/latest`。 -4. 检查 `frozen` 和 `frozen_until`。 -5. 读取 `compare_version` 作为比较版本。 -6. 读取 `bundle_url` 作为下载 URL。 -7. 缓存 manifest，用于后续 bundle sha256 校验。 - -### 执行升级 - -`perform_update()` 对 Pro bundle 执行以下步骤： - -1. 下载 bundle。 -2. 使用 manifest 中的 `bundle_sha256` 校验下载文件。 -3. 备份当前安装目录。 -4. 解压 bundle。 -5. 识别 bundle 结构： - - 如果存在 `manifest.json` 和 `flocks/`，认为是 Pro bundle。 - - 使用 `flocks/` 作为 OSS 源码根目录。 - - 从 `manifest.json` 的 `flockspro_wheel` 或 `wheels/*.whl` 找到 Pro wheel。 -6. 预构建前端。 -7. 替换安装目录。 -8. 运行 `uv sync`。 -9. 使用 `uv pip install --python .venv/bin/python wheels/flockspro-*.whl` 安装 Pro wheel。 -10. 写入 `~/.flocks/run/pro-bundle-installed.json`，记录： - - `installed_version` - - `oss_version` - - `flockspro_component_version` - - `build_id` - - `installed_at` -11. 写入当前版本 marker。 -12. 刷新 CLI entry。 -13. 重启服务或在自动安装场景下跳过 restart。 -14. 失败时从备份回滚。 - -## 从 OSS 升级到 Pro 的流程 - -OSS 到 Pro 是一次 edition switch，不是普通 OSS release 升级。 - -### 申请阶段 - -1. OSS 用户在本地发起 Pro 升级申请。 -2. `flocks/server/routes/cloud_upgrade.py` 创建本地升级申请记录。 -3. 客户端要求已有 cloud binding session。 -4. OSS 节点向 Console 创建 upgrade request。 -5. Console 审核台展示申请信息、latest core、latest Pro wheel、latest bundle、build job、安装回执。 - -### 审核阶段 - -1. Console 运维审核申请。 -2. 审核通过后生成 activate key。 -3. 审核记录绑定当前 latest Pro bundle 的 `manifest_url`。 -4. OSS 客户端刷新申请状态。 - -### 激活与安装阶段 - -当 OSS 客户端发现申请状态为 `approved`： - -1. `_maybe_activate_pro_license()` 调用 Pro license checker 激活 license。 -2. `_maybe_refresh_pro_license()` 刷新 cloud license 状态。 -3. `_run_auto_upgrade_install()` 调用 `check_update()`。 -4. 由于已进入 Pro 流程，升级源为 Console cloud manifest。 -5. 下载并安装 latest Pro bundle。 -6. 安装成功后本地状态变为 `activated`。 - -### 回执阶段 - -安装完成后： - -1. 客户端读取 `~/.flocks/run/pro-bundle-installed.json`。 -2. 调用 Console： - -```http -POST /v1/pro-bundles/installations -``` - -请求字段： - -```json -{ - "license_id": "act_...", - "fingerprint": "...", - "install_id": "...", - "installed_version": "v2026.5.18", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10", - "build_id": "job_...", - "install_result": "success", - "reported_at": "2026-05-18T00:00:00Z" -} -``` - -失败时 `install_result` 为 `failed`，并附带 `error_message`。 - -## 运维 API - -### Artifact API - -```http -POST /v1/ops/artifacts/flocks-core -GET /v1/ops/artifacts/flocks-core -POST /v1/ops/artifacts/flockspro-wheel -GET /v1/ops/artifacts/flockspro-wheel -``` - -### Bundle Build API - -```http -POST /v1/ops/pro-bundles/builds -GET /v1/ops/pro-bundles/builds -``` - -### Bundle Release API - -```http -POST /v1/ops/pro-bundles/publish -GET /v1/ops/pro-bundles -POST /v1/ops/pro-bundles/{release_id}/freeze -POST /v1/ops/pro-bundles/{release_id}/promote -``` - -`publish` 当前保留为内部发布步骤或兼容运维入口。正常流程中，成功的 Console build job 会自动写入 `pro_bundle_releases`。 - -### Installation API - -```http -POST /v1/pro-bundles/installations -GET /v1/ops/pro-bundles/installations -``` - -## 冻结与回滚 - -### 冻结 - -如果某个 Pro bundle 有问题，运维可调用： - -```http -POST /v1/ops/pro-bundles/{release_id}/freeze -``` - -冻结后 latest manifest 不再向客户端下发该 release。 - -### 回滚 - -如果需要回滚到旧 bundle，运维可调用： - -```http -POST /v1/ops/pro-bundles/{release_id}/promote -``` - -该 release 会成为 latest，并解除 frozen 状态。 - -## 配置项 - -### GitHub Actions Secrets - -`flocks` 仓： - -- `FLOCKS_CONSOLE_API_BASE` -- `FLOCKS_CONSOLE_OPS_TOKEN` - -`flockspro` 仓： - -- `FLOCKS_CONSOLE_API_BASE` -- `FLOCKS_CONSOLE_OPS_TOKEN` -- `FLOCKSPRO_WHEEL_BASE_URL` - -### Console 环境变量 - -- `FLOCKS_CONSOLE_OPS_TOKEN`：保护 ops API。 -- `FLOCKSPRO_MANIFEST_SIGNING_SECRET`：签名 manifest。 -- `FLOCKS_CONSOLE_BUNDLE_DIR`：本地 bundle 存储目录，默认 `~/.flocks/console/bundles`。 -- `FLOCKS_CONSOLE_BUNDLE_BASE_URL`：bundle 对外访问 base URL，默认 `https://cdn.agentflocks.com/flockspro`。 - -### Pro 客户端环境变量 - -- `FLOCKS_EDITION=flockspro`：强制进入 Pro edition。 -- `FLOCKS_MANIFEST_BASE_URL`：Console manifest base URL。 -- `FLOCKS_UPDATE_CHANNEL=flockspro`：Pro bundle channel。 - -## 当前实现边界 - -当前实现已经完成主链路： - -- OSS release 上报 core artifact。 -- Pro release 上报 wheel artifact。 -- Console 保存 artifact。 -- Console 根据 latest core + latest Pro wheel 构建 bundle。 -- Console 下发 latest manifest。 -- Pro 客户端下载 bundle、校验 sha256、安装 OSS core 和 Pro wheel。 -- OSS 到 Pro 通过审核、license 激活、bundle 安装、安装回执闭环。 - -仍需部署侧保证： - -- workflow 中上报的 `archive_url`、`wheel_url` 必须能被 Console builder 下载。 -- 如果使用对象存储/CDN，需要由 CI 或发布平台先上传 artifact，再上报 Console。 -- 当前 Console builder 在 API 进程内同步执行，生产环境建议迁移为后台 worker/job runner。 -- `FLOCKS_CONSOLE_BUNDLE_BASE_URL` 需要指向真实可下载的 bundle 对外地址。 - -## 测试覆盖 - -当前相关测试： - -- `flockspro/tests/test_manifest_contract.py`：Pro manifest 合约解析与签名。 -- `flocks/tests/updater/test_updater_cloud_manifest_bundle.py`：Pro cloud manifest bundle URL、冻结逻辑。 -- `flocks/tests/updater/test_updater_edition_sources.py`：Pro edition 强制 cloud manifest。 -- `flocks/tests/server/routes/test_cloud_upgrade_routes.py`：OSS 到 Pro 升级申请、自动激活、安装触发。 -- `flocks_console/tests/test_api.py`：artifact 上报、bundle build、latest manifest、安装回执。 -- `flocks_console/tests/test_schema_migrations.py`：schema 表结构覆盖。 - diff --git a/flocks/acp/agent.py b/flocks/acp/agent.py index 3fbf8533b..3b9796e51 100644 --- a/flocks/acp/agent.py +++ b/flocks/acp/agent.py @@ -512,10 +512,20 @@ async def _handle_tool_part_update(self, session_id: str, part: Dict[str, Any]) "newText": new_text, }) - # Handle todowrite - send plan update - if tool_name == "todowrite": + # Handle todo writes - send plan update + if tool_name == "todo": try: - todos = json.loads(output) + metadata = state.get("metadata") or {} + parsed_output = json.loads(output) if output else [] + todos = ( + metadata.get("newTodos") + or metadata.get("todos") + or ( + parsed_output.get("newTodos") + if isinstance(parsed_output, dict) + else parsed_output + ) + ) if isinstance(todos, list): entries = [] for todo in todos: diff --git a/flocks/agent/agent.py b/flocks/agent/agent.py index f2f61f90d..e224aa24b 100644 --- a/flocks/agent/agent.py +++ b/flocks/agent/agent.py @@ -118,6 +118,9 @@ class AgentInfo(BaseModel): model_config = {"populate_by_name": True} name: str + # Chinese display name for localized UI. The canonical ``name`` remains the + # stable identifier used by tools, routing, storage, and @mentions. + name_cn: Optional[str] = None description: Optional[str] = None # Chinese UI label; English ``description`` is used for delegation prompts / tooling. description_cn: Optional[str] = None diff --git a/flocks/agent/agent_factory.py b/flocks/agent/agent_factory.py index 63981e62b..d8c08f265 100644 --- a/flocks/agent/agent_factory.py +++ b/flocks/agent/agent_factory.py @@ -163,9 +163,13 @@ def load_agent(agent_dir: Path, native: bool = False) -> Optional[AgentInfo]: desc_cn = raw.get("description_cn") if desc_cn is None and isinstance(raw.get("descriptionCn"), str): desc_cn = raw.get("descriptionCn") + name_cn = raw.get("name_cn") + if name_cn is None and isinstance(raw.get("nameCn"), str): + name_cn = raw.get("nameCn") return AgentInfo( name=name, + name_cn=name_cn, description=raw.get("description"), description_cn=desc_cn, mode=raw.get("mode", "subagent"), @@ -393,9 +397,13 @@ def yaml_to_agent_info(raw: dict, yaml_path: Path) -> AgentInfo: desc_cn = raw.get("description_cn") if desc_cn is None and isinstance(raw.get("descriptionCn"), str): desc_cn = raw.get("descriptionCn") + name_cn = raw.get("name_cn") + if name_cn is None and isinstance(raw.get("nameCn"), str): + name_cn = raw.get("nameCn") return AgentInfo( name=name, + name_cn=name_cn, description=raw.get("description"), description_cn=desc_cn, mode=raw.get("mode", "subagent"), diff --git a/flocks/agent/agents/explore/agent.yaml b/flocks/agent/agents/explore/agent.yaml index 538b01dcd..55c6048dc 100644 --- a/flocks/agent/agents/explore/agent.yaml +++ b/flocks/agent/agents/explore/agent.yaml @@ -1,4 +1,5 @@ name: explore +name_cn: 代码探索智能体 description: >- Fast agent specialized for exploring codebases. Use this when you need to quickly find files by patterns (eg. "src/components/**/*.tsx"), search code @@ -32,4 +33,4 @@ prompt_metadata: - You know exactly what to search - Single keyword/pattern suffices - Known file location - key_trigger: "2+ modules involved -> fire `explore` background" + key_trigger: "2+ modules involved -> delegate to `explore`" diff --git a/flocks/agent/agents/hephaestus/agent.yaml b/flocks/agent/agents/hephaestus/agent.yaml index d2e2de631..feacd0ca1 100644 --- a/flocks/agent/agents/hephaestus/agent.yaml +++ b/flocks/agent/agents/hephaestus/agent.yaml @@ -1,7 +1,8 @@ name: hephaestus +name_cn: 深度执行智能体 description: >- - Autonomous deep worker. Explores thoroughly before acting, uses - explore/librarian agents for comprehensive context, completes tasks end-to-end. + Autonomous deep worker. Explores thoroughly with direct tools before acting + and completes tasks end-to-end without delegating further. mode: subagent hidden: false tags: [system] @@ -18,12 +19,7 @@ tools: - websearch - webfetch - skill_load - - delegate_task - - task - - todoread - - todowrite + - todo - run_workflow - run_workflow_node - - background_output - - background_cancel # prompt is left empty – injected dynamically by prompt_builder.py diff --git a/flocks/agent/agents/hephaestus/prompt_builder.py b/flocks/agent/agents/hephaestus/prompt_builder.py index c92611914..31d91f859 100644 --- a/flocks/agent/agents/hephaestus/prompt_builder.py +++ b/flocks/agent/agents/hephaestus/prompt_builder.py @@ -115,7 +115,7 @@ def build_hephaestus_prompt( |------|--------|--------| | **Trivial** | Single file, known location, <10 lines | Direct tools only (UNLESS Key Trigger applies) | | **Explicit** | Specific file/line, clear command | Execute directly | -| **Exploratory** | "How does X work?", "Find Y" | Fire explore (1-3) + tools in parallel | +| **Exploratory** | "How does X work?", "Find Y" | Use direct search/read tools in parallel | | **Open-ended** | "Improve", "Refactor", "Add feature" | Full Execution Loop required | | **Ambiguous** | Unclear scope, multiple interpretations | Ask ONE clarifying question | @@ -177,9 +177,8 @@ def build_hephaestus_prompt( **Exploration Hierarchy (MANDATORY before any question):** 1. **Direct tools**: `gh pr list`, `git log`, `grep`, `rg`, file reads -2. **Explore agents**: Fire 2-3 parallel background searches -3. **Librarian agents**: Check docs, GitHub, external sources -4. **Context inference**: Use surrounding context to make educated guess +2. **Documentation/source checks**: Check docs, GitHub, external sources directly +3. **Context inference**: Use surrounding context to make educated guess 5. **LAST RESORT**: Ask ONE precise question (only if 1-4 all failed) ## Phase 1 - Systematic Exploration @@ -233,7 +232,7 @@ def build_hephaestus_prompt( - User's request fully addressed Before final response: -- Cancel background tasks: `background_cancel(all=true)` +- Summarize verification and any remaining uncertainty. """ prompt = template @@ -298,17 +297,17 @@ def _todo_discipline_section(use_task_system: bool) -> str: | Trigger | Action | |---------|--------| -| 2+ step task | `todowrite` FIRST, atomic breakdown | -| Uncertain scope | `todowrite` to clarify thinking | +| 2+ step task | `todo(action="write")` FIRST, atomic breakdown | +| Uncertain scope | `todo(action="write")` to clarify thinking | | Complex single task | Break down into trackable steps | ### Workflow (STRICT) -1. **On task start**: `todowrite` with atomic steps-no announcements, just create +1. **On task start**: `todo(action="write")` with atomic steps-no announcements, just create 2. **Before each step**: Mark `in_progress` (ONE at a time) 3. **After each step**: Mark `completed` IMMEDIATELY (NEVER batch) 4. **Scope changes**: Update todos BEFORE proceeding -5. **Todo payload shape**: `todowrite` must receive structured objects with `id`, `content`, and `status`, never a string array +5. **Todo payload shape**: `todo(action="write")` must receive structured objects with `id`, `content`, and `status`, never a string array ### Why This Matters diff --git a/flocks/agent/agents/librarian/agent.yaml b/flocks/agent/agents/librarian/agent.yaml index f333a960a..c65380ba9 100644 --- a/flocks/agent/agents/librarian/agent.yaml +++ b/flocks/agent/agents/librarian/agent.yaml @@ -1,4 +1,5 @@ name: librarian +name_cn: 资料检索智能体 description: >- Specialized codebase understanding agent for multi-repository analysis, searching remote codebases, retrieving official documentation, and finding @@ -28,5 +29,5 @@ prompt_metadata: - "Why does [external dependency] behave this way?" - Find examples of [library] usage - Working with unfamiliar npm/pip/cargo packages - key_trigger: "External library/source mentioned -> fire `librarian` background" + key_trigger: "External library/source mentioned -> delegate to `librarian`" # prompt injected by prompt_builder.py (year-aware) diff --git a/flocks/agent/agents/multimodal_looker/agent.yaml b/flocks/agent/agents/multimodal_looker/agent.yaml index 783ca2b26..4f7b9fe58 100644 --- a/flocks/agent/agents/multimodal_looker/agent.yaml +++ b/flocks/agent/agents/multimodal_looker/agent.yaml @@ -1,4 +1,5 @@ name: multimodal-looker +name_cn: 多模态分析智能体 description: >- Analyze media files (PDFs, images, diagrams) that require interpretation beyond raw text. Extracts specific information or summaries from documents, diff --git a/flocks/agent/agents/oracle/agent.yaml b/flocks/agent/agents/oracle/agent.yaml index 7a883cff2..01010a42e 100644 --- a/flocks/agent/agents/oracle/agent.yaml +++ b/flocks/agent/agents/oracle/agent.yaml @@ -1,4 +1,5 @@ name: oracle +name_cn: 架构顾问智能体 description: >- Read-only consultation agent. Reasoning specialist for debugging hard problems and high-difficulty architecture design. diff --git a/flocks/agent/agents/prometheus/agent.yaml b/flocks/agent/agents/prometheus/agent.yaml index e170f2d19..df7ba4ff2 100644 --- a/flocks/agent/agents/prometheus/agent.yaml +++ b/flocks/agent/agents/prometheus/agent.yaml @@ -1,4 +1,5 @@ name: prometheus +name_cn: 任务规划智能体 description: >- Strategic planner. Clarifies scope through interview-style questions, researches the codebase, writes executable work plans under .flocks/plans/, and @@ -37,7 +38,7 @@ prompt_metadata: trigger: User wants to discuss approach, scope, or tradeoffs before execution use_when: - Non-trivial tasks with multiple steps or unclear scope - - User asks for a plan, roadmap, or /plan-style workflow + - User asks for a plan or roadmap - Before delegating hephaestus or rex-junior for implementation avoid_when: - Trivial single-file fixes with clear instructions diff --git a/flocks/agent/agents/rex/agent.yaml b/flocks/agent/agents/rex/agent.yaml index c10003b25..6030d896c 100644 --- a/flocks/agent/agents/rex/agent.yaml +++ b/flocks/agent/agents/rex/agent.yaml @@ -1,4 +1,5 @@ name: rex +name_cn: Rex 主智能体 description: >- Powerful AI orchestrator for security operations. Analyzes threats, plans strategically, and delegates to specialized agents. diff --git a/flocks/agent/agents/rex/prompt_builder.py b/flocks/agent/agents/rex/prompt_builder.py index 89e51578b..ffdcabc74 100644 --- a/flocks/agent/agents/rex/prompt_builder.py +++ b/flocks/agent/agents/rex/prompt_builder.py @@ -147,7 +147,8 @@ def build_dynamic_rex_prompt( - Match existing codebase patterns when editing. - Fix bugs minimally; do not refactor during a bugfix unless required. - Keep search bounded: stop when you have enough context, when results repeat, or when direct evidence already answers the question. -- Use parallel background delegation only when you will benefit from independent branches of work. +- For independent parallel branches whose results are needed this turn, emit multiple foreground `delegate_task` / `task` tool calls in the same assistant turn. The runtime executes those sibling tool calls concurrently and returns all tool results before you continue. +- Do not use `run_in_background=true`; background subagent execution is disabled. ## 5. Verify @@ -279,7 +280,7 @@ def _build_clarification_protocol() -> str: def _task_management_section(use_task_system: bool) -> str: title = "Task Management" if use_task_system else "Todo Management" unit = "tasks" if use_task_system else "todos" - create_action = "`TaskCreate`" if use_task_system else "`todowrite`" + create_action = "`TaskCreate`" if use_task_system else '`todo(action="write")`' progress_action = ( '`TaskUpdate(status="in_progress")`' if use_task_system @@ -445,7 +446,7 @@ def _build_im_send_section() -> str: | No such block | User is chatting via **Flocks Web UI** — this is NOT an IM session. You do NOT have a target session ID yet. | Proceed to Step 2 | #### Step 2 — Discover sessions (only if Step 1 found nothing) -Call `session_list(category="user", status="active")`. +Call `session_manage(action="list", category="user", status="active")`. Filter results to sessions whose `title` starts with `[Wecom]`, `[Feishu]`, or `[Dingtalk]`. If no IM sessions found → stop and tell the user: diff --git a/flocks/agent/agents/rex_junior/agent.yaml b/flocks/agent/agents/rex_junior/agent.yaml index c33afbe77..dba253a1b 100644 --- a/flocks/agent/agents/rex_junior/agent.yaml +++ b/flocks/agent/agents/rex_junior/agent.yaml @@ -1,27 +1,10 @@ name: rex-junior -description: "Focused task executor. Same discipline, no delegation." +name_cn: Rex 执行智能体 +description: "Focused task executor. A general-purpose agent for simple tasks." mode: subagent hidden: false tags: [system] color: "#20B2AA" -delegatable: false +delegatable: true # Explicit tool allowlist for the focused executor. -tools: - - read - - glob - - grep - - edit - - write - - bash - - skill_load - - delegate_task - - task - - todoread - - todowrite - - run_workflow - - run_workflow_node - - background_output - - background_cancel - - session_list - - channel_message - +tools: [] diff --git a/flocks/agent/agents/rex_junior/prompt_builder.py b/flocks/agent/agents/rex_junior/prompt_builder.py index 40d4d5b9b..56e847254 100644 --- a/flocks/agent/agents/rex_junior/prompt_builder.py +++ b/flocks/agent/agents/rex_junior/prompt_builder.py @@ -32,7 +32,6 @@ def _build_prompt(prompt_append: Optional[str] = None) -> str: BLOCKED ACTIONS (will fail if attempted): -- task tool: BLOCKED - delegate_task for implementation work: BLOCKED ALLOWED: delegate_task with `subagent_type="explore"` or `subagent_type="librarian"` for research only. @@ -41,7 +40,7 @@ def _build_prompt(prompt_append: Optional[str] = None) -> str: TODO OBSESSION (NON-NEGOTIABLE): -- 2+ steps -> todowrite FIRST, atomic breakdown +- 2+ steps -> `todo(action="write")` FIRST, atomic breakdown - Mark in_progress before starting (ONE at a time) - Mark completed IMMEDIATELY after each step - NEVER batch completions diff --git a/flocks/agent/agents/self_enhance/agent.yaml b/flocks/agent/agents/self_enhance/agent.yaml index 3f04cb1b7..35bb6fbd2 100644 --- a/flocks/agent/agents/self_enhance/agent.yaml +++ b/flocks/agent/agents/self_enhance/agent.yaml @@ -1,4 +1,5 @@ name: self-enhance +name_cn: 能力增强智能体 description: >- Capability acquisition agent. Delegate when Rex encounters a capability gap (missing library, no tool for a task). Researches solutions, installs packages @@ -22,8 +23,6 @@ tools: - websearch - webfetch - skill_load - - background_output - - background_cancel prompt_metadata: category: self-enhancement cost: medium diff --git a/flocks/agent/prompt_utils.py b/flocks/agent/prompt_utils.py index 53f93738c..9c7beb943 100644 --- a/flocks/agent/prompt_utils.py +++ b/flocks/agent/prompt_utils.py @@ -259,7 +259,7 @@ def build_category_skills_delegation_guide( "```\n\n" "**ANTI-PATTERN (will produce poor results):**\n" "```typescript\n" - 'delegate_task(category="...", load_skills=[], run_in_background=false, prompt="...") // Empty load_skills without justification\n' + 'delegate_task(category="...", load_skills=[], prompt="...") // Empty load_skills without justification\n' "```" ) diff --git a/flocks/agent/registry.py b/flocks/agent/registry.py index fade71dc4..6f179c4a0 100644 --- a/flocks/agent/registry.py +++ b/flocks/agent/registry.py @@ -160,6 +160,7 @@ def _storage_custom_agent_to_info(agent_data: Dict[str, Any]) -> Optional[AgentI return AgentInfo( name=name, + name_cn=agent_data.get("name_cn") or agent_data.get("nameCn"), description=agent_data.get("description"), description_cn=agent_data.get("description_cn") or agent_data.get("descriptionCn"), prompt=agent_data.get("prompt"), @@ -873,8 +874,6 @@ def _build_base_permissions(user_perms, cli_overrides): f"{Truncate.GLOB}": "allow", }, "question": "deny", - "plan_enter": "deny", - "plan_exit": "deny", "read": { "*": "allow", "*.env": "ask", diff --git a/flocks/browser/admin.py b/flocks/browser/admin.py index 31b96020f..f082a738a 100644 --- a/flocks/browser/admin.py +++ b/flocks/browser/admin.py @@ -453,7 +453,7 @@ def run_setup() -> int: def run_doctor() -> int: - """Read-only diagnostics. Exit 0 iff everything looks healthy.""" + """Read-only diagnostics. Exit 0 iff a browser target and daemon exist.""" import platform import sys @@ -471,6 +471,16 @@ def row(label: str, ok: bool, detail: str = "") -> None: mark = "ok " if ok else "FAIL" print(f" [{mark}] {label}{(' — ' + detail) if detail else ''}") + target_available = browser_running or bool(endpoint_name) + if connections: + next_action = "ready; use `flocks browser -c 'print(page_info())'`" + elif target_available and daemon: + next_action = "attach; run `flocks browser -c 'print(page_info())'` before setup" + elif target_available: + next_action = "setup; run `flocks browser --setup`" + else: + next_action = "start Chrome/Chromium/Edge or provide BU_CDP_URL/BU_CDP_WS, then run `flocks browser --setup`" + print(f"{BROWSER_LABEL} doctor") print(f" platform {platform.system()} {platform.release()}") print(f" python {sys.version.split()[0]}") @@ -497,4 +507,5 @@ def row(label: str, ok: bool, detail: str = "") -> None: print(f" {conn['name']} — active page: {title} — {url}") else: print(f" {conn['name']} — active page: (no real page)") - return 0 if ((browser_running or endpoint_name) and daemon) else 1 + print(f" next action {next_action}") + return 0 if (target_available and daemon) else 1 diff --git a/flocks/channel/builtin/dingtalk/channel.py b/flocks/channel/builtin/dingtalk/channel.py index 8a9274ebf..276112bc9 100644 --- a/flocks/channel/builtin/dingtalk/channel.py +++ b/flocks/channel/builtin/dingtalk/channel.py @@ -127,6 +127,134 @@ async def send_text(self, ctx: OutboundContext) -> DeliveryResult: retryable=retryable, ) + async def send_media(self, ctx: OutboundContext) -> DeliveryResult: + """Send a media message via the DingTalk app-robot OAPI. + + The OAPI ``msgKey=file`` path is the only one that supports + uploading a *local* file. ``msgKey=image`` requires a public + ``photoURL`` reachable by DingTalk's servers — we fall back to + that path only when the inbound ``media_url`` is already a + reachable https URL. + """ + from flocks.channel.builtin.dingtalk.client import DingTalkApiError + from flocks.channel.builtin.dingtalk.media import ( + prepare_dingtalk_media, + ) + from flocks.channel.builtin.dingtalk.config import ( + resolve_target_kind, + ) + + if not ctx.to or not strip_target_prefix(ctx.to): + return DeliveryResult( + channel_id="dingtalk", + message_id="", + success=False, + error="DingTalk send_media requires 'to' (user: or chat:)", + ) + if not ctx.media_url: + return await self.send_text(ctx) + + try: + send_config = resolve_account_config(self._config or {}, ctx.account_id) + prepared = await prepare_dingtalk_media( + config=send_config, + account_id=ctx.account_id, + media_url=ctx.media_url, + ) + + if prepared.media_type == "image" and ctx.media_url.startswith( + ("http://", "https://") + ): + # Public image URL — use the inline image msgKey. + msg_key = "image" + msg_param = {"photoURL": ctx.media_url} + else: + # Local upload (any type) or non-image — go through the + # file msgKey with the upload's downloadCode. + msg_key = "file" + msg_param = { + "downloadCode": prepared.download_code, + "fileName": prepared.filename, + } + + # DingTalk's file/image msgKey does not carry companion text. + # Send the attachment first, then send the caption as a + # follow-up markdown message — matches the wecom ordering. + target_kind = resolve_target_kind(ctx.to) + from flocks.channel.builtin.dingtalk.client import api_request_for_account + from flocks.channel.builtin.dingtalk.config import resolve_account_credentials + import json as _json + + _, _, robot_code = resolve_account_credentials( + send_config, ctx.account_id, + ) + bare = strip_target_prefix(ctx.to) + if target_kind == "group": + body = { + "robotCode": robot_code, + "openConversationId": bare, + "msgKey": msg_key, + "msgParam": _json.dumps(msg_param, ensure_ascii=False), + } + data = await api_request_for_account( + "POST", "/v1.0/robot/groupMessages/send", + config=send_config, account_id=ctx.account_id, json_body=body, + ) + else: + body = { + "robotCode": robot_code, + "userIds": [bare], + "msgKey": msg_key, + "msgParam": _json.dumps(msg_param, ensure_ascii=False), + } + data = await api_request_for_account( + "POST", "/v1.0/robot/oToMessages/batchSend", + config=send_config, account_id=ctx.account_id, json_body=body, + ) + + if ctx.text: + from flocks.channel.builtin.dingtalk.send import send_message_app + await send_message_app( + config=send_config, + to=ctx.to, + text=ctx.text, + account_id=ctx.account_id, + ) + + self.record_message() + return DeliveryResult( + channel_id="dingtalk", + message_id=str(data.get("processQueryKey", "")), + chat_id=bare, + ) + except DingTalkApiError as exc: + retryable = getattr(exc, "retryable", False) + log.warning("dingtalk.send_media.failed", { + "to": ctx.to, "error": str(exc), "retryable": retryable, + }) + return DeliveryResult( + channel_id="dingtalk", + message_id="", + success=False, + error=str(exc), + retryable=retryable, + ) + except Exception as exc: + retryable = False + msg = str(exc).lower() + if "rate limit" in msg or "timeout" in msg: + retryable = True + log.warning("dingtalk.send_media.failed", { + "to": ctx.to, "error": str(exc), "retryable": retryable, + }) + return DeliveryResult( + channel_id="dingtalk", + message_id="", + success=False, + error=str(exc), + retryable=retryable, + ) + @property def text_chunk_limit(self) -> int: return int((self._config or {}).get("textChunkLimit", 4000)) diff --git a/flocks/channel/builtin/dingtalk/inbound_media.py b/flocks/channel/builtin/dingtalk/inbound_media.py new file mode 100644 index 000000000..b3e17e75f --- /dev/null +++ b/flocks/channel/builtin/dingtalk/inbound_media.py @@ -0,0 +1,283 @@ +""" +DingTalk inbound media download helpers. + +DingTalk delivers media references as opaque ``download_code`` strings +that must be exchanged for a short-lived HTTPS URL via the OAPI before +they can be downloaded. This module handles that exchange + the actual +download, returning a local file URI the dispatcher can hand to the +session pipeline as a :class:`FilePart`. +""" + +from __future__ import annotations + +import datetime +import mimetypes +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from urllib.parse import unquote, urlparse + +import httpx + +from flocks.channel.base import InboundMessage +from flocks.channel.media_filename import sanitize_filename +from flocks.channel.builtin.dingtalk.client import ( + DingTalkApiError, + api_request_for_account, +) +from flocks.channel.builtin.dingtalk.config import resolve_account_credentials +from flocks.utils.log import Log + +log = Log.create(service="channel.dingtalk.media") + +_DEFAULT_MAX_INBOUND_MEDIA_BYTES = 20 * 1024 * 1024 # DingTalk caps inbound files at 20MB +_DOWNLOAD_PATH = "/v1.0/robot/messageFiles/download" + + +class DingTalkInboundMediaTooLarge(ValueError): + """DingTalk 入站媒体超过允许大小。""" + + +@dataclass +class DownloadedInboundMedia: + filename: str + mime: str + url: str + source: dict + + +def _sanitize_filename(name: str) -> str: + return sanitize_filename(name) + + +def _media_storage_dir(account_id: str) -> Path: + return ( + Path.home() + / ".flocks" + / "data" + / "channel_media" + / "dingtalk" + / account_id + / datetime.date.today().isoformat() + ) + + +def _guess_mime_from_ext(filename: str) -> Optional[str]: + _, ext = os.path.splitext(filename) + if ext: + return mimetypes.guess_type(filename)[0] + return None + + +def _is_download_code(value: str) -> bool: + """True if *value* looks like a raw ``download_code`` rather than a URL. + + DingTalk's ``download_code`` is a short opaque token — no scheme, no + path separators. Anything that parses as an absolute URL is treated + as ready-to-fetch. + """ + if not value: + return False + parsed = urlparse(value) + if parsed.scheme in ("http", "https", "file"): + return False + return bool(value.strip()) + + +async def _exchange_download_code( + *, + config: dict, + account_id: Optional[str], + download_code: str, +) -> tuple[str, Optional[str]]: + """Exchange *download_code* for a short-lived HTTPS URL. + + Returns ``(download_url, filename)``; ``filename`` is best-effort and + may be ``None`` when the OAPI response omits it. + """ + app_key, app_secret, robot_code = resolve_account_credentials(config, account_id) + if not app_key or not app_secret: + raise DingTalkApiError( + "DingTalk appKey/appSecret not configured" + + (f" for account '{account_id}'" if account_id else ""), + ) + body = { + "robotCode": robot_code, + "downloadCode": download_code, + } + data = await api_request_for_account( + "POST", _DOWNLOAD_PATH, + config=config, account_id=account_id, json_body=body, + ) + download_url = ( + data.get("downloadUrl") + or data.get("download_url") + or "" + ) + if not download_url: + raise DingTalkApiError( + "DingTalk media download code exchange returned no URL", + response=data, + ) + filename = data.get("fileName") or data.get("filename") or None + return str(download_url), (str(filename) if filename else None) + + +async def _download_remote_bytes_limited( + url: str, max_bytes: int, +) -> tuple[bytes, Optional[str]]: + """Stream *url* into bytes, aborting when the body exceeds *max_bytes*.""" + async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client: + async with client.stream("GET", url) as resp: + resp.raise_for_status() + headers = {k.lower(): v for k, v in resp.headers.items()} + content_length = headers.get("content-length") + if content_length and content_length.isdigit() and int(content_length) > max_bytes: + raise DingTalkInboundMediaTooLarge( + f"DingTalk inbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + cd = headers.get("content-disposition") or "" + cd_match = re.search(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', cd, re.I) + cd_filename = unquote(cd_match.group(1).strip()) if cd_match else None + + chunks: list[bytes] = [] + total = 0 + async for chunk in resp.aiter_bytes(8192): + total += len(chunk) + if total > max_bytes: + raise DingTalkInboundMediaTooLarge( + f"DingTalk inbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks.append(chunk) + return b"".join(chunks), cd_filename + + +def _guess_filename( + msg: InboundMessage, + media_ref: str, + cd_filename: Optional[str], +) -> str: + """Resolve a filename from message raw body / header / URL fallback.""" + raw = msg.raw if isinstance(msg.raw, dict) else {} + content = _extract_content_dict(msg.raw) + for key in ("fileName", "filename", "name"): + candidate = str(content.get(key) or "").strip() + if candidate: + return _sanitize_filename(candidate) + for key in ("fileName", "filename", "name"): + candidate = str(raw.get(key) or "").strip() + if candidate: + return _sanitize_filename(candidate) + if raw.get("msgtype") == "richText" and isinstance(raw.get("rich_text_list"), list): + for item in raw["rich_text_list"]: + if isinstance(item, dict): + for key in ("fileName", "filename", "name"): + candidate = str(item.get(key) or "").strip() + if candidate: + return _sanitize_filename(candidate) + if cd_filename: + return _sanitize_filename(cd_filename) + url_path = urlparse(media_ref).path if not _is_download_code(media_ref) else "" + url_basename = os.path.basename(url_path) + if url_basename and "." in url_basename: + return _sanitize_filename(url_basename) + msg_id = msg.message_id or "unknown" + return _sanitize_filename(f"dingtalk_{msg_id[:12]}") + + +def _extract_content_dict(raw: Any) -> dict[str, Any]: + if isinstance(raw, dict): + content = raw.get("content") + return content if isinstance(content, dict) else {} + + content = getattr(raw, "content", None) + if isinstance(content, dict): + return content + + extensions = getattr(raw, "extensions", None) + if isinstance(extensions, dict): + content = extensions.get("content") + if isinstance(content, dict): + return content + + file_content = getattr(raw, "file_content", None) + if isinstance(file_content, dict): + return file_content + if file_content is not None: + result: dict[str, Any] = {} + for key in ("fileName", "filename", "name"): + value = getattr(file_content, key, None) + if value: + result[key] = value + return result + + return {} + + +async def download_inbound_media( + msg: InboundMessage, + config: dict, + *, + max_bytes: int = _DEFAULT_MAX_INBOUND_MEDIA_BYTES, +) -> Optional[DownloadedInboundMedia]: + media_ref = msg.media_url or "" + if not media_ref: + return None + + try: + if _is_download_code(media_ref): + download_url, name_hint = await _exchange_download_code( + config=config, account_id=msg.account_id, + download_code=media_ref, + ) + else: + download_url = media_ref + name_hint = None + + buffer, cd_filename = await _download_remote_bytes_limited( + download_url, max_bytes, + ) + except DingTalkInboundMediaTooLarge as e: + log.warning("dingtalk.media.file_too_large", { + "message_id": msg.message_id, "error": str(e), + }) + return None + except DingTalkApiError as e: + log.warning("dingtalk.media.exchange_failed", { + "message_id": msg.message_id, "error": str(e), + }) + return None + except Exception as e: + log.warning("dingtalk.media.download_failed", { + "message_id": msg.message_id, "error": str(e), + }) + return None + + filename = _guess_filename(msg, media_ref, cd_filename or name_hint) + if "." not in filename: + guessed_mime = _guess_mime_from_ext(filename) + ext = mimetypes.guess_extension(guessed_mime) if guessed_mime else "" + if ext: + filename = f"{filename}{ext}" + mime = _guess_mime_from_ext(filename) or "application/octet-stream" + + storage_dir = _media_storage_dir(msg.account_id or "default") + storage_dir.mkdir(parents=True, exist_ok=True) + msg_id = msg.message_id or "unknown" + file_path = storage_dir / _sanitize_filename(f"{msg_id}_{filename}") + file_path.write_bytes(buffer) + + return DownloadedInboundMedia( + filename=filename, + mime=mime, + url=file_path.resolve().as_uri(), + source={ + "channel": "dingtalk", + "account_id": msg.account_id, + "message_id": msg.message_id, + "media_url": msg.media_url, + "download_code": media_ref if _is_download_code(media_ref) else None, + }, + ) diff --git a/flocks/channel/builtin/dingtalk/media.py b/flocks/channel/builtin/dingtalk/media.py new file mode 100644 index 000000000..f831f9e14 --- /dev/null +++ b/flocks/channel/builtin/dingtalk/media.py @@ -0,0 +1,203 @@ +""" +DingTalk outbound media helpers. + +Uploads a local or remote media file via the DingTalk OAPI robot +``/v1.0/robot/messageFiles/upload`` endpoint and returns a +``PreparedDingTalkMedia`` with the ``media_id`` + ``downloadCode`` +the channel's :meth:`send_media` needs to construct a ``file`` / +``image`` message. +""" + +from __future__ import annotations + +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Literal +from urllib.parse import unquote, urlparse + +import httpx + +from flocks.channel.media_filename import sanitize_filename +from flocks.channel.builtin.dingtalk.client import ( + DingTalkApiError, + api_request_for_account, +) +from flocks.channel.builtin.dingtalk.config import resolve_account_credentials +from flocks.utils.log import Log + +log = Log.create(service="channel.dingtalk.send_media") + +_DEFAULT_MAX_MEDIA_BYTES = 20 * 1024 * 1024 +UPLOAD_PATH = "/v1.0/robot/messageFiles/upload" +DingTalkOutboundMediaType = Literal["image", "file", "voice", "video"] + + +@dataclass +class PreparedDingTalkMedia: + data: bytes + filename: str + mime: str + media_type: DingTalkOutboundMediaType + media_id: str + download_code: str + + +def _sanitize_filename(name: str) -> str: + return sanitize_filename(name) + + +def _media_type_from_filename(filename: str) -> DingTalkOutboundMediaType: + mime = mimetypes.guess_type(filename)[0] or "" + if mime.startswith("image/"): + return "image" + if mime.startswith("audio/"): + return "voice" + if mime.startswith("video/"): + return "video" + return "file" + + +def _path_from_media_url(media_url: str) -> Path | None: + parsed = urlparse(media_url) + if parsed.scheme == "file": + return Path(unquote(parsed.path)) + if parsed.scheme: + return None + return Path(media_url) + + +def _read_local_file_limited(path: Path, max_bytes: int) -> bytes: + if not path.is_file(): + raise FileNotFoundError(f"DingTalk media file not found: {path}") + size = path.stat().st_size + if size > max_bytes: + raise ValueError( + f"DingTalk outbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + return path.read_bytes() + + +async def _fetch_http_file_limited( + url: str, max_bytes: int, +) -> tuple[bytes, str]: + async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client: + async with client.stream("GET", url) as resp: + resp.raise_for_status() + content_length = resp.headers.get("content-length") + if content_length and content_length.isdigit() and int(content_length) > max_bytes: + raise ValueError( + f"DingTalk outbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks: list[bytes] = [] + total = 0 + async for chunk in resp.aiter_bytes(8192): + total += len(chunk) + if total > max_bytes: + raise ValueError( + f"DingTalk outbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks.append(chunk) + basename = os.path.basename(urlparse(url).path) or "attachment" + return b"".join(chunks), _sanitize_filename(unquote(basename)) + + +async def _read_payload( + media_url: str, max_bytes: int, +) -> tuple[bytes, str]: + parsed = urlparse(media_url) + if parsed.scheme in ("http", "https"): + return await _fetch_http_file_limited(media_url, max_bytes) + path = _path_from_media_url(media_url) + if path is None: + raise ValueError(f"Unsupported DingTalk media URL scheme: {parsed.scheme}") + data = _read_local_file_limited(path, max_bytes) + return data, _sanitize_filename(path.name) + + +async def upload_dingtalk_media( + *, + config: dict, + account_id: str | None, + data: bytes, + filename: str, +) -> tuple[str, str]: + """Upload *data* via the OAPI and return ``(media_id, download_code)``.""" + app_key, app_secret, robot_code = resolve_account_credentials(config, account_id) + if not app_key or not app_secret: + raise DingTalkApiError( + "DingTalk appKey/appSecret not configured" + + (f" for account '{account_id}'" if account_id else ""), + ) + mime = mimetypes.guess_type(filename)[0] or "application/octet-stream" + files = {"media": (filename, data, mime)} + form: dict[str, str] = {"robotCode": robot_code} + # The OAPI endpoint accepts a multipart/form-data upload. We invoke it + # via the raw client so we can pass through the file payload — the + # generic ``api_request`` helper is JSON-only. + from flocks.channel.builtin.dingtalk.client import ( + _get_http_client, + get_access_token, + ) + token = await get_access_token(app_key, app_secret) + client = await _get_http_client() + resp = await client.post( + f"https://api.dingtalk.com{UPLOAD_PATH}", + headers={"x-acs-dingtalk-access-token": token}, + data=form, + files=files, + timeout=60.0, + ) + if resp.status_code >= 400: + try: + err = resp.json() + except Exception: + err = {} + raise DingTalkApiError( + f"DingTalk media upload failed: HTTP {resp.status_code}", + http_status=resp.status_code, + response=err, + ) + data_obj = resp.json() if resp.content else {} + media_id = ( + data_obj.get("mediaId") + or data_obj.get("media_id") + or "" + ) + download_code = ( + data_obj.get("downloadCode") + or data_obj.get("download_code") + or "" + ) + if not media_id or not download_code: + raise DingTalkApiError( + "DingTalk media upload returned no mediaId/downloadCode", + response=data_obj, + ) + return str(media_id), str(download_code) + + +async def prepare_dingtalk_media( + *, + config: dict, + account_id: str | None, + media_url: str, + max_bytes: int = _DEFAULT_MAX_MEDIA_BYTES, +) -> PreparedDingTalkMedia: + """Read *media_url*, upload it, and return the metadata needed for send.""" + data, filename = await _read_payload(media_url, max_bytes) + mime = mimetypes.guess_type(filename)[0] or "application/octet-stream" + media_type = _media_type_from_filename(filename) + media_id, download_code = await upload_dingtalk_media( + config=config, account_id=account_id, + data=data, filename=filename, + ) + return PreparedDingTalkMedia( + data=data, + filename=filename, + mime=mime, + media_type=media_type, + media_id=media_id, + download_code=download_code, + ) diff --git a/flocks/channel/builtin/dingtalk/stream.py b/flocks/channel/builtin/dingtalk/stream.py index bcce9859a..75581860b 100644 --- a/flocks/channel/builtin/dingtalk/stream.py +++ b/flocks/channel/builtin/dingtalk/stream.py @@ -336,6 +336,17 @@ def _extract_media_url(message: Any) -> Optional[str]: if code: return str(code) + content = _extract_content_dict(message) + if content: + code = ( + content.get("downloadCode") + or content.get("download_code") + or content.get("pictureDownloadCode") + or content.get("picture_download_code") + ) + if code: + return str(code) + rich_text = getattr(message, "rich_text_content", None) or getattr( message, "rich_text", None ) @@ -354,6 +365,42 @@ def _extract_media_url(message: Any) -> Optional[str]: return None +def _extract_content_dict(message: Any) -> dict[str, Any]: + if isinstance(message, dict): + content = message.get("content") + return content if isinstance(content, dict) else {} + + content = getattr(message, "content", None) + if isinstance(content, dict): + return content + + extensions = getattr(message, "extensions", None) + if isinstance(extensions, dict): + content = extensions.get("content") + if isinstance(content, dict): + return content + + file_content = getattr(message, "file_content", None) + if isinstance(file_content, dict): + return file_content + if file_content is not None: + result: dict[str, Any] = {} + for key in ( + "downloadCode", + "download_code", + "fileName", + "filename", + "name", + "fileId", + ): + value = getattr(file_content, key, None) + if value: + result[key] = value + return result + + return {} + + # --------------------------------------------------------------------------- # Gating decisions per inbound message # --------------------------------------------------------------------------- diff --git a/flocks/channel/builtin/telegram/channel.py b/flocks/channel/builtin/telegram/channel.py index 2b5142344..21062bace 100644 --- a/flocks/channel/builtin/telegram/channel.py +++ b/flocks/channel/builtin/telegram/channel.py @@ -53,6 +53,18 @@ log = Log.create(service="channel.telegram") +# Mapping from :class:`PreparedTelegramMedia.kind` to Bot API endpoint +# + multipart field name. Animation covers GIFs that should NOT be sent +# via the photo endpoint. +_TELEGRAM_KIND_TO_ENDPOINT: dict[str, tuple[str, str]] = { + "photo": ("sendPhoto", "photo"), + "document": ("sendDocument", "document"), + "video": ("sendVideo", "video"), + "audio": ("sendAudio", "audio"), + "voice": ("sendVoice", "voice"), + "animation": ("sendAnimation", "animation"), +} + class TelegramChannel(ChannelPlugin): """Telegram bot channel — polling (default) or webhook mode.""" @@ -231,6 +243,130 @@ async def handle_webhook( # Outbound # ------------------------------------------------------------------ + async def send_media(self, ctx: OutboundContext) -> DeliveryResult: + """Send a media message via the Telegram Bot HTTP API. + + Routes the call to ``sendPhoto`` / ``sendDocument`` / + ``sendVideo`` / ``sendAudio`` / ``sendVoice`` / ``sendAnimation`` + based on the inferred kind of *ctx.media_url* (or ``ctx.text`` + carrying a ``KIND:...`` prefix when overridden by the agent). + Local files are read directly; ``http(s)`` URLs are passed + through to the Bot API (Telegram fetches them itself for + sub-5MB photos / sub-20MB files). + """ + try: + account_id, account = resolve_account_config(self._config, ctx.account_id) + except ValueError as exc: + return DeliveryResult( + channel_id="telegram", message_id="", success=False, error=str(exc), + ) + + token = coerce_str(account.get("botToken")) + if not token: + return DeliveryResult( + channel_id="telegram", message_id="", success=False, error="Missing botToken", + ) + + chat_id, target_thread_id = parse_target(ctx.to) + if not chat_id: + return DeliveryResult( + channel_id="telegram", message_id="", success=False, + error="Invalid Telegram target", + ) + + if not ctx.media_url: + return await self.send_text(ctx) + + message_thread_id = coerce_int(ctx.thread_id) or target_thread_id + reply_to_message_id = coerce_int(ctx.reply_to_id) + base_url = resolve_api_base(account, token) + timeout_seconds = max(coerce_int(account.get("timeoutSeconds")) or 60, 1) + client = await get_http_client() + + try: + from flocks.channel.builtin.telegram.media import prepare_telegram_media + + kind_override = None + media_source = ctx.media_url + # Optional agent-side override: ``telegram:document:`` + # forces the document endpoint (e.g. for images that fail + # photo dimension checks). + if media_source.startswith("telegram:"): + head, _, tail = media_source.partition(":") + # head is e.g. ``telegram:document`` then a ``:``-separated URL + kind_part = media_source.split(":", 2) + if len(kind_part) == 3 and kind_part[0] == "telegram": + candidate = kind_part[1].strip().lower() + if candidate in {"photo", "document", "video", "audio", "voice", "animation"}: + kind_override = candidate + media_source = kind_part[2] + + prepared = await prepare_telegram_media( + media_source, kind_override=kind_override, + ) + + endpoint, param_name = _TELEGRAM_KIND_TO_ENDPOINT[prepared.kind] + fields: dict[str, Any] = { + "chat_id": chat_id, + } + if ctx.text: + fields["caption"] = (ctx.text or "")[:1024] + if message_thread_id is not None: + fields["message_thread_id"] = message_thread_id + if reply_to_message_id is not None: + fields["reply_to_message_id"] = reply_to_message_id + if ctx.silent: + fields["disable_notification"] = True + files = { + param_name: (prepared.filename, prepared.data, prepared.mime), + } + + response = await client.post( + f"{base_url}/{endpoint}", + data=fields, + files=files, + timeout=timeout_seconds, + ) + except FileNotFoundError as exc: + return DeliveryResult( + channel_id="telegram", message_id="", success=False, + error=str(exc), + ) + except Exception as exc: + return DeliveryResult( + channel_id="telegram", message_id="", success=False, + error=f"Telegram send_media failed: {exc}", + retryable=is_retryable(0, exc if isinstance(exc, httpx.HTTPError) else None), + ) + + try: + data = response.json() + except ValueError: + data = {} + + if response.status_code >= 400 or not data.get("ok", response.is_success): + description = ( + data.get("description") or data.get("error") + or f"HTTP {response.status_code}" + ) + return DeliveryResult( + channel_id="telegram", message_id="", success=False, + error=f"Telegram send_media failed: {description}", + retryable=is_retryable(response.status_code), + ) + + result = data.get("result") or {} + message_id = coerce_str(result.get("message_id")) + returned_chat_id = coerce_str((result.get("chat") or {}).get("id")) or chat_id + self.record_message() + return DeliveryResult( + channel_id="telegram", + message_id=message_id, + chat_id=returned_chat_id, + success=True, + ) + + async def send_text(self, ctx: OutboundContext) -> DeliveryResult: try: account_id, account = resolve_account_config(self._config, ctx.account_id) diff --git a/flocks/channel/builtin/telegram/inbound.py b/flocks/channel/builtin/telegram/inbound.py index c43df6607..9f1cefbcb 100644 --- a/flocks/channel/builtin/telegram/inbound.py +++ b/flocks/channel/builtin/telegram/inbound.py @@ -216,8 +216,14 @@ async def build_inbound_message( # Check for media first so we always surface the media type to the AI, # even when the message also carries a text caption. media_desc = extract_media_description(message) + media_url: Optional[str] = None if media_desc: text = media_desc + # Surface the Telegram file_id as an opaque URI so the dispatcher + # can route it to the per-channel downloader (Telegram getFile). + file_id, media_kind = _extract_primary_file_id(message) + if file_id: + media_url = f"telegram://{media_kind}/{file_id}" else: text = extract_text(message) if not text: @@ -268,9 +274,40 @@ async def build_inbound_message( chat_id=chat_id, chat_type=chat_type, text=text.strip(), + media_url=media_url, reply_to_id=reply_to_id, thread_id=thread_id, mentioned=mentioned, mention_text=mention_text, raw=message, ) + + +def _extract_primary_file_id( + message: dict[str, Any], +) -> tuple[Optional[str], str]: + """Return the first Telegram ``file_id`` in *message* + the media kind. + + Media kinds follow the Telegram Bot API parameter names so the + outbound ``send_*`` methods can use them directly: ``photo``, + ``document``, ``video``, ``audio``, ``voice``, ``animation``. + Photos are special — Telegram exposes them as a list of size variants; + we pick the largest one. + """ + photos = message.get("photo") + if isinstance(photos, list) and photos: + largest = max( + (p for p in photos if isinstance(p, dict)), + key=lambda p: coerce_int(p.get("file_size")) or 0, + default=None, + ) + if largest is not None: + return coerce_str(largest.get("file_id")), "photo" + + for kind in ("document", "video", "audio", "voice", "animation"): + block = message.get(kind) + if isinstance(block, dict): + file_id = coerce_str(block.get("file_id")) + if file_id: + return file_id, kind + return None, "document" diff --git a/flocks/channel/builtin/telegram/inbound_media.py b/flocks/channel/builtin/telegram/inbound_media.py new file mode 100644 index 000000000..173d339d4 --- /dev/null +++ b/flocks/channel/builtin/telegram/inbound_media.py @@ -0,0 +1,324 @@ +""" +Telegram inbound media download helpers. + +The Telegram Bot API gives the bot a stable ``file_id`` for every file +the bot can see; the actual bytes are fetched in two steps: + +1. ``GET /getFile?file_id=`` → ``{"file_path": "documents/file_0.pdf"}`` +2. ``GET https://api.telegram.org/file/bot/`` + +This module wraps those two calls and returns a local file URI the +dispatcher can hand to the session pipeline as a +:class:`flocks.session.message.FilePart`. +""" + +from __future__ import annotations + +import datetime +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from urllib.parse import unquote, urlparse + +import httpx + +from flocks.channel.base import InboundMessage +from flocks.channel.media_filename import sanitize_filename +from flocks.utils.log import Log +from .config import resolve_account_config, resolve_api_base + +log = Log.create(service="channel.telegram.media") + +_DEFAULT_MAX_INBOUND_MEDIA_BYTES = 20 * 1024 * 1024 + + +class TelegramInboundMediaTooLarge(ValueError): + """Telegram 入站媒体超过允许大小。""" + + +@dataclass +class DownloadedInboundMedia: + filename: str + mime: str + url: str + source: dict + + +def _sanitize_filename(name: str) -> str: + return sanitize_filename(name) + + +def _media_storage_dir(account_id: str) -> Path: + return ( + Path.home() + / ".flocks" + / "data" + / "channel_media" + / "telegram" + / account_id + / datetime.date.today().isoformat() + ) + + +def _guess_mime_from_ext(filename: str) -> Optional[str]: + _, ext = os.path.splitext(filename) + if ext: + return mimetypes.guess_type(filename)[0] + return None + + +def _parse_telegram_uri(uri: str) -> tuple[Optional[str], Optional[str]]: + """Parse ``telegram:///`` into ``(kind, file_id)``.""" + if not uri.startswith("telegram://"): + return None, None + rest = uri[len("telegram://"):] + if "/" not in rest: + return None, None + kind, _, file_id = rest.partition("/") + return kind or None, file_id or None + + +def _resolve_credentials( + config: dict, account_id: Optional[str], +) -> Optional[str]: + """Look up the bot token for the given account (helper for tests).""" + if not isinstance(config, dict): + return None + if account_id and account_id in (config.get("accounts") or {}): + return ( + config["accounts"][account_id].get("botToken") + or config.get("botToken") + ) + return config.get("botToken") or config.get("BOT_TOKEN") + + +async def _get_file_path( + *, + bot_token: str, + api_base: str, + file_id: str, + timeout: float, +) -> tuple[str, str]: + """Call ``/getFile`` and return ``(file_path, file_id)``.""" + url = f"{api_base.rstrip('/')}/getFile" + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get( + url, params={"file_id": file_id}, + ) + data = resp.json() if resp.content else {} + if not resp.is_success or not data.get("ok"): + raise RuntimeError( + f"Telegram getFile failed: {data.get('description') or resp.text}" + ) + result = data.get("result") or {} + file_path = coerce_str(result.get("file_path")) + if not file_path: + raise RuntimeError("Telegram getFile returned no file_path") + return file_path, file_id + + +def coerce_str(value: Any) -> Optional[str]: + if value is None: + return None + s = str(value).strip() + return s or None + + +async def _download_file( + *, + download_base: str, + file_path: str, + max_bytes: int, + timeout: float, +) -> bytes: + url = f"{download_base.rstrip('/')}/{file_path.lstrip('/')}" + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + async with client.stream("GET", url) as resp: + resp.raise_for_status() + content_length = resp.headers.get("content-length") + if content_length and content_length.isdigit() and int(content_length) > max_bytes: + raise TelegramInboundMediaTooLarge( + f"Telegram inbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks: list[bytes] = [] + total = 0 + async for chunk in resp.aiter_bytes(8192): + total += len(chunk) + if total > max_bytes: + raise TelegramInboundMediaTooLarge( + f"Telegram inbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks.append(chunk) + return b"".join(chunks) + + +def _guess_filename( + msg: InboundMessage, + kind: str, + file_id: str, + file_path: str, +) -> str: + raw = msg.raw if isinstance(msg.raw, dict) else {} + + # Document blocks carry a `file_name`. Photos have no name. + if kind in {"document", "audio", "video", "animation", "voice"}: + block = raw.get(kind) + if isinstance(block, dict): + for key in ("file_name", "fileName"): + candidate = str(block.get(key) or "").strip() + if candidate: + return _sanitize_filename(candidate) + for key in ("file_name", "fileName"): + candidate = str(raw.get(key) or "").strip() + if candidate: + return _sanitize_filename(candidate) + # file_path looks like ``documents/file_42.pdf`` — use the basename. + url_basename = os.path.basename(file_path) + if url_basename and "." in url_basename: + return _sanitize_filename(unquote(url_basename)) + # photo kinds: synthesise a stable name from message_id + file_id hash. + if kind == "photo": + suffix = file_id[-6:] if file_id else "x" + return _sanitize_filename(f"photo_{msg.message_id}_{suffix}.jpg") + return _sanitize_filename(f"telegram_{kind}_{file_id[:12]}") + + +def _legacy_api_root( + config: dict[str, Any], + account_id: Optional[str], + bot_token: str, +) -> Optional[str]: + if not isinstance(config, dict): + return None + if account_id and isinstance(config.get("accounts"), dict): + acc = config["accounts"].get(account_id) or {} + base = acc.get("apiBase") or acc.get("api_base") + if base: + return _normalize_legacy_api_root(str(base), bot_token) + base = config.get("apiBase") or config.get("api_base") + if base: + return _normalize_legacy_api_root(str(base), bot_token) + return None + + +def _normalize_legacy_api_root(base: str, bot_token: str) -> str: + normalized = base.rstrip("/") + if "{token}" in normalized: + normalized = normalized.replace("{token}", "").rstrip("/") + if normalized.endswith(bot_token): + normalized = normalized[: -len(bot_token)].rstrip("/") + if normalized.endswith("/bot"): + return normalized[: -len("/bot")] + return normalized + + +def _resolve_api_roots( + config: dict[str, Any], + account_id: Optional[str], + bot_token: str, +) -> tuple[str, str]: + try: + _, account = resolve_account_config(config, account_id) + api_root = coerce_str(account.get("apiRoot")) + if api_root: + api_call_base = resolve_api_base(account, bot_token) + api_root = api_call_base.rsplit("/bot", 1)[0] + else: + legacy_root = _legacy_api_root(config, account_id, bot_token) + api_root = legacy_root or "https://api.telegram.org" + api_call_base = f"{api_root.rstrip('/')}/bot{bot_token}" + except ValueError: + legacy_root = _legacy_api_root(config, account_id, bot_token) + api_root = legacy_root or "https://api.telegram.org" + api_call_base = f"{api_root.rstrip('/')}/bot{bot_token}" + download_base = f"{api_root.rstrip('/')}/file/bot{bot_token}" + return api_call_base, download_base + + +async def download_inbound_media( + msg: InboundMessage, + config: dict, + *, + max_bytes: int = _DEFAULT_MAX_INBOUND_MEDIA_BYTES, +) -> Optional[DownloadedInboundMedia]: + media_ref = msg.media_url or "" + if not media_ref: + return None + + kind, file_id = _parse_telegram_uri(media_ref) + if not file_id: + log.warning("telegram.media.invalid_uri", {"media_url": media_ref[:200]}) + return None + + bot_token = _resolve_credentials(config, msg.account_id) + if not bot_token: + log.warning("telegram.media.no_token", { + "channel_id": msg.channel_id, + "account_id": msg.account_id, + }) + return None + + api_call_base, download_base = _resolve_api_roots( + config, msg.account_id, bot_token, + ) + + try: + file_path, _ = await _get_file_path( + bot_token=bot_token, api_base=api_call_base, + file_id=file_id, timeout=30.0, + ) + buffer = await _download_file( + download_base=download_base, file_path=file_path, + max_bytes=max_bytes, timeout=60.0, + ) + except TelegramInboundMediaTooLarge as e: + log.warning("telegram.media.file_too_large", { + "message_id": msg.message_id, "error": str(e), + }) + return None + except Exception as e: + log.warning("telegram.media.download_failed", { + "message_id": msg.message_id, "error": str(e), + }) + return None + + filename = _guess_filename(msg, kind or "document", file_id, file_path) + if "." not in filename: + guessed_mime = _guess_mime_from_ext(filename) + ext = mimetypes.guess_extension(guessed_mime) if guessed_mime else "" + if ext: + filename = f"{filename}{ext}" + mime = _guess_mime_from_ext(filename) or _guess_mime_for_kind(kind or "document") + + storage_dir = _media_storage_dir(msg.account_id or "default") + storage_dir.mkdir(parents=True, exist_ok=True) + msg_id = msg.message_id or "unknown" + file_path_local = storage_dir / _sanitize_filename(f"{msg_id}_{filename}") + file_path_local.write_bytes(buffer) + + return DownloadedInboundMedia( + filename=filename, + mime=mime, + url=file_path_local.resolve().as_uri(), + source={ + "channel": "telegram", + "account_id": msg.account_id, + "message_id": msg.message_id, + "media_url": msg.media_url, + "file_id": file_id, + "kind": kind, + }, + ) + + +def _guess_mime_for_kind(kind: str) -> str: + return { + "photo": "image/jpeg", + "voice": "audio/ogg", + "audio": "audio/mpeg", + "video": "video/mp4", + "animation": "video/mp4", + "document": "application/octet-stream", + }.get(kind, "application/octet-stream") diff --git a/flocks/channel/builtin/telegram/media.py b/flocks/channel/builtin/telegram/media.py new file mode 100644 index 000000000..516dc719d --- /dev/null +++ b/flocks/channel/builtin/telegram/media.py @@ -0,0 +1,132 @@ +""" +Telegram outbound media helpers. + +Prepares a local or remote media file for delivery through the Bot API +and exposes a small ``PreparedTelegramMedia`` dataclass carrying the +bytes, filename, and inferred ``kind`` (photo / document / video / etc.) +so :meth:`TelegramChannel.send_media` can pick the right endpoint. +""" + +from __future__ import annotations + +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Literal +from urllib.parse import unquote, urlparse + +import httpx + +from flocks.channel.media_filename import sanitize_filename + +DingTalkOutboundMediaType = Literal[ # type: ignore[misc] + "photo", "document", "video", "audio", "voice", "animation", +] +TelegramOutboundMediaType = Literal[ + "photo", "document", "video", "audio", "voice", "animation", +] + +_DEFAULT_MAX_MEDIA_BYTES = 50 * 1024 * 1024 # 50MB — Telegram Bot API cap + + +@dataclass +class PreparedTelegramMedia: + data: bytes + filename: str + mime: str + kind: TelegramOutboundMediaType + + +def _sanitize_filename(name: str) -> str: + return sanitize_filename(name) + + +def _media_kind_from_filename(filename: str) -> TelegramOutboundMediaType: + mime = mimetypes.guess_type(filename)[0] or "" + if mime.startswith("image/"): + # Telegram distinguishes photo (jpeg-only, no animation) from + # animation (GIF / H.264). Re-use the file name + mime to pick. + if mime in {"image/gif"} or filename.lower().endswith(".gif"): + return "animation" + return "photo" + if mime.startswith("video/"): + return "video" + if mime == "audio/ogg" or filename.lower().endswith(".ogg"): + return "voice" + if mime.startswith("audio/"): + return "audio" + return "document" + + +def _path_from_media_url(media_url: str) -> Path | None: + parsed = urlparse(media_url) + if parsed.scheme == "file": + return Path(unquote(parsed.path)) + if parsed.scheme: + return None + return Path(media_url) + + +def _read_local_file_limited(path: Path, max_bytes: int) -> bytes: + if not path.is_file(): + raise FileNotFoundError(f"Telegram media file not found: {path}") + size = path.stat().st_size + if size > max_bytes: + raise ValueError( + f"Telegram outbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + return path.read_bytes() + + +async def _fetch_http_file_limited( + url: str, max_bytes: int, +) -> tuple[bytes, str]: + async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client: + async with client.stream("GET", url) as resp: + resp.raise_for_status() + content_length = resp.headers.get("content-length") + if content_length and content_length.isdigit() and int(content_length) > max_bytes: + raise ValueError( + f"Telegram outbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks: list[bytes] = [] + total = 0 + async for chunk in resp.aiter_bytes(8192): + total += len(chunk) + if total > max_bytes: + raise ValueError( + f"Telegram outbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + chunks.append(chunk) + basename = os.path.basename(urlparse(url).path) or "attachment" + return b"".join(chunks), _sanitize_filename(unquote(basename)) + + +async def prepare_telegram_media( + media_url: str, + *, + kind_override: TelegramOutboundMediaType | None = None, + max_bytes: int = _DEFAULT_MAX_MEDIA_BYTES, +) -> PreparedTelegramMedia: + """Read *media_url* (local or remote) and infer its Telegram kind. + + *kind_override* lets the caller force ``document`` (e.g. for an image + that should bypass the photo dimension limits and use the generic + file-upload path). + """ + parsed = urlparse(media_url) + if parsed.scheme in ("http", "https"): + data, filename = await _fetch_http_file_limited(media_url, max_bytes) + else: + path = _path_from_media_url(media_url) + if path is None: + raise ValueError(f"Unsupported Telegram media URL scheme: {parsed.scheme}") + data = _read_local_file_limited(path, max_bytes) + filename = _sanitize_filename(path.name) + + mime = mimetypes.guess_type(filename)[0] or "application/octet-stream" + kind = kind_override or _media_kind_from_filename(filename) + return PreparedTelegramMedia( + data=data, filename=filename, mime=mime, kind=kind, + ) diff --git a/flocks/channel/builtin/wecom/channel.py b/flocks/channel/builtin/wecom/channel.py index 0e66b5c88..19cd99a3e 100644 --- a/flocks/channel/builtin/wecom/channel.py +++ b/flocks/channel/builtin/wecom/channel.py @@ -214,6 +214,77 @@ async def send_text(self, ctx: OutboundContext) -> DeliveryResult: success=False, error=str(e), retryable=retryable, ) + async def send_media(self, ctx: OutboundContext) -> DeliveryResult: + if not self._ws_client: + return DeliveryResult( + channel_id="wecom", message_id="", + success=False, error="WebSocket not connected", + ) + if not ctx.media_url: + return await self.send_text(ctx) + + try: + from flocks.channel.builtin.wecom.media import prepare_wecom_media + + media = await prepare_wecom_media(ctx.media_url) + upload = await self._ws_client.upload_media( + media.data, + type=media.media_type, + filename=media.filename, + ) + media_id = upload.get("media_id", "") + if not media_id: + raise RuntimeError(f"WeCom media upload failed: {upload}") + + frame = ( + self._frame_cache.pop(ctx.reply_to_id, None) + if ctx.reply_to_id else None + ) + if frame: + sent = await self._ws_client.reply_media( + frame, + media.media_type, + media_id, + video_title=media.filename if media.media_type == "video" else None, + ) + else: + sent = await self._ws_client.send_media_message( + ctx.to, + media.media_type, + media_id, + video_title=media.filename if media.media_type == "video" else None, + ) + + # WeCom's media payload does not carry companion text. Keep the + # attachment first, then send text as a separate message. + message_id = _extract_sent_message_id(sent) + if ctx.text: + text_result = await self.send_text( + OutboundContext(**{**vars(ctx), "media_url": None}) + ) + if not text_result.success: + return DeliveryResult( + channel_id="wecom", + message_id=message_id, + chat_id=ctx.to, + success=False, + error=f"WeCom media sent but caption failed: {text_result.error}", + retryable=text_result.retryable, + ) + + self.record_message() + return DeliveryResult( + channel_id="wecom", + message_id=message_id, + chat_id=ctx.to, + ) + except Exception as e: + retryable = "timeout" in str(e).lower() or "rate limit" in str(e).lower() + return DeliveryResult( + channel_id="wecom", message_id="", + success=False, error=str(e), retryable=retryable, + ) + def format_message(self, text: str, format_hint: str = "markdown") -> str: return text @@ -428,6 +499,15 @@ def _parse_frame(frame: dict, config: dict) -> Optional[InboundMessage]: ) +def _extract_sent_message_id(frame: Any) -> str: + if not isinstance(frame, dict): + return "" + body = frame.get("body") or {} + if not isinstance(body, dict): + return "" + return str(body.get("msgid") or body.get("message_id") or "") + + def _extract_content(body: dict) -> tuple[str, Optional[str]]: """Extract ``(text, media_url)`` from the frame body.""" msg_type = body.get("msgtype", "") @@ -443,22 +523,38 @@ def _extract_content(body: dict) -> tuple[str, Optional[str]]: return body.get("voice", {}).get("content", "[语音消息]"), None if msg_type == "file": - url = body.get("file", {}).get("url", "") + file_block = body.get("file", {}) + url = file_block.get("url", "") + filename = str(file_block.get("filename", "") or "").strip() + if filename: + return f"[文件消息: {filename}]", url or None return "[文件消息]", url or None if msg_type == "mixed": - return _extract_mixed(body.get("mixed", {})), None + text, media_url = _extract_mixed(body.get("mixed", {})) + return text, media_url return "", None -def _extract_mixed(mixed: dict) -> str: - """Flatten a mixed (图文混排) message into text.""" +def _extract_mixed(mixed: dict) -> tuple[str, Optional[str]]: + """Flatten a mixed (图文混排) message into text + first media URL.""" parts: list[str] = [] + first_media_url: Optional[str] = None for item in mixed.get("msg_item", []): item_type = item.get("msgtype", "") if item_type == "text": parts.append(item.get("text", {}).get("content", "")) elif item_type == "image": parts.append("[图片]") - return " ".join(parts).strip() + url = item.get("image", {}).get("url", "") + if not first_media_url and url: + first_media_url = url + elif item_type == "file": + file_block = item.get("file", {}) + filename = str(file_block.get("filename", "") or "").strip() + parts.append(f"[文件: {filename}]" if filename else "[文件]") + url = file_block.get("url", "") + if not first_media_url and url: + first_media_url = url + return " ".join(parts).strip(), first_media_url diff --git a/flocks/channel/builtin/wecom/inbound_media.py b/flocks/channel/builtin/wecom/inbound_media.py new file mode 100644 index 000000000..fe8c2631a --- /dev/null +++ b/flocks/channel/builtin/wecom/inbound_media.py @@ -0,0 +1,278 @@ +""" +WeCom inbound media download helpers. + +Downloads and decrypts file/image media received via the WeCom AI Bot +WebSocket channel. WeCom encrypts all media with AES-256-CBC; the +decryption key (``aeskey``) is provided in the message frame alongside +the download URL. +""" + +from __future__ import annotations + +import mimetypes +import os +import re +import datetime +import importlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from urllib.parse import unquote, urlparse + +from flocks.channel.base import InboundMessage +from flocks.channel.media_filename import sanitize_filename +from flocks.utils.log import Log + +log = Log.create(service="channel.wecom.media") + +_DEFAULT_MAX_INBOUND_MEDIA_BYTES = 30 * 1024 * 1024 + + +class WeComInboundMediaTooLarge(ValueError): + """企微入站媒体超过允许大小。""" + + +@dataclass +class DownloadedInboundMedia: + filename: str + mime: str + url: str + source: dict + + +def _media_storage_dir(account_id: str) -> Path: + return ( + Path.home() + / ".flocks" + / "data" + / "channel_media" + / "wecom" + / account_id + / datetime.date.today().isoformat() + ) + + +def _sanitize_filename(name: str) -> str: + return sanitize_filename(name) + + +def _guess_mime_from_ext(filename: str) -> Optional[str]: + _, ext = os.path.splitext(filename) + if ext: + return mimetypes.guess_type(filename)[0] + return None + + +def _filename_from_content_disposition(value: str) -> Optional[str]: + match = re.search(r'filename\*?=(?:UTF-8\'\')?"?([^";]+)"?', value, re.I) + if not match: + return None + return unquote(match.group(1).strip()) + + +def _max_size_error(max_bytes: int) -> ValueError: + return WeComInboundMediaTooLarge( + f"WeCom inbound media too large: >{max_bytes // (1024 * 1024)}MB" + ) + + +def _guess_filename(msg: InboundMessage, media_url: str, cd_filename: Optional[str] = None) -> str: + raw_body = msg.raw if isinstance(msg.raw, dict) else {} + msg_type = raw_body.get("msgtype", "") + + if msg_type == "file": + raw_name = str(raw_body.get("file", {}).get("filename", "") or "").strip() + if raw_name: + return _sanitize_filename(raw_name) + + if msg_type == "image": + raw_name = str(raw_body.get("image", {}).get("filename", "") or "").strip() + if raw_name: + return _sanitize_filename(raw_name) + + if msg_type == "mixed": + for item in raw_body.get("mixed", {}).get("msg_item", []): + item_type = item.get("msgtype", "") + if item_type == "file": + raw_name = str(item.get("file", {}).get("filename", "") or "").strip() + if raw_name: + return _sanitize_filename(raw_name) + if item_type == "image": + raw_name = str(item.get("image", {}).get("filename", "") or "").strip() + if raw_name: + return _sanitize_filename(raw_name) + + if cd_filename: + return _sanitize_filename(cd_filename) + + url_path = urlparse(media_url).path + url_filename = os.path.basename(url_path) + if url_filename and "." in url_filename: + return _sanitize_filename(url_filename) + + prefix = "image" if msg_type == "image" else "file" + msg_id = msg.message_id or "unknown" + return _sanitize_filename(f"{prefix}_{msg_id[:12]}") + + +def _extract_aes_key(msg: InboundMessage) -> Optional[str]: + raw_body = msg.raw if isinstance(msg.raw, dict) else {} + msg_type = raw_body.get("msgtype", "") + + if msg_type == "file": + return str(raw_body.get("file", {}).get("aeskey", "") or "").strip() or None + if msg_type == "image": + return str(raw_body.get("image", {}).get("aeskey", "") or "").strip() or None + if msg_type == "mixed": + for item in raw_body.get("mixed", {}).get("msg_item", []): + item_type = item.get("msgtype", "") + if item_type == "file": + key = str(item.get("file", {}).get("aeskey", "") or "").strip() + if key: + return key + if item_type == "image": + key = str(item.get("image", {}).get("aeskey", "") or "").strip() + if key: + return key + return None + + +async def _close_api_client(api_client: Any) -> None: + client = getattr(api_client, "_client", None) + close = getattr(client, "aclose", None) + if close: + try: + await close() + except Exception as e: + log.warning("wecom.media.client_close_failed", {"error": str(e)}) + + +async def _download_file_limited( + api_client: Any, + media_url: str, + max_bytes: int, +) -> tuple[bytes, Optional[str]]: + client = getattr(api_client, "_client", None) + stream = getattr(client, "stream", None) + if callable(stream): + chunks: list[bytes] = [] + total = 0 + filename: Optional[str] = None + async with stream("GET", media_url) as resp: + if hasattr(resp, "raise_for_status"): + resp.raise_for_status() + headers = getattr(resp, "headers", {}) or {} + content_length = headers.get("content-length") or headers.get("Content-Length") + if content_length: + try: + if int(content_length) > max_bytes: + raise _max_size_error(max_bytes) + except ValueError as e: + if "invalid literal" not in str(e): + raise + content_disposition = ( + headers.get("content-disposition") + or headers.get("Content-Disposition") + or "" + ) + filename = _filename_from_content_disposition(content_disposition) + async for chunk in resp.aiter_bytes(8192): + total += len(chunk) + if total > max_bytes: + raise _max_size_error(max_bytes) + chunks.append(chunk) + return b"".join(chunks), filename + + result = await api_client.download_file_raw(media_url) + buffer: bytes = result["buffer"] + if len(buffer) > max_bytes: + raise _max_size_error(max_bytes) + return buffer, result.get("filename") + + +async def download_inbound_media( + msg: InboundMessage, + config: dict, + *, + max_bytes: int = _DEFAULT_MAX_INBOUND_MEDIA_BYTES, +) -> Optional[DownloadedInboundMedia]: + media_url = msg.media_url + if not media_url: + return None + + aes_key = _extract_aes_key(msg) + + api_client = None + try: + sdk = importlib.import_module("wecom_aibot_sdk") + api_client = sdk.WeComApiClient(log, timeout=30000) + buffer, cd_filename = await _download_file_limited( + api_client, + media_url, + max_bytes, + ) + + if aes_key: + try: + buffer = sdk.decrypt_file(buffer, aes_key) + except Exception as e: + log.warning("wecom.media.decrypt_failed", { + "url": media_url[:200], + "message_id": msg.message_id, + "error": str(e), + }) + return None + if len(buffer) > max_bytes: + raise _max_size_error(max_bytes) + + except ImportError: + log.warning("wecom.media.sdk_not_available") + return None + + except WeComInboundMediaTooLarge as e: + log.warning("wecom.media.file_too_large", { + "url": media_url[:200], + "message_id": msg.message_id, + "error": str(e), + }) + return None + + except Exception as e: + log.warning("wecom.media.download_failed", { + "url": media_url[:200], + "message_id": msg.message_id, + "error": str(e), + }) + return None + + finally: + if api_client is not None: + await _close_api_client(api_client) + + filename = _guess_filename(msg, media_url, cd_filename) + + if not filename or "." not in filename: + guessed_mime = _guess_mime_from_ext(filename or "") + ext = mimetypes.guess_extension(guessed_mime) if guessed_mime else "" + if ext: + filename = f"{filename}{ext}" + + mime = _guess_mime_from_ext(filename) or "application/octet-stream" + + storage_dir = _media_storage_dir(msg.account_id or "default") + storage_dir.mkdir(parents=True, exist_ok=True) + msg_id = msg.message_id or "unknown" + file_path = storage_dir / _sanitize_filename(f"{msg_id}_{filename}") + file_path.write_bytes(buffer) + + return DownloadedInboundMedia( + filename=filename, + mime=mime, + url=file_path.resolve().as_uri(), + source={ + "channel": "wecom", + "account_id": msg.account_id, + "message_id": msg.message_id, + "media_url": msg.media_url, + }, + ) diff --git a/flocks/channel/builtin/wecom/media.py b/flocks/channel/builtin/wecom/media.py new file mode 100644 index 000000000..7102d04ca --- /dev/null +++ b/flocks/channel/builtin/wecom/media.py @@ -0,0 +1,106 @@ +""" +WeCom outbound media helpers. +""" + +from __future__ import annotations + +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Literal +from urllib.parse import unquote, urlparse + +import httpx + +from flocks.channel.media_filename import sanitize_filename + +_DEFAULT_MAX_MEDIA_BYTES = 50 * 1024 * 1024 +WeComOutboundMediaType = Literal["file", "image", "voice", "video"] + + +@dataclass +class PreparedWeComMedia: + data: bytes + filename: str + media_type: WeComOutboundMediaType + + +def _sanitize_filename(name: str) -> str: + return sanitize_filename(name) + + +def _media_type_from_filename(filename: str) -> WeComOutboundMediaType: + mime = mimetypes.guess_type(filename)[0] or "" + if mime.startswith("image/"): + return "image" + if mime.startswith("audio/"): + return "voice" + if mime.startswith("video/"): + return "video" + return "file" + + +def _filename_from_url(url: str) -> str: + parsed = urlparse(url) + basename = os.path.basename(parsed.path) + return _sanitize_filename(unquote(basename)) if basename else "attachment" + + +def _path_from_media_url(media_url: str) -> Path | None: + parsed = urlparse(media_url) + if parsed.scheme == "file": + return Path(unquote(parsed.path)) + if parsed.scheme: + return None + return Path(media_url) + + +def _read_local_file_limited(path: Path, max_bytes: int) -> bytes: + if not path.is_file(): + raise FileNotFoundError(f"WeCom media file not found: {path}") + size = path.stat().st_size + if size > max_bytes: + raise ValueError(f"WeCom outbound media too large: >{max_bytes // (1024 * 1024)}MB") + return path.read_bytes() + + +async def _fetch_http_file_limited(url: str, max_bytes: int) -> tuple[bytes, str]: + async with httpx.AsyncClient(timeout=60) as client: + async with client.stream("GET", url, follow_redirects=True) as resp: + resp.raise_for_status() + content_length = resp.headers.get("content-length") + if content_length and int(content_length) > max_bytes: + raise ValueError(f"WeCom outbound media too large: >{max_bytes // (1024 * 1024)}MB") + + chunks: list[bytes] = [] + total = 0 + async for chunk in resp.aiter_bytes(8192): + total += len(chunk) + if total > max_bytes: + raise ValueError(f"WeCom outbound media too large: >{max_bytes // (1024 * 1024)}MB") + chunks.append(chunk) + + return b"".join(chunks), _filename_from_url(url) + + +async def prepare_wecom_media( + media_url: str, + *, + max_bytes: int = _DEFAULT_MAX_MEDIA_BYTES, +) -> PreparedWeComMedia: + parsed = urlparse(media_url) + if parsed.scheme in ("http", "https"): + data, filename = await _fetch_http_file_limited(media_url, max_bytes) + else: + path = _path_from_media_url(media_url) + if path is None: + raise ValueError(f"Unsupported WeCom media URL scheme: {parsed.scheme}") + data = _read_local_file_limited(path, max_bytes) + filename = _sanitize_filename(path.name) + + return PreparedWeComMedia( + data=data, + filename=filename, + media_type=_media_type_from_filename(filename), + ) diff --git a/flocks/channel/inbound/dispatcher.py b/flocks/channel/inbound/dispatcher.py index 70647aadd..96987c237 100644 --- a/flocks/channel/inbound/dispatcher.py +++ b/flocks/channel/inbound/dispatcher.py @@ -1172,28 +1172,29 @@ async def _append_user_message( parsed = urlparse(msg.media_url) scheme = parsed.scheme.lower() - if msg.channel_id == "feishu" and channel_config is not None: - # Feishu: media is still on the remote server, download first. - from flocks.channel.builtin.feishu.inbound_media import download_inbound_media - - raw_cfg = channel_config.model_dump(by_alias=True, exclude_none=True) - media = await download_inbound_media(msg, raw_cfg) - if not media: - return + raw_cfg = ( + channel_config.model_dump(by_alias=True, exclude_none=True) + if channel_config is not None else {} + ) - await Message.store_part( - session_id, - message.id, - FilePart( - sessionID=session_id, - messageID=message.id, - mime=media.mime, - filename=media.filename, - url=media.url, - source=media.source, - ), + media = await _download_channel_media(msg, raw_cfg) + + file_part: Optional[FilePart] = None + if media is not None: + file_part = FilePart( + sessionID=session_id, + messageID=message.id, + mime=media.mime, + filename=media.filename, + url=media.url, + source=media.source, ) - + await Message.store_part(session_id, message.id, file_part) + log.info("dispatcher.inbound_media_attached", { + "channel_id": msg.channel_id, + "filename": media.filename, + "mime": media.mime, + }) elif scheme in ("", "file"): # Local file already downloaded by the channel plugin (e.g. weixin). # file:// URIs may have URL-encoded paths (e.g. Chinese filenames). @@ -1211,23 +1212,72 @@ async def _append_user_message( or "application/octet-stream" ) file_uri = Path(local_path).resolve().as_uri() - await Message.store_part( - session_id, - message.id, - FilePart( - sessionID=session_id, - messageID=message.id, - mime=mime, - filename=filename, - url=file_uri, - source=None, - ), + file_part = FilePart( + sessionID=session_id, + messageID=message.id, + mime=mime, + filename=filename, + url=file_uri, + source=None, ) + await Message.store_part(session_id, message.id, file_part) log.info("dispatcher.inbound_media_attached", { "channel_id": msg.channel_id, "filename": filename, "mime": mime, }) + else: + # Remote URL with no channel-specific downloader — keep the + # reference as a TextPart hint so the agent at least knows + # the message carried media. Do not fabricate a FilePart. + log.debug("dispatcher.inbound_media_remote_only", { + "channel_id": msg.channel_id, + "media_url": msg.media_url[:200], + }) + return + + # Replace the placeholder text ([图片消息] / [文件消息] / [文件消息: x]) + # with a path hint so the agent can see the attachment is local. + try: + from pathlib import PurePosixPath + from flocks.session.message import TextPart + + file_path_str = ( + file_part.url.replace("file://", "") if file_part else "" + ) + display_path = str(PurePosixPath(file_path_str)) if file_path_str else "" + parts = await Message.parts(message.id, session_id=session_id) + for p in parts: + if p.type == "text" and hasattr(p, "text") and p.text: + if _is_placeholder_text(p.text): + updated = TextPart( + id=p.id, + sessionID=session_id, + messageID=message.id, + type="text", + text=f"Attached files:\n- {display_path}", + ) + await Message.store_part(session_id, message.id, updated) + try: + from flocks.server.routes.event import publish_event + await publish_event("message.part.updated", { + "part": updated.model_dump(by_alias=True, exclude_none=True), + "sessionID": session_id, + }) + except Exception: + pass + break + except Exception: + pass + + try: + from flocks.server.routes.event import publish_event + await publish_event("message.part.updated", { + "part": file_part.model_dump(by_alias=True, exclude_none=True), + "sessionID": session_id, + }) + except Exception: + pass except Exception as e: log.warning("dispatcher.inbound_media_download_failed", { @@ -1431,3 +1481,88 @@ async def _fetch_quoted_message( # 权限错误通知冷却时间（同一账号 5 分钟内只通知一次） _PERM_NOTICE_COOLDOWN = 300 _perm_notice_last: dict[str, float] = {} + + +# --------------------------------------------------------------------------- +# Per-channel inbound media download +# --------------------------------------------------------------------------- + +# Mapping of channel_id → module-level ``download_inbound_media`` callable. +# A channel opts in by adding an entry here. The dispatcher looks up +# ``msg.channel_id`` to decide whether to invoke a custom downloader +# (e.g. WeCom decrypt + download, Telegram getFile resolve) or fall +# back to the generic ``file://`` / no-scheme handler. +_DOWNLOADERS: dict[str, Any] = {} + + +def register_inbound_media_downloader(channel_id: str, downloader: Any) -> None: + """Register a per-channel ``download_inbound_media`` callable.""" + _DOWNLOADERS[channel_id] = downloader + + +def _is_placeholder_text(text: str) -> bool: + """True if *text* is one of the channel-generated media placeholders.""" + if not text: + return False + placeholders = ( + "[图片消息]", + "[文件消息]", + "[Image]", + "[Attachment]", + "[图片]", + "[文件]", + ) + if text in placeholders: + return True + return text.startswith("[文件消息:") + + +# Best-effort eager registration at import time. Channels that need +# more control can call ``register_inbound_media_downloader`` themselves. +try: + from flocks.channel.builtin.feishu import inbound_media as _feishu_inbound_media + register_inbound_media_downloader("feishu", _feishu_inbound_media.download_inbound_media) +except Exception: # pragma: no cover - feishu not installed + pass + +try: + from flocks.channel.builtin.wecom import inbound_media as _wecom_inbound_media + register_inbound_media_downloader("wecom", _wecom_inbound_media.download_inbound_media) +except Exception: # pragma: no cover - wecom SDK not installed + pass + +try: + from flocks.channel.builtin.dingtalk import inbound_media as _dingtalk_inbound_media + register_inbound_media_downloader("dingtalk", _dingtalk_inbound_media.download_inbound_media) +except Exception: # pragma: no cover + pass + +try: + from flocks.channel.builtin.telegram import inbound_media as _telegram_inbound_media + register_inbound_media_downloader("telegram", _telegram_inbound_media.download_inbound_media) +except Exception: # pragma: no cover + pass + + +async def _download_channel_media(msg: "InboundMessage", config: dict) -> Any: + """Dispatch inbound media download to the appropriate channel handler. + + Looks up the channel's downloader module dynamically so test + monkeypatches that target ``flocks.channel.builtin..inbound_media`` + still apply — caching a bound callable would otherwise freeze the + reference at module-import time and bypass the patch. + """ + channel_id = msg.channel_id + if channel_id == "feishu": + from flocks.channel.builtin.feishu import inbound_media as _feishu_inbound_media + return await _feishu_inbound_media.download_inbound_media(msg, config) + if channel_id == "wecom": + from flocks.channel.builtin.wecom import inbound_media as _wecom_inbound_media + return await _wecom_inbound_media.download_inbound_media(msg, config) + if channel_id == "dingtalk": + from flocks.channel.builtin.dingtalk import inbound_media as _dingtalk_inbound_media + return await _dingtalk_inbound_media.download_inbound_media(msg, config) + if channel_id == "telegram": + from flocks.channel.builtin.telegram import inbound_media as _telegram_inbound_media + return await _telegram_inbound_media.download_inbound_media(msg, config) + return None diff --git a/flocks/channel/media_filename.py b/flocks/channel/media_filename.py new file mode 100644 index 000000000..0f181bc5f --- /dev/null +++ b/flocks/channel/media_filename.py @@ -0,0 +1,18 @@ +"""Filename helpers shared by channel media integrations.""" + +from __future__ import annotations + +import re +import unicodedata + +_INVALID_FILENAME_CHARS_RE = re.compile(r'[\x00-\x1f\x7f/\\:*?"<>|]+') + + +def sanitize_filename(name: str, *, fallback: str = "attachment", max_chars: int = 120) -> str: + """Return a filesystem-safe filename while preserving Unicode text.""" + cleaned = unicodedata.normalize("NFC", str(name or "")).strip() + cleaned = _INVALID_FILENAME_CHARS_RE.sub("_", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + if cleaned in {".", ".."}: + cleaned = "" + return cleaned[:max_chars] or fallback diff --git a/flocks/channel/registry.py b/flocks/channel/registry.py index 03819050e..c782f937a 100644 --- a/flocks/channel/registry.py +++ b/flocks/channel/registry.py @@ -32,7 +32,21 @@ def __init__(self) -> None: def register(self, plugin: ChannelPlugin) -> None: meta = plugin.meta() - self._channels[meta.id.lower()] = plugin + key = meta.id.lower() + existing = self._channels.get(key) + if existing is not None: + log.debug( + "channel.register.skip_existing", + { + "id": meta.id, + "aliases": meta.aliases, + "existing_instance": f"{id(existing):x}", + "candidate_instance": f"{id(plugin):x}", + }, + ) + return + + self._channels[key] = plugin for alias in meta.aliases: self._channels[alias.lower()] = plugin log.info("channel.registered", {"id": meta.id, "aliases": meta.aliases}) @@ -105,6 +119,7 @@ def _consume_channels(items: list, source: str) -> None: dedup_key=lambda ch: ch.meta().id, recursive=True, max_depth=2, + load_once=True, )) def _load_plugin_channels(self) -> None: diff --git a/flocks/cli/commands/skill.py b/flocks/cli/commands/skill.py index a7661beff..ef9b6c01b 100644 --- a/flocks/cli/commands/skill.py +++ b/flocks/cli/commands/skill.py @@ -4,14 +4,16 @@ Provides skill management commands: flocks skills list – list all discovered skills flocks skills status – show eligibility status (deps check) - flocks skills find – search installable skills (clawhub + GitHub) - flocks skills install – install a skill from URL/GitHub/clawhub/local + flocks skills find – search installable skills + flocks skills install – install a skill from URL/GitHub/clawhub/skills.sh/SafeSkill/local flocks skills remove – uninstall a user-managed skill flocks skills install-deps – install a skill's declared tool dependencies """ import asyncio +import json import re +import shutil from typing import Optional import typer @@ -146,22 +148,25 @@ def check_status( expand=False, )) - if not_ready: - console.print("\n[yellow bold]Skills with missing dependencies:[/yellow bold]") - table = Table(show_header=True, header_style="bold yellow", box=None, pad_edge=False) + if skills: + table = Table(show_header=True, header_style="bold cyan", box=None, pad_edge=False) table.add_column("Name", min_width=20) - table.add_column("Missing", min_width=40) - table.add_column("Install with") - for s in not_ready: - missing_str = ", ".join(s.missing or []) + table.add_column("Status", min_width=16) + table.add_column("Details", min_width=30) + for s in sorted(skills, key=lambda item: item.name): install_hint = "" if s.install_specs: - spec = s.install_specs[0] - if spec.kind == "brew" and spec.formula: - install_hint = f"flocks skills install-deps {s.name}" - elif spec.kind in ("npm", "uv", "pip") and spec.package: - install_hint = f"flocks skills install-deps {s.name}" - table.add_row(s.name, missing_str, install_hint or "—") + install_hint = f"flocks skills install-deps {s.name}" + if s.eligible is False: + status = "[yellow]missing deps[/yellow]" + details = ", ".join(s.missing or []) or install_hint or "—" + elif s.requires: + status = "[green]ready[/green]" + details = install_hint or "requirements satisfied" + else: + status = "[dim]no requirements[/dim]" + details = "—" + table.add_row(s.name, status, details) console.print(table) @@ -174,7 +179,7 @@ def find_skills( query: str = typer.Argument(..., help="Search keyword, e.g. 'threat intel' or 'github'"), json_output: bool = typer.Option(False, "--json", help="Output as JSON"), ): - """Search installable skills from clawhub and installed skills.""" + """Search installable skills from local, clawhub, skills.sh, SafeSkill, and GitHub.""" results = asyncio.run(_search_skills(query)) if json_output: @@ -184,7 +189,7 @@ def find_skills( if not results: console.print(f"[dim]No skills found matching '{query}'.[/dim]") - console.print("\n[dim]Tip: try browsing https://clawhub.com for more skills.[/dim]") + console.print("\n[dim]Tip: try browsing https://skills.sh or https://safeskill.cn for more skills.[/dim]") return table = Table(show_header=True, header_style="bold cyan", box=None, pad_edge=False) @@ -211,7 +216,9 @@ async def _search_skills(query: str) -> list: Search for skills matching query from multiple sources: 1. Already-installed local skills 2. clawhub.com search API - 3. Curated GitHub collections (Anthropic-Cybersecurity-Skills etc.) + 3. skills.sh search API + 4. SafeSkill CLI search + 5. Curated GitHub collections (Anthropic-Cybersecurity-Skills etc.) """ results = [] query_lower = query.lower() @@ -231,7 +238,13 @@ async def _search_skills(query: str) -> list: clawhub_results = await _search_clawhub(query) results.extend(clawhub_results) - # 3. Search curated GitHub skill collections + # 3. Search skills.sh + results.extend(await _search_skills_sh(query)) + + # 4. Search SafeSkill when the CLI is available + results.extend(await _search_safeskill(query)) + + # 5. Search curated GitHub skill collections github_results = await _search_github_collections(query) # Deduplicate by name existing_names = {r["name"] for r in results} @@ -272,6 +285,119 @@ async def _search_clawhub(query: str) -> list: return [] +async def _search_skills_sh(query: str) -> list: + """Query skills.sh search API.""" + try: + import httpx + async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client: + resp = await client.get( + "https://skills.sh/api/search", + params={"q": query, "limit": 10}, + headers={"Accept": "application/json"}, + ) + if resp.status_code != 200: + return [] + data = resp.json() + skills = data.get("skills", []) if isinstance(data, dict) else [] + results = [] + for item in skills: + if not isinstance(item, dict): + continue + identifier = item.get("id") + repo = item.get("source") + skill_id = item.get("skillId") + if not identifier and isinstance(repo, str) and isinstance(skill_id, str): + identifier = f"{repo}/{skill_id}" + if not isinstance(identifier, str) or identifier.count("/") < 2: + continue + parts = identifier.split("/", 2) + name = str(item.get("name") or parts[-1].split("/")[-1]) + installs = item.get("installs") + installs_suffix = ( + f" · {installs:,} installs" + if isinstance(installs, int) + else "" + ) + results.append({ + "name": name, + "description": f"Indexed by skills.sh from {parts[0]}/{parts[1]}{installs_suffix}", + "source": "skills.sh", + "install_hint": f"skills-sh:{identifier}", + }) + return results + except Exception: + return [] + + +async def _search_safeskill(query: str) -> list: + """Search SafeSkill through its npx CLI when available.""" + npx = shutil.which("npx") + if not npx: + return [] + try: + proc = await asyncio.create_subprocess_exec( + npx, + "-y", + "@safeskill/cli", + "find", + query, + "--json", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout_b, _stderr_b = await asyncio.wait_for(proc.communicate(), timeout=20) + if proc.returncode != 0: + return [] + text = stdout_b.decode(errors="replace").strip() + if not text: + return [] + try: + data = json.loads(text) + except json.JSONDecodeError: + return _parse_safeskill_text_results(text) + items = data if isinstance(data, list) else data.get("skills", data.get("results", [])) + if not isinstance(items, list): + return [] + results = [] + for item in items: + if not isinstance(item, dict): + continue + source = item.get("source") or item.get("install") or item.get("url") or item.get("id") + name = item.get("name") or item.get("slug") or source + if not source or not name: + continue + results.append({ + "name": str(name), + "description": str(item.get("description") or ""), + "source": "safeskill.cn", + "install_hint": f"safeskill:{source}", + }) + return results + except Exception: + return [] + + +def _parse_safeskill_text_results(text: str) -> list: + """Best-effort parsing for SafeSkill CLI text output.""" + results = [] + for line in text.splitlines(): + clean = line.strip(" -\t") + if not clean or clean.lower().startswith(("name", "found", "search")): + continue + match = re.search(r"(safeskill://\S+|https?://\S+|[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+)", clean) + if not match: + continue + source = match.group(1).rstrip(",.;") + name = source.rstrip("/").split("/")[-1] + results.append({ + "name": name, + "description": clean, + "source": "safeskill.cn", + "install_hint": f"safeskill:{source}", + }) + return results + + async def _search_github_collections(query: str) -> list: """Search curated GitHub skill collection repositories.""" collections = [ @@ -345,6 +471,8 @@ def install_skill( help=( "Install source:\n" " clawhub: – clawhub.com registry\n" + " skills-sh: – skills.sh identifier (owner/repo/skill)\n" + " safeskill: – SafeSkill Hub/GitHub/local source via SafeSkill CLI\n" " github:/ – GitHub repository\n" " / – GitHub shorthand\n" " https://... – direct SKILL.md URL\n" @@ -355,18 +483,24 @@ def install_skill( None, "--skill", "-s", - help="Skill subdirectory name within the source repo (e.g. --skill find-skills)", + help="Skill subdirectory name within the source repo (e.g. --skill code-review)", ), scope: str = typer.Option( "global", "--scope", help="'global' (default) or 'project'", ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts from downstream CLIs (e.g. `skills add`) for non-interactive installs.", + ), ): """Install a skill from an external source.""" # If --skill is provided, append it as a subpath to the source - # e.g. source=https://github.com/owner/repo --skill find-skills - # → resolved as github:owner/repo/find-skills + # e.g. source=https://github.com/owner/repo --skill code-review + # → resolved as github:owner/repo/code-review effective_source = source if skill: # Strip trailing slash from source and append skill subpath @@ -374,7 +508,7 @@ def install_skill( with console.status(f"[bold cyan]Installing skill from {effective_source!r}...[/bold cyan]"): result = asyncio.run( - SkillInstaller.install_from_source(effective_source, scope=scope) + SkillInstaller.install_from_source(effective_source, scope=scope, yes=yes) ) if result.success: @@ -450,7 +584,7 @@ def install_deps( all_ok = True for r in results: - cmd_str = " ".join(r.command) if r.command else "—" + cmd_str = " ".join(r.command) if r.command else (r.message or "—") if r.success: console.print(f"[green]✓[/green] {cmd_str}") if r.stdout.strip(): diff --git a/flocks/cli/service_manager.py b/flocks/cli/service_manager.py index d0de2066a..edb207108 100644 --- a/flocks/cli/service_manager.py +++ b/flocks/cli/service_manager.py @@ -26,7 +26,6 @@ import httpx from flocks.browser.admin import stop_all_daemons as stop_all_browser_daemons -from flocks.utils.log import rotate_log_file try: import fcntl @@ -799,11 +798,10 @@ def _port_owner_lookup_available() -> bool: def _bind_port_available(port: int) -> bool: - """Return True when the TCP port can be bound locally.""" + """Return True when the TCP port can be bound on any local IPv4 interface.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: - sock.bind(("127.0.0.1", port)) + sock.bind(("0.0.0.0", port)) except OSError: return False return True @@ -1682,7 +1680,6 @@ def _spawn_process( kwargs["start_new_session"] = True log_path.parent.mkdir(parents=True, exist_ok=True) - rotate_log_file(log_path) handle = log_path.open("a", encoding="utf-8") try: return subprocess.Popen( diff --git a/flocks/cli/session_runner.py b/flocks/cli/session_runner.py index d85967189..ccbf254cf 100644 --- a/flocks/cli/session_runner.py +++ b/flocks/cli/session_runner.py @@ -54,8 +54,7 @@ def _get_cli_callbacks() -> Optional['RunnerCallbacks']: # Tool display styles TOOL_STYLES: Dict[str, tuple] = { - "todowrite": ("Todo", "yellow bold"), - "todoread": ("Todo", "yellow bold"), + "todo": ("Todo", "yellow bold"), "bash": ("Bash", "red bold"), "edit": ("Edit", "green bold"), "write": ("Write", "green bold"), diff --git a/flocks/command/command.py b/flocks/command/command.py index 000e7db0f..7e9190753 100644 --- a/flocks/command/command.py +++ b/flocks/command/command.py @@ -260,22 +260,6 @@ def _ensure_defaults(cls) -> None: execution_kind="llm", allow_attachments=True, ), - CommandDef( - name="plan", - description="Create a plan for a task", - template="Create a detailed plan for: $ARGUMENTS", - agent="prometheus", - execution_kind="llm", - allow_attachments=True, - ), - CommandDef( - name="ask", - description="Ask a question without making changes", - template="$ARGUMENTS", - agent="ask", - execution_kind="llm", - allow_attachments=True, - ), CommandDef( name="tasks", description="Show task center overview", diff --git a/flocks/config/config.py b/flocks/config/config.py index d40b0ad3a..387f9ce4b 100644 --- a/flocks/config/config.py +++ b/flocks/config/config.py @@ -29,6 +29,35 @@ class PermissionAction(str, Enum): PermissionRule = Union[PermissionAction, Dict[str, Union[PermissionAction, Dict[str, PermissionAction]]]] +_LEGACY_TODO_TOOL_NAMES = {"todowrite", "todoread"} + + +def _canonical_permission_tool_name(tool: str) -> str: + if tool in _LEGACY_TODO_TOOL_NAMES: + return "todo" + return tool + + +def _merge_permission_action(existing: Any, incoming: Any) -> Any: + """Merge duplicate legacy permission names conservatively.""" + existing_value = existing.value if hasattr(existing, "value") else existing + incoming_value = incoming.value if hasattr(incoming, "value") else incoming + if existing_value == PermissionAction.DENY.value or incoming_value == PermissionAction.DENY.value: + return PermissionAction.DENY + return existing if existing is not None else incoming + + +def _assign_permission(permission_dict: Dict[str, Any], tool: str, action: Any) -> None: + canonical_tool = _canonical_permission_tool_name(tool) + if canonical_tool in permission_dict: + permission_dict[canonical_tool] = _merge_permission_action( + permission_dict[canonical_tool], + action, + ) + return + permission_dict[canonical_tool] = action + + class PermissionConfig(BaseModel): """Permission configuration (simplified for Phase 1-3)""" model_config = {"extra": "allow"} # Allow additional fields @@ -40,16 +69,24 @@ class PermissionConfig(BaseModel): bash: Optional[PermissionRule] = None task: Optional[PermissionRule] = None external_directory: Optional[PermissionRule] = None - todowrite: Optional[PermissionAction] = None - todoread: Optional[PermissionAction] = None + todo: Optional[PermissionAction] = None question: Optional[PermissionAction] = None webfetch: Optional[PermissionAction] = None websearch: Optional[PermissionAction] = None lsp: Optional[PermissionRule] = None doom_loop: Optional[PermissionAction] = None delegate_task: Optional[PermissionRule] = None - background_output: Optional[PermissionRule] = None - background_cancel: Optional[PermissionRule] = None + + @model_validator(mode="before") + @classmethod + def migrate_legacy_todo_permissions(cls, data): + if not isinstance(data, dict): + return data + migrated = dict(data) + for legacy_name in _LEGACY_TODO_TOOL_NAMES: + if legacy_name in migrated: + _assign_permission(migrated, legacy_name, migrated.pop(legacy_name)) + return migrated # ==================== Agent Configuration ==================== @@ -103,9 +140,9 @@ def process_agent(self): action = PermissionAction.ALLOW if enabled else PermissionAction.DENY # Map write/edit/patch to edit if tool in ["write", "edit", "patch"]: - permission_dict["edit"] = action + _assign_permission(permission_dict, "edit", action) else: - permission_dict[tool] = action + _assign_permission(permission_dict, tool, action) self.permission = permission_dict @@ -679,9 +716,9 @@ def post_process(self): for tool, enabled in self.tools.items(): action = PermissionAction.ALLOW if enabled else PermissionAction.DENY if tool in ["write", "edit", "patch"]: - permission_dict["edit"] = action + _assign_permission(permission_dict, "edit", action) else: - permission_dict[tool] = action + _assign_permission(permission_dict, tool, action) self.permission = permission_dict diff --git a/flocks/ingest/kafka/manager.py b/flocks/ingest/kafka/manager.py index 4cec529ad..37187f732 100644 --- a/flocks/ingest/kafka/manager.py +++ b/flocks/ingest/kafka/manager.py @@ -23,8 +23,9 @@ import hashlib import json import time +import uuid from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional from flocks.storage.storage import Storage from flocks.utils.log import Log @@ -40,6 +41,9 @@ from flocks.workflow.runner import run_workflow from flocks.ingest.kafka.constants import WORKFLOW_KAFKA_CONFIG_PREFIX +from flocks.workflow.triggers.compat import legacy_kafka_trigger_from_config +from flocks.workflow.triggers.dispatcher import EventDispatcher, TriggerDispatchError, build_trigger_event +from flocks.workflow.triggers.models import TriggerDefinition, workflow_json_declares_triggers, workflow_trigger_definitions_from_json log = Log.create(service="kafka.manager") @@ -178,13 +182,18 @@ def _compact_for_kafka_storage(outputs: Any) -> Dict[str, Any]: return compacted -def _compact_history_for_kafka_storage(history: Any, *, input_key: str) -> List[Any]: +def _compact_history_for_kafka_storage( + history: Any, + *, + input_key: str, + input_keys: Iterable[str] | None = None, +) -> List[Any]: compacted = compact_history_for_storage( history, keys=_KAFKA_STORAGE_LIST_KEYS, size_threshold=0, ) - raw_input_keys = _KAFKA_RAW_INPUT_KEYS | frozenset({input_key}) + raw_input_keys = _KAFKA_RAW_INPUT_KEYS | frozenset(input_keys or {input_key}) for step in compacted: if not isinstance(step, dict): continue @@ -216,11 +225,46 @@ def __init__(self) -> None: # Per-workflow event signalled once the consumer has either connected # successfully or failed; used by ``restart_workflow``. self._ready: dict[str, asyncio.Event] = {} + self._dispatcher = EventDispatcher() @staticmethod def _config_key(workflow_id: str) -> str: return f"{WORKFLOW_KAFKA_CONFIG_PREFIX}{workflow_id}" + @staticmethod + def _default_trigger_from_config(data: Dict[str, Any]) -> TriggerDefinition: + trigger = legacy_kafka_trigger_from_config(data) + if trigger is None: + return TriggerDefinition.model_validate( + { + "id": "kafka-default", + "type": "kafka", + "enabled": bool(data.get("enabled")), + "source": { + "inputBroker": data.get("inputBroker") or "", + "inputTopic": data.get("inputTopic") or "", + "inputGroupId": data.get("inputGroupId") or "", + "autoOffsetReset": data.get("autoOffsetReset") or "latest", + }, + "mapping": { + str(data.get("inputKey") or "kafka_message"): "$.body", + }, + "inputs": _strip_execution_only_comments( + data.get("inputs") if isinstance(data.get("inputs"), dict) else {} + ), + "updatedAt": data.get("updatedAt"), + } + ) + return trigger + + def _resolve_active_trigger(self, workflow_json: Dict[str, Any], data: Dict[str, Any]) -> TriggerDefinition: + if workflow_json_declares_triggers(workflow_json): + triggers = workflow_trigger_definitions_from_json(workflow_json) + trigger = next((item for item in triggers if item.type == "kafka"), None) + if trigger is not None: + return trigger + return self._default_trigger_from_config(data) + async def start_all(self) -> None: try: keys = await Storage.list_keys(WORKFLOW_KAFKA_CONFIG_PREFIX) @@ -343,10 +387,10 @@ async def restart_workflow(self, workflow_id: str) -> Dict[str, Any]: log.warning("kafka.workflow_json_missing_on_start", {"workflow_id": workflow_id}) return {"state": "failed", "error": err} + trigger = self._resolve_active_trigger(workflow_json, data) group_id = str(data.get("inputGroupId") or "").strip() or f"flocks-consumer-{workflow_id}" - input_key = str(data.get("inputKey") or "kafka_message") configured_inputs = _strip_execution_only_comments( - data.get("inputs") if isinstance(data.get("inputs"), dict) else {} + trigger.inputs if isinstance(trigger.inputs, dict) else {} ) queue: asyncio.Queue = asyncio.Queue(maxsize=_MAX_QUEUE_SIZE) @@ -373,7 +417,7 @@ async def restart_workflow(self, workflow_id: str) -> Dict[str, Any]: workers.append( asyncio.create_task( self._worker_loop( - workflow_id, workflow_json, input_key, configured_inputs, queue, abort, + workflow_id, workflow_json, trigger, configured_inputs, queue, abort, input_topic, ), name=f"kafka-worker-{workflow_id}-{i}", ) @@ -534,10 +578,11 @@ async def _worker_loop( self, workflow_id: str, workflow_json: Any, - input_key: str, + trigger: TriggerDefinition, configured_inputs: Dict[str, Any], queue: asyncio.Queue, abort: asyncio.Event, + source: str, ) -> None: while not abort.is_set(): try: @@ -550,7 +595,13 @@ async def _worker_loop( if isinstance(msg, _QueuedKafkaMessage): msg = _decode_message(msg.raw_value) await self._trigger_workflow( - workflow_id, workflow_json, msg, input_key, configured_inputs, + workflow_id, + workflow_json, + msg, + next(iter(trigger.mapping or {}), "kafka_message"), + configured_inputs, + trigger=trigger, + source=source, ) except asyncio.CancelledError: return @@ -567,67 +618,112 @@ async def _trigger_workflow( message: Any, input_key: str, configured_inputs: Optional[Dict[str, Any]] = None, + *, + trigger: Optional[TriggerDefinition] = None, + source: Optional[str] = None, ) -> None: + trigger = trigger or TriggerDefinition.model_validate( + { + "id": "kafka-default", + "type": "kafka", + "enabled": True, + "mapping": {input_key: "$.body"}, + "inputs": _strip_execution_only_comments( + configured_inputs if isinstance(configured_inputs, dict) else {} + ), + } + ) configured_inputs = _strip_execution_only_comments( configured_inputs if isinstance(configured_inputs, dict) else {} ) - inputs = {**configured_inputs, input_key: message} - input_params = {"_trigger": "kafka", input_key: _summarize_large_value(message)} - for key, value in configured_inputs.items(): - if key == input_key: - continue - input_params[key] = _summarize_large_value(value) - - exec_data = await create_execution_record( - workflow_id, - input_params=input_params, + event = build_trigger_event( + workflow_id=workflow_id, + trigger=trigger, + body=message, + raw=message, + source=source or str((trigger.source or {}).get("inputTopic") or "kafka"), + delivery_id=f"kafka-{uuid.uuid4().hex}", ) - exec_id = exec_data["id"] - start_time = time.time() - result = None - try: - result = await asyncio.to_thread( - run_workflow, - workflow=workflow_json, - inputs=inputs, - trace=False, - history_mode="summary", - ) - status, error_msg = resolve_execution_outcome(result) - duration = time.time() - start_time - exec_data.update({ - "status": status, - "outputResults": _compact_for_kafka_storage(result.outputs), - "finishedAt": int(time.time() * 1000), - "duration": duration, - "errorMessage": error_msg, - "executionLog": _compact_history_for_kafka_storage( - result.history, - input_key=input_key, - ), - "currentNodeId": result.last_node_id, - "currentPhase": status, - "currentStepIndex": result.steps, - }) - except Exception as exc: - duration = time.time() - start_time - log.error( - "kafka.workflow_run_failed", - {"workflow_id": workflow_id, "exec_id": exec_id, "error": str(exc)}, + async def _executor(mapped_inputs: Dict[str, Any]) -> Dict[str, Any]: + summarized_inputs = {"_trigger": trigger.type} + for key, value in mapped_inputs.items(): + summarized_inputs[key] = _summarize_large_value(value) + + exec_data = await create_execution_record( + workflow_id, + input_params=summarized_inputs, ) - exec_data.update({ - "status": "error", - "errorMessage": str(exc), - "finishedAt": int(time.time() * 1000), - "duration": duration, - "currentPhase": "error", - }) - finally: + exec_id = exec_data["id"] + start_time = time.time() + trigger_meta = mapped_inputs.get("_flocks", {}).get("trigger", {}) + trigger_input_keys = list((trigger.mapping or {}).keys()) or [input_key] try: - await record_execution_result(workflow_id, exec_id, exec_data) + result = await asyncio.to_thread( + run_workflow, + workflow=workflow_json, + inputs=mapped_inputs, + trace=False, + history_mode="summary", + ) + status, error_msg = resolve_execution_outcome(result) + duration = time.time() - start_time + exec_data.update({ + "status": status, + "outputResults": _compact_for_kafka_storage(result.outputs), + "finishedAt": int(time.time() * 1000), + "duration": duration, + "errorMessage": error_msg, + "executionLog": _compact_history_for_kafka_storage( + result.history, + input_key=input_key, + input_keys=trigger_input_keys, + ), + "currentNodeId": result.last_node_id, + "currentPhase": status, + "currentStepIndex": result.steps, + "triggerId": trigger.id, + "triggerType": trigger.type, + "deliveryId": trigger_meta.get("deliveryId"), + "attempt": trigger_meta.get("attempt"), + "triggerSource": trigger_meta.get("source"), + }) except Exception as exc: - log.warning("kafka.exec_record_failed", {"exec_id": exec_id, "error": str(exc)}) + duration = time.time() - start_time + log.error( + "kafka.workflow_run_failed", + {"workflow_id": workflow_id, "exec_id": exec_id, "error": str(exc)}, + ) + exec_data.update({ + "status": "error", + "errorMessage": str(exc), + "finishedAt": int(time.time() * 1000), + "duration": duration, + "currentPhase": "error", + "triggerId": trigger.id, + "triggerType": trigger.type, + "deliveryId": trigger_meta.get("deliveryId"), + "attempt": trigger_meta.get("attempt"), + "triggerSource": trigger_meta.get("source"), + }) + finally: + try: + await record_execution_result(workflow_id, exec_id, exec_data) + except Exception as exc: + log.warning("kafka.exec_record_failed", {"exec_id": exec_id, "error": str(exc)}) + return exec_data + + try: + await self._dispatcher.dispatch( + trigger=trigger, + event=event, + executor=_executor, + ) + except TriggerDispatchError as exc: + log.warning( + "kafka.trigger_dispatch_failed", + {"workflow_id": workflow_id, "trigger_id": trigger.id, "error": str(exc)}, + ) default_manager = KafkaManager() diff --git a/flocks/ingest/syslog/manager.py b/flocks/ingest/syslog/manager.py index 0ef9aae6b..8e938dbd4 100644 --- a/flocks/ingest/syslog/manager.py +++ b/flocks/ingest/syslog/manager.py @@ -4,6 +4,7 @@ import asyncio import time +import uuid from typing import Any, Dict, List from flocks.storage.storage import Storage @@ -20,6 +21,9 @@ from flocks.ingest.syslog.constants import WORKFLOW_SYSLOG_CONFIG_PREFIX from flocks.ingest.syslog.listener import run_tcp_syslog_server, run_udp_syslog_server +from flocks.workflow.triggers.compat import legacy_syslog_trigger_from_config +from flocks.workflow.triggers.dispatcher import EventDispatcher, TriggerDispatchError, build_trigger_event +from flocks.workflow.triggers.models import TriggerDefinition, workflow_json_declares_triggers, workflow_trigger_definitions_from_json log = Log.create(service="syslog.manager") @@ -125,11 +129,43 @@ def __init__(self) -> None: # successfully or failed. Used by ``restart_workflow`` so the HTTP # save endpoint can report bind failures synchronously. self._listener_ready: dict[str, asyncio.Event] = {} + self._dispatcher = EventDispatcher() @staticmethod def _config_key(workflow_id: str) -> str: return f"{WORKFLOW_SYSLOG_CONFIG_PREFIX}{workflow_id}" + @staticmethod + def _default_trigger_from_config(data: Dict[str, Any]) -> TriggerDefinition: + trigger = legacy_syslog_trigger_from_config(data) + if trigger is None: + return TriggerDefinition.model_validate( + { + "id": "syslog-default", + "type": "syslog", + "enabled": bool(data.get("enabled")), + "source": { + "protocol": data.get("protocol") or "udp", + "host": data.get("host") or "0.0.0.0", + "port": int(data.get("port") or 5140), + "format": data.get("format") or "auto", + }, + "mapping": { + str(data.get("inputKey") or "syslog_message"): "$.body", + }, + "updatedAt": data.get("updatedAt"), + } + ) + return trigger + + def _resolve_active_trigger(self, workflow_json: Dict[str, Any], data: Dict[str, Any]) -> TriggerDefinition: + if workflow_json_declares_triggers(workflow_json): + triggers = workflow_trigger_definitions_from_json(workflow_json) + trigger = next((item for item in triggers if item.type == "syslog"), None) + if trigger is not None: + return trigger + return self._default_trigger_from_config(data) + async def start_all(self) -> None: try: keys = await Storage.list_keys(WORKFLOW_SYSLOG_CONFIG_PREFIX) @@ -242,6 +278,7 @@ async def restart_workflow(self, workflow_id: str) -> Dict[str, Any]: log.warning("syslog.workflow_json_missing_on_start", {"workflow_id": workflow_id}) return {"state": "failed", "error": err} + trigger = self._resolve_active_trigger(workflow_json, data) queue: asyncio.Queue = asyncio.Queue(maxsize=_MAX_QUEUE_SIZE) self._queues[workflow_id] = queue @@ -262,8 +299,6 @@ async def restart_workflow(self, workflow_id: str) -> Dict[str, Any]: "protocol": protocol, } - input_key = str(data.get("inputKey") or "syslog_message") - # Spin up a fixed worker pool: exactly _MAX_CONCURRENT_EXECUTIONS # coroutines drain the queue. pending tasks cannot exceed this number, # which is the actual backpressure invariant we want. @@ -271,7 +306,7 @@ async def restart_workflow(self, workflow_id: str) -> Dict[str, Any]: for i in range(_MAX_CONCURRENT_EXECUTIONS): workers.append( asyncio.create_task( - self._worker_loop(workflow_id, workflow_json, input_key, queue, abort), + self._worker_loop(workflow_id, workflow_json, trigger, queue, abort), name=f"syslog-worker-{workflow_id}-{i}", ) ) @@ -417,7 +452,7 @@ async def _worker_loop( self, workflow_id: str, workflow_json: Any, - input_key: str, + trigger: TriggerDefinition, queue: asyncio.Queue, abort: asyncio.Event, ) -> None: @@ -435,7 +470,14 @@ async def _worker_loop( except asyncio.CancelledError: return try: - await self._trigger_workflow(workflow_id, workflow_json, msg, input_key) + await self._trigger_workflow( + workflow_id, + workflow_json, + msg, + next(iter(trigger.mapping or {}), "syslog_message"), + trigger=trigger, + source=f"{(trigger.source or {}).get('protocol', 'udp')}://{(trigger.source or {}).get('host', '0.0.0.0')}:{(trigger.source or {}).get('port', 5140)}", + ) except asyncio.CancelledError: return except Exception as exc: @@ -450,54 +492,99 @@ async def _trigger_workflow( workflow_json: Any, syslog_msg: dict, input_key: str, + *, + trigger: Optional[TriggerDefinition] = None, + source: Optional[str] = None, ) -> None: - inputs = {input_key: syslog_msg} - - exec_data = await create_execution_record( - workflow_id, - input_params={"_trigger": "syslog", **inputs}, + trigger = trigger or TriggerDefinition.model_validate( + { + "id": "syslog-default", + "type": "syslog", + "enabled": True, + "mapping": {input_key: "$.body"}, + } + ) + event = build_trigger_event( + workflow_id=workflow_id, + trigger=trigger, + body=syslog_msg, + raw=syslog_msg, + source=source or "syslog", + delivery_id=f"syslog-{uuid.uuid4().hex}", ) - exec_id = exec_data["id"] - start_time = time.time() - try: - result = await asyncio.to_thread( - run_workflow, - workflow=workflow_json, - inputs=inputs, - trace=False, - ) - status, error_msg = resolve_execution_outcome(result) - duration = time.time() - start_time - exec_data.update({ - "status": status, - "outputResults": compact_outputs_for_storage(result.outputs), - "finishedAt": int(time.time() * 1000), - "duration": duration, - "errorMessage": error_msg, - "executionLog": compact_history_for_storage(result.history), - "currentNodeId": result.last_node_id, - "currentPhase": status, - "currentStepIndex": result.steps, - }) - except Exception as exc: - duration = time.time() - start_time - log.error( - "syslog.workflow_run_failed", - {"workflow_id": workflow_id, "exec_id": exec_id, "error": str(exc)}, + async def _executor(mapped_inputs: Dict[str, Any]) -> Dict[str, Any]: + summarized_inputs = {"_trigger": trigger.type} + summarized_inputs.update(mapped_inputs) + + exec_data = await create_execution_record( + workflow_id, + input_params=summarized_inputs, ) - exec_data.update({ - "status": "error", - "errorMessage": str(exc), - "finishedAt": int(time.time() * 1000), - "duration": duration, - "currentPhase": "error", - }) - finally: + exec_id = exec_data["id"] + start_time = time.time() + trigger_meta = mapped_inputs.get("_flocks", {}).get("trigger", {}) try: - await record_execution_result(workflow_id, exec_id, exec_data) + result = await asyncio.to_thread( + run_workflow, + workflow=workflow_json, + inputs=mapped_inputs, + trace=False, + ) + status, error_msg = resolve_execution_outcome(result) + duration = time.time() - start_time + exec_data.update({ + "status": status, + "outputResults": compact_outputs_for_storage(result.outputs), + "finishedAt": int(time.time() * 1000), + "duration": duration, + "errorMessage": error_msg, + "executionLog": compact_history_for_storage(result.history), + "currentNodeId": result.last_node_id, + "currentPhase": status, + "currentStepIndex": result.steps, + "triggerId": trigger.id, + "triggerType": trigger.type, + "deliveryId": trigger_meta.get("deliveryId"), + "attempt": trigger_meta.get("attempt"), + "triggerSource": trigger_meta.get("source"), + }) except Exception as exc: - log.warning("syslog.exec_record_failed", {"exec_id": exec_id, "error": str(exc)}) + duration = time.time() - start_time + log.error( + "syslog.workflow_run_failed", + {"workflow_id": workflow_id, "exec_id": exec_id, "error": str(exc)}, + ) + exec_data.update({ + "status": "error", + "errorMessage": str(exc), + "finishedAt": int(time.time() * 1000), + "duration": duration, + "currentPhase": "error", + "triggerId": trigger.id, + "triggerType": trigger.type, + "deliveryId": trigger_meta.get("deliveryId"), + "attempt": trigger_meta.get("attempt"), + "triggerSource": trigger_meta.get("source"), + }) + finally: + try: + await record_execution_result(workflow_id, exec_id, exec_data) + except Exception as exc: + log.warning("syslog.exec_record_failed", {"exec_id": exec_id, "error": str(exc)}) + return exec_data + + try: + await self._dispatcher.dispatch( + trigger=trigger, + event=event, + executor=_executor, + ) + except TriggerDispatchError as exc: + log.warning( + "syslog.trigger_dispatch_failed", + {"workflow_id": workflow_id, "trigger_id": trigger.id, "error": str(exc)}, + ) default_manager = SyslogManager() diff --git a/flocks/plugin/loader.py b/flocks/plugin/loader.py index 757112260..0c1c897e2 100644 --- a/flocks/plugin/loader.py +++ b/flocks/plugin/loader.py @@ -173,7 +173,16 @@ class ExtensionPoint: E.g. ``frozenset({"mcp", "generated"})`` — these are managed by dedicated subsystems rather than the generic PluginLoader.""" + load_once: bool = False + """When True, skip later ``load_all`` passes after this extension has loaded. + + Stateful extension points, such as channels, own runtime connections and + callbacks on their plugin instances. Re-importing those modules can create + replacement instances that have not been started. + """ + _seen_keys: Set[str] = field(default_factory=set, repr=False) + _loaded: bool = field(default=False, repr=False) class PluginLoader: @@ -225,6 +234,15 @@ def load_all( project_plugin_root = project_dir / ".flocks" / "plugins" for ext in cls._extension_points.values(): + if ext.load_once and ext._loaded: + log.debug( + "plugin.load_all.skip_load_once", + { + "attr": ext.attr_name, + }, + ) + continue + ext._seen_keys = set() # 1. User-level plugin subdirectory (~/.flocks/plugins/{subdir}/) @@ -269,6 +287,9 @@ def load_all( if extra_sources: cls._load_sources_for_ext(ext, extra_sources, project_dir) + if ext.load_once: + ext._loaded = True + # 4. Installed package entry-points cls._load_entry_points() @@ -299,6 +320,8 @@ def _collecting_consumer(items: List[Any], source: str) -> None: ext._seen_keys = set() try: cls._load_sources_for_ext(ext, sources, base_dir) + if ext.load_once and sources: + ext._loaded = True finally: ext.consumer = original_consumer return collected diff --git a/flocks/provider/catalog.json b/flocks/provider/catalog.json index 182654157..b2a02615d 100644 --- a/flocks/provider/catalog.json +++ b/flocks/provider/catalog.json @@ -1055,6 +1055,54 @@ "output": 16.0, "currency": "CNY" } + }, + "deepseek-v4-flash": { + "name": "DeepSeek V4 Flash", + "family": "deepseek-v4", + "capabilities": { + "supports_tools": true, + "supports_reasoning": true, + "interleaved": { + "field": "reasoning_content", + "echo": "tool_calls", + "placeholder": " ", + "cross_provider_policy": "placeholder" + }, + "supports_streaming": true + }, + "limits": { + "context_window": 1000000, + "max_output_tokens": 384000 + }, + "pricing": { + "input": 1.0, + "output": 2.0, + "currency": "CNY" + } + }, + "deepseek-v4-pro": { + "name": "DeepSeek V4 Pro", + "family": "deepseek-v4", + "capabilities": { + "supports_tools": true, + "supports_reasoning": true, + "interleaved": { + "field": "reasoning_content", + "echo": "tool_calls", + "placeholder": " ", + "cross_provider_policy": "placeholder" + }, + "supports_streaming": true + }, + "limits": { + "context_window": 1000000, + "max_output_tokens": 384000 + }, + "pricing": { + "input": 3.0, + "output": 9.0, + "currency": "CNY" + } } } }, diff --git a/flocks/provider/interleaved.py b/flocks/provider/interleaved.py index aebadeec2..ad1b869b5 100644 --- a/flocks/provider/interleaved.py +++ b/flocks/provider/interleaved.py @@ -34,11 +34,12 @@ } _STRICT_REASONING_CONTENT_TOKENS = ( - "deepseek-reasoner", - "deepseek-r1", + "deepseek-v3", "deepseek-v4", "deepseek-v4-pro", "deepseek-v4-flash", + "deepseek-reasoner", + "deepseek-r1", "reasoner", "r1-0528", "kimi-k2.5", diff --git a/flocks/provider/model_catalog.py b/flocks/provider/model_catalog.py index d87cf18ac..9ac6407cb 100644 --- a/flocks/provider/model_catalog.py +++ b/flocks/provider/model_catalog.py @@ -132,14 +132,14 @@ def _parse_model_definitions( features.append(ModelFeature.TOOL_CALL) if caps_raw.get("supports_vision"): features.append(ModelFeature.VISION) - if caps_raw.get("supports_reasoning"): + if caps_raw.get("supports_reasoning", True): features.append(ModelFeature.REASONING) capabilities = ModelCapabilitiesV2( features=features, supports_tools=caps_raw.get("supports_tools", False), supports_vision=caps_raw.get("supports_vision", False), - supports_reasoning=caps_raw.get("supports_reasoning", False), + supports_reasoning=caps_raw.get("supports_reasoning", True), interleaved=caps_raw.get("interleaved"), supports_streaming=caps_raw.get("supports_streaming", True), ) diff --git a/flocks/provider/options.py b/flocks/provider/options.py index bd69f2ec2..d3222f979 100644 --- a/flocks/provider/options.py +++ b/flocks/provider/options.py @@ -13,6 +13,7 @@ from flocks.provider.interleaved import ( REASONING_TRANSPORT_ANTHROPIC_MESSAGES, + REASONING_TRANSPORT_GENERIC_CHAT, resolve_interleaved_capability, resolve_reasoning_transport, ) @@ -27,14 +28,10 @@ DEFAULT_THINKING_BUDGET = 16000 DEFAULT_OUTPUT_BUFFER = 8192 -_ENABLE_THINKING_EXTRA_BODY_TOKENS = ( - "qwen3", - "qwq", - "qwen-max", - "kimi", - "k2-thinking", - "mimo", -) +_GENERIC_CHAT_REASONING_EXTRA_BODY_KEYS = { + "reasoning_content": "enable_thinking", + "reasoning_details": "reasoning_split", +} def _coerce_optional_bool(value: Any) -> Optional[bool]: @@ -178,9 +175,63 @@ def _resolve_reasoning_transport(provider_id: str, model_id: str) -> str: return transport -def _needs_enable_thinking_extra_body(model_id: str) -> bool: - lowered = model_id.lower() - return any(token in lowered for token in _ENABLE_THINKING_EXTRA_BODY_TOKENS) +def _build_generic_chat_extra_body( + provider_id: str, + model_id: str, + interleaved_capability: Optional[Dict[str, Any]], + reasoning_enabled: Optional[bool], +) -> Optional[Dict[str, Any]]: + """Build OpenAI-compatible reasoning params for the active replay field.""" + provider_lower = provider_id.lower() + model_lower = model_id.lower() + enabled = reasoning_enabled is not False + + if "deepseek" in model_lower or provider_lower == "deepseek": + return { + "thinking": ( + {"type": "enabled"} + if enabled + else {"type": "disabled"} + ) + } + + if "glm" in model_lower or provider_lower == "zhipu": + return { + "thinking": ( + {"type": "enabled", "clear_thinking": False} + if enabled + else {"type": "disabled"} + ) + } + + if "mimo" in model_lower: + return { + "thinking": ( + {"type": "enabled"} + if enabled + else {"type": "disabled"} + ) + } + + if "kimi" in model_lower: + return { + "thinking": ( + {"type": "enabled"} + if enabled + else {"type": "disabled"} + ) + } + + if isinstance(interleaved_capability, dict): + field = interleaved_capability.get("field") + key = _GENERIC_CHAT_REASONING_EXTRA_BODY_KEYS.get(field) + if key: + return {key: enabled} + + if reasoning_enabled is True: + return {"enable_thinking": True} + + return None def build_provider_options( @@ -265,23 +316,29 @@ def build_provider_options( if reasoning_enabled is not False: options["thinkingLevel"] = "high" - # -- Qwen reasoning (ThreatBook-hosted or Alibaba DashScope) ------------- + # -- Generic-chat (OpenAI-compat) interleaved thinking -------------------- + # The Anthropic branch above handles ``anthropic_messages`` transport. + # Generic-chat endpoints expose provider-specific extra_body toggles: + # most reasoning_content models use enable_thinking, while MiniMax's + # OpenAI-compatible interleaved format uses reasoning_split so the model + # returns reasoning_details that can be replayed in later tool turns. elif ( - provider_id in ("threatbook-cn-llm", "threatbook-io-llm", "alibaba", "moonshot") - or ( - interleaved_enabled - and _needs_enable_thinking_extra_body(model_id) - and provider_id not in {"openai", "anthropic", "google"} - ) + (interleaved_enabled or reasoning_enabled is True) + and reasoning_transport == REASONING_TRANSPORT_GENERIC_CHAT ): - if "qwen" in model_lower or "qwq" in model_lower: - options["extra_body"] = { - "enable_thinking": True if reasoning_enabled is None else reasoning_enabled - } - elif any(token in model_lower for token in ("kimi", "k2-thinking", "mimo")): - options["extra_body"] = { - "enable_thinking": True if reasoning_enabled is None else reasoning_enabled - } + extra_body = _build_generic_chat_extra_body( + provider_id, + model_id, + interleaved_capability, + reasoning_enabled, + ) + if extra_body: + options["extra_body"] = extra_body + log.debug("options.thinking_params.resolved", { + "provider_id": provider_id, + "model_id": model_id, + "extra_body_keys": list(options["extra_body"].keys()), + }) # -- max_tokens fallback from model config ------------------------------ if resolve_max_tokens and "max_tokens" not in options: diff --git a/flocks/provider/provider.py b/flocks/provider/provider.py index d42ce6023..180d4e97c 100644 --- a/flocks/provider/provider.py +++ b/flocks/provider/provider.py @@ -111,7 +111,7 @@ class ModelCapabilities(BaseModel): supports_streaming: bool = True supports_tools: bool = True supports_vision: bool = False - supports_reasoning: bool = False + supports_reasoning: bool = True interleaved: Optional[Dict[str, Any]] = None max_tokens: Optional[int] = None context_window: Optional[int] = None @@ -848,7 +848,7 @@ async def apply_config(cls, config: Optional[Any] = None, provider_id: Optional[ supports_streaming=model_dict.get("supports_streaming", True), supports_tools=model_dict.get("supports_tools", True), supports_vision=model_dict.get("supports_vision", False), - supports_reasoning=model_dict.get("supports_reasoning", False), + supports_reasoning=model_dict.get("supports_reasoning", True), interleaved=model_dict.get("interleaved"), max_tokens=model_dict.get("max_output_tokens") or model_dict.get("max_tokens"), context_window=model_dict.get("context_window"), @@ -1216,7 +1216,7 @@ def _build_model_definition(self, model: "ModelInfo") -> "ModelDefinition": supports_streaming=model.capabilities.supports_streaming, supports_tools=model.capabilities.supports_tools, supports_vision=model.capabilities.supports_vision, - supports_reasoning=getattr(model.capabilities, "supports_reasoning", False), + supports_reasoning=getattr(model.capabilities, "supports_reasoning", True), interleaved=getattr(model.capabilities, "interleaved", None), ), limits=ModelLimits( @@ -1250,7 +1250,7 @@ def _apply_config_overrides(self, catalog_def: "ModelDefinition", model: "ModelI never touched keep their richer catalog values. This avoids e.g. a catalog ``supports_reasoning=True`` being silently reset to ``False`` by a default. """ - from flocks.provider.types import PriceConfig + from flocks.provider.types import ModelFeature, PriceConfig overridden = catalog_def.model_copy(deep=True) keys = getattr(model, "_explicit_keys", set()) @@ -1267,9 +1267,19 @@ def _apply_config_overrides(self, catalog_def: "ModelDefinition", model: "ModelI if "supports_vision" in keys: overridden.capabilities.supports_vision = model.capabilities.supports_vision if "supports_reasoning" in keys: - overridden.capabilities.supports_reasoning = getattr( - model.capabilities, "supports_reasoning", False + supports_reasoning = getattr( + model.capabilities, "supports_reasoning", True ) + overridden.capabilities.supports_reasoning = supports_reasoning + if supports_reasoning: + if ModelFeature.REASONING not in overridden.capabilities.features: + overridden.capabilities.features.append(ModelFeature.REASONING) + else: + overridden.capabilities.features = [ + feature + for feature in overridden.capabilities.features + if feature != ModelFeature.REASONING and feature != ModelFeature.REASONING.value + ] if "interleaved" in keys: overridden.capabilities.interleaved = getattr( model.capabilities, "interleaved", None diff --git a/flocks/provider/sdk/openai_compatible.py b/flocks/provider/sdk/openai_compatible.py index 5e86d9592..c38183e20 100644 --- a/flocks/provider/sdk/openai_compatible.py +++ b/flocks/provider/sdk/openai_compatible.py @@ -182,19 +182,24 @@ async def chat( max_tokens = kwargs.get("max_tokens") tools = kwargs.get("tools") thinking = kwargs.get("thinking") - + # Make request request_params = { "model": model_id, "messages": formatted_messages, } + # See the streaming chat_stream() counterpart for the rationale; the + # non-streaming path had the same extra_body-swallow bug. + extra_body = dict(kwargs.get("extra_body") or {}) if thinking: - request_params["extra_body"] = {"thinking": thinking} + extra_body["thinking"] = thinking else: temperature = kwargs.get("temperature") if temperature is not None: request_params["temperature"] = temperature + if extra_body: + request_params["extra_body"] = extra_body if max_tokens: request_params["max_tokens"] = max_tokens @@ -242,7 +247,7 @@ async def chat_stream( max_tokens = kwargs.get("max_tokens") tools = kwargs.get("tools") thinking = kwargs.get("thinking") - + # Make streaming request request_params = { "model": model_id, @@ -251,12 +256,23 @@ async def chat_stream( "stream_options": {"include_usage": True}, } + # extra_body assembly: mirror openai_base.py:905-913. Caller-supplied + # extra_body (e.g. ``enable_thinking`` produced by + # ``build_provider_options`` for DashScope-style endpoints) MUST be + # forwarded — the old code dropped it whenever ``thinking`` was None, + # silently disabling thinking for user-configured openai-compatible + # endpoints that set ``default_parameters.enable_thinking`` in + # flocks.json. ``thinking`` is merged in last so it can override a + # caller-supplied extra_body.thinking if both are set. + extra_body = dict(kwargs.get("extra_body") or {}) if thinking: - request_params["extra_body"] = {"thinking": thinking} + extra_body["thinking"] = thinking else: temperature = kwargs.get("temperature") if temperature is not None: request_params["temperature"] = temperature + if extra_body: + request_params["extra_body"] = extra_body if max_tokens: request_params["max_tokens"] = max_tokens diff --git a/flocks/provider/types.py b/flocks/provider/types.py index 6754be636..c7233e370 100644 --- a/flocks/provider/types.py +++ b/flocks/provider/types.py @@ -244,7 +244,7 @@ class ModelCapabilitiesV2(BaseModel): supports_streaming: bool = True supports_tools: bool = True supports_vision: bool = False - supports_reasoning: bool = False + supports_reasoning: bool = True interleaved: Optional[Dict[str, Any]] = None supports_temperature: bool = True supports_json_mode: bool = False diff --git a/flocks/server/app.py b/flocks/server/app.py index 402a75a9a..95f7a6f5e 100644 --- a/flocks/server/app.py +++ b/flocks/server/app.py @@ -182,6 +182,20 @@ async def lifespan(app: FastAPI): await _run_startup_phase(log, "storage.init", Storage.init) log.info("storage.initialized") + async def _recover_orphan_tool_parts() -> None: + from flocks.session.orphan_tools import abort_all_orphan_running_parts + + repaired = await abort_all_orphan_running_parts() + if repaired: + log.info("session.orphan_tools.recovered", {"count": repaired}) + + _schedule_startup_phase( + app, + log, + "session.recover_orphan_tools", + _recover_orphan_tool_parts, + ) + # Ensure default device room exists, then migrate legacy device API # configs from flocks.json → device_integrations table. try: @@ -381,74 +395,55 @@ def _start_tool_watcher() -> None: except Exception as e: log.warning("tool.watcher.init_failed", {"error": str(e)}) - # Start Channel Gateway (connect enabled IM channels) + # Start user-defined pages watcher (auto-build user custom pages) try: - from flocks.channel.gateway.manager import default_manager + from flocks.user_defined_pages.bootstrap import reconcile_user_defined_pages + from flocks.user_defined_pages.watcher import set_event_loop, start_watcher - async def _start_channel_gateway() -> None: - await default_manager.start_all() - log.info("channel.gateway.started") + set_event_loop(asyncio.get_running_loop()) - _schedule_startup_phase(app, log, "channel.gateway.start", _start_channel_gateway) - except Exception as e: - log.warning("channel.gateway.start_failed", {"error": str(e)}) + _schedule_startup_phase( + app, + log, + "user_defined_pages.bootstrap", + reconcile_user_defined_pages, + ) - # Start syslog listeners for workflows with syslog enabled. - # Use a background task with a short delay so the main startup path is not - # blocked and to break the crash-restart loop where an immediate syslog - # flood would bring the server back down before it is fully ready. - try: - from flocks.ingest.syslog.manager import default_manager as default_syslog_manager + def _start_user_defined_pages_watcher() -> None: + start_watcher() + log.info("user_defined_pages.watcher.initialized") - async def _delayed_syslog_start() -> None: - # Wait for storage and tool registry to be fully initialised before - # resuming syslog listeners. - await asyncio.sleep(3) - try: - await default_syslog_manager.start_all() - log.info("syslog.manager.started") - except Exception as exc: - log.warning("syslog.manager.start_failed", {"error": str(exc)}) - - _schedule_startup_phase(app, log, "syslog.manager.start", _delayed_syslog_start) + _schedule_startup_phase(app, log, "user_defined_pages.watcher.start", _start_user_defined_pages_watcher) except Exception as e: - log.warning("syslog.manager.start_failed", {"error": str(e)}) + log.warning("user_defined_pages.watcher.init_failed", {"error": str(e)}) - # Start Kafka consumers for workflows with kafka input enabled. - # Mirrors the syslog startup: a short delayed background task keeps the main - # startup path unblocked and avoids a crash-restart loop if a broker is down. + # Start Channel Gateway (connect enabled IM channels) try: - from flocks.ingest.kafka.manager import default_manager as default_kafka_manager + from flocks.channel.gateway.manager import default_manager - async def _delayed_kafka_start() -> None: - await asyncio.sleep(3) - try: - await default_kafka_manager.start_all() - log.info("kafka.manager.started") - except Exception as exc: - log.warning("kafka.manager.start_failed", {"error": str(exc)}) + async def _start_channel_gateway() -> None: + await default_manager.start_all() + log.info("channel.gateway.started") - _schedule_startup_phase(app, log, "kafka.manager.start", _delayed_kafka_start) + _schedule_startup_phase(app, log, "channel.gateway.start", _start_channel_gateway) except Exception as e: - log.warning("kafka.manager.start_failed", {"error": str(e)}) + log.warning("channel.gateway.start_failed", {"error": str(e)}) - # Start workflow pollers for workflows with poller enabled. - # Mirrors Kafka/syslog startup so persistent slow-path workflows resume - # automatically without delaying server readiness. + # Start the unified workflow trigger runtime after the server is ready. try: - from flocks.workflow.poller_manager import default_manager as default_poller_manager + from flocks.workflow.triggers.runtime import default_runtime as default_trigger_runtime - async def _delayed_poller_start() -> None: + async def _delayed_trigger_runtime_start() -> None: await asyncio.sleep(3) try: - await default_poller_manager.start_all() - log.info("workflow.poller.started") + await default_trigger_runtime.start_all() + log.info("workflow.trigger_runtime.started") except Exception as exc: - log.warning("workflow.poller.start_failed", {"error": str(exc)}) + log.warning("workflow.trigger_runtime.start_failed", {"error": str(exc)}) - _schedule_startup_phase(app, log, "workflow.poller.start", _delayed_poller_start) + _schedule_startup_phase(app, log, "workflow.trigger_runtime.start", _delayed_trigger_runtime_start) except Exception as e: - log.warning("workflow.poller.start_failed", {"error": str(e)}) + log.warning("workflow.trigger_runtime.start_failed", {"error": str(e)}) try: from flocks.updater.updater import recover_upgrade_state @@ -513,23 +508,14 @@ async def _delayed_poller_start() -> None: except Exception as e: log.warning("channel.gateway.stop_failed", {"error": str(e)}) - # Stop syslog listeners - try: - from flocks.ingest.syslog.manager import default_manager as default_syslog_manager - - await default_syslog_manager.stop_all() - log.info("syslog.manager.stopped") - except Exception as e: - log.warning("syslog.manager.stop_failed", {"error": str(e)}) - - # Stop Kafka consumers + # Stop the unified workflow trigger runtime. try: - from flocks.ingest.kafka.manager import default_manager as default_kafka_manager + from flocks.workflow.triggers.runtime import default_runtime as default_trigger_runtime - await default_kafka_manager.stop_all() - log.info("kafka.manager.stopped") + await default_trigger_runtime.stop_all() + log.info("workflow.trigger_runtime.stopped") except Exception as e: - log.warning("kafka.manager.stop_failed", {"error": str(e)}) + log.warning("workflow.trigger_runtime.stop_failed", {"error": str(e)}) # Stop Task Center try: @@ -548,6 +534,13 @@ async def _delayed_poller_start() -> None: except Exception as e: log.warning("skill.watcher.stop_failed", {"error": str(e)}) + # Stop user-defined pages watcher + try: + from flocks.user_defined_pages.watcher import stop_watcher + stop_watcher() + except Exception as e: + log.warning("user_defined_pages.watcher.stop_failed", {"error": str(e)}) + # Shutdown MCP connections try: from flocks.mcp import MCP @@ -986,7 +979,7 @@ async def general_exception_handler(request: Request, exc: Exception): # P3: TUI control routes for remote TUI control from flocks.server.routes.tui import router as tui_router # WebUI: Workflow routes -from flocks.server.routes.workflow import router as workflow_router +from flocks.server.routes.workflow import router as workflow_router, webhook_router as workflow_webhook_router # WebUI: Skill & Command routes from flocks.server.routes.skill import router as skill_router from flocks.server.routes.hub import router as hub_router @@ -1015,6 +1008,7 @@ async def general_exception_handler(request: Request, exc: Exception): from flocks.server.routes.notifications import router as notifications_router from flocks.server.routes.device import router as device_router from flocks.server.routes.console_upgrade import router as console_upgrade_router +from flocks.server.routes.user_defined_pages import router as user_defined_pages_router # Original routes with /api/ prefix app.include_router(health_router, prefix="/api", tags=["Health"]) app.include_router(session_router, prefix="/api/session", tags=["Session"]) @@ -1036,6 +1030,7 @@ async def general_exception_handler(request: Request, exc: Exception): app.include_router(mcp_router, prefix="/api/mcp", tags=["MCP"]) # WebUI: Workflow routes app.include_router(workflow_router, prefix="/api", tags=["Workflow"]) +app.include_router(workflow_webhook_router, tags=["WorkflowWebhook"]) # WebUI: Skill & Command routes app.include_router(skill_router, prefix="/api", tags=["Skill"]) # WebUI: Hub routes @@ -1073,6 +1068,7 @@ async def general_exception_handler(request: Request, exc: Exception): # Device integration (named instances, SQL-backed) app.include_router(device_router, prefix="/api/devices", tags=["Device"]) app.include_router(console_upgrade_router, prefix="/api/console", tags=["ConsoleUpgrade"]) +app.include_router(user_defined_pages_router, prefix="/api", tags=["UserDefinedPages"]) # ============================================================ # TUI Compatible Routes (without /api/ prefix) diff --git a/flocks/server/auth.py b/flocks/server/auth.py index 5f6422908..654f7d381 100644 --- a/flocks/server/auth.py +++ b/flocks/server/auth.py @@ -67,6 +67,7 @@ # entries that touch user data without a per-request integrity check. PUBLIC_PATH_REGEXES = ( re.compile(r"^/(?:api/)?channel/[^/]+/webhook/?$"), + re.compile(r"^/webhook/workflows/[^/]+/[^/]+/?$"), ) diff --git a/flocks/server/routes/agent.py b/flocks/server/routes/agent.py index e8b30da39..e36583253 100644 --- a/flocks/server/routes/agent.py +++ b/flocks/server/routes/agent.py @@ -61,6 +61,7 @@ class AgentResponse(BaseModel): Includes required 'permission' and 'options' fields. """ name: str + nameCn: Optional[str] = None description: Optional[str] = None descriptionCn: Optional[str] = None mode: str = "primary" @@ -114,6 +115,7 @@ def agent_to_response( return AgentResponse( name=agent.name, + nameCn=agent.name_cn, description=agent.description, descriptionCn=agent.description_cn, mode=agent.mode, @@ -146,6 +148,7 @@ def _agent_data_to_info(agent_data: Dict[str, Any]) -> AgentInfoModel: delegatable = mode != "primary" return AgentInfoModel( name=agent_data["name"], + name_cn=agent_data.get("name_cn") or agent_data.get("nameCn"), description=agent_data.get("description") or "", description_cn=agent_data.get("description_cn") or agent_data.get("descriptionCn"), prompt=agent_data.get("prompt") or "", @@ -172,6 +175,7 @@ def _custom_agent_data_to_response(agent_data: Dict[str, Any]) -> AgentResponse: delegatable = mode != "primary" return AgentResponse( name=agent_data["name"], + nameCn=agent_data.get("name_cn") or agent_data.get("nameCn"), description=agent_data.get("description"), descriptionCn=agent_data.get("description_cn") or agent_data.get("descriptionCn"), prompt=agent_data.get("prompt"), @@ -338,6 +342,7 @@ async def get_agent_prompt(name: str): class AgentCreateRequest(BaseModel): """Request to create a custom agent""" name: str = Field(..., description="Agent name") + nameCn: Optional[str] = Field(None, description="Chinese UI agent name") description: Optional[str] = Field(None, description="Agent description (English; used for delegation)") descriptionCn: Optional[str] = Field(None, description="Chinese UI description") prompt: str = Field(..., description="System prompt") @@ -352,6 +357,7 @@ class AgentCreateRequest(BaseModel): class AgentUpdateRequest(BaseModel): """Request to update a custom agent""" + nameCn: Optional[str] = Field(None, description="Chinese UI agent name") description: Optional[str] = Field(None, description="Agent description (English; used for delegation)") descriptionCn: Optional[str] = Field(None, description="Chinese UI description") prompt: Optional[str] = Field(None, description="System prompt") @@ -390,6 +396,7 @@ async def create_agent(req: AgentCreateRequest): agent_data: Dict[str, Any] = { "name": req.name, + "name_cn": req.nameCn, "description": req.description, "description_cn": req.descriptionCn, "prompt": req.prompt, @@ -436,6 +443,8 @@ async def update_agent(name: str, req: AgentUpdateRequest): # YAML agents may also have an overlay entry (skills/tools only) which # lacks the "name" key; those should fall through to the YAML path. if agent_data is not None and agent_data.get("name"): + if req.nameCn is not None: + agent_data["name_cn"] = req.nameCn if req.description is not None: agent_data["description"] = req.description if req.descriptionCn is not None: @@ -468,6 +477,8 @@ async def update_agent(name: str, req: AgentUpdateRequest): # --- Fall back to YAML plugin agent --- if find_yaml_agent(name) is not None: updates: Dict[str, Any] = {} + if req.nameCn is not None: + updates["name_cn"] = req.nameCn if req.description is not None: updates["description"] = req.description if req.descriptionCn is not None: @@ -500,6 +511,8 @@ async def update_agent(name: str, req: AgentUpdateRequest): # Sync: apply updates to the in-memory AgentInfo cache agent = await Agent.get(name) if agent: + if req.nameCn is not None: + agent.name_cn = req.nameCn if req.description is not None: agent.description = req.description if req.descriptionCn is not None: diff --git a/flocks/server/routes/custom_provider.py b/flocks/server/routes/custom_provider.py index 475c21c11..19f1c361a 100644 --- a/flocks/server/routes/custom_provider.py +++ b/flocks/server/routes/custom_provider.py @@ -72,7 +72,7 @@ class CreateModelReq(BaseModel): supports_vision: bool = False supports_tools: bool = True supports_streaming: bool = True - supports_reasoning: bool = False + supports_reasoning: bool = True input_price: float = Field(0.0, ge=0) output_price: float = Field(0.0, ge=0) currency: str = "USD" @@ -403,7 +403,7 @@ async def load_custom_providers_on_startup(): supports_streaming=mcfg.get("supports_streaming", True), supports_tools=mcfg.get("supports_tools", True), supports_vision=mcfg.get("supports_vision", False), - supports_reasoning=mcfg.get("supports_reasoning", False), + supports_reasoning=mcfg.get("supports_reasoning", True), max_tokens=mcfg.get("max_output_tokens", 4096), context_window=mcfg.get("context_window", 128000), ), diff --git a/flocks/server/routes/device.py b/flocks/server/routes/device.py index 49ca185a9..3fbbae766 100644 --- a/flocks/server/routes/device.py +++ b/flocks/server/routes/device.py @@ -5,52 +5,106 @@ """ from __future__ import annotations -import json -import time -import uuid -from typing import List, Optional +from typing import Any, List, Optional import aiosqlite -import httpx -from fastapi import APIRouter, HTTPException, status as http_status +from fastapi import APIRouter, HTTPException, Request, status as http_status from pydantic import BaseModel, Field +from flocks.audit import emit_audit_event +from flocks.server.auth import get_request_ip, get_request_user_agent, require_user from flocks.tool.device import ( DEFAULT_GROUP_ID, MULTI_GROUP_ENABLED, DeviceGroup, DeviceGroupCreate, DeviceGroupUpdate, + DeviceCredentialResponse, DeviceIntegration, DeviceIntegrationCreate, DeviceIntegrationUpdate, + DeviceTemplate, + DeviceTestRequest, DeviceTestResult, + CustomDeviceTemplateCreate, +) +from flocks.tool.device.intake import ( + DeviceNotFoundError, + create_device, + delete_device, + ensure_user_device_instances, + test_device, + update_device, +) +from flocks.tool.device.plugin_index import ( + create_custom_device_template, + list_device_templates, ) -from flocks.tool.device.secrets import delete_secrets, persist_fields, resolve_for_runtime from flocks.tool.device.store import ( create_group, - delete_device_row, delete_device_tool_setting, delete_group, fetch_device, get_group, - group_exists, - insert_device, list_device_tool_settings, list_devices, list_groups, - record_test_result, row_to_device, set_device_tool_enabled, - storage_key_to_service_id, - update_device_row, update_group, ) -from flocks.tool.device.sync import sync_service_tool_state router = APIRouter() +async def _emit_device_audit_fallback(event_type: str, payload: dict[str, Any]) -> None: + """Persist device audit even when the default sink is still a no-op.""" + try: + from flocks.audit import NullAuditSink, get_sink + + sink_cls = get_sink() + if sink_cls is not NullAuditSink: + return + except Exception: + return + + try: + from flockspro.audit.service import AuditEvent + from flockspro.audit.sinks import SqliteAuditSink + except Exception: + # OSS or flockspro not installed: nothing to persist. + return + + failed = bool(payload.get("error") or payload.get("reason")) + event = AuditEvent( + event_type=event_type, + category="device", + action="credentials_reveal", + status="error" if failed else "ok", + result="failed" if failed else "success", + user_id=str(payload.get("user_id")) if payload.get("user_id") else None, + user_name=str(payload.get("username")) if payload.get("username") else None, + resource_type="device", + resource_id=str(payload.get("device_id")) if payload.get("device_id") else None, + ip=str(payload.get("ip")) if payload.get("ip") else None, + payload=payload, + metadata=payload, + ) + await SqliteAuditSink().write(event) + + +async def _emit_device_audit(event_type: str, payload: dict[str, Any]) -> None: + try: + await emit_audit_event(event_type, payload) + except Exception: + # Audit failures must not block credential reveal. + pass + try: + await _emit_device_audit_fallback(event_type, payload) + except Exception: + pass + + # =========================================================================== # Group routes # =========================================================================== @@ -119,10 +173,30 @@ async def route_delete_group(group_id: str): # =========================================================================== @router.get("", response_model=List[DeviceIntegration]) -async def route_list_devices(group_id: Optional[str] = None): +async def route_list_devices(group_id: Optional[str] = None, refresh: bool = False): + await ensure_user_device_instances(refresh_templates=refresh) return await list_devices(group_id) +@router.get("/templates", response_model=List[DeviceTemplate]) +async def route_list_device_templates(refresh: bool = False): + return list_device_templates(refresh=refresh) + + +@router.post( + "/templates/custom", + response_model=DeviceTemplate, + status_code=http_status.HTTP_201_CREATED, +) +async def route_create_custom_device_template(body: CustomDeviceTemplateCreate): + try: + return create_custom_device_template(body) + except FileExistsError as exc: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=str(exc)) + except ValueError as exc: + raise HTTPException(status_code=http_status.HTTP_400_BAD_REQUEST, detail=str(exc)) + + @router.get("/{device_id}", response_model=DeviceIntegration) async def route_get_device(device_id: str): row = await fetch_device(device_id) @@ -131,101 +205,81 @@ async def route_get_device(device_id: str): return row_to_device(row) -@router.post("", response_model=DeviceIntegration, status_code=http_status.HTTP_201_CREATED) -async def route_create_device(body: DeviceIntegrationCreate): - name = body.name.strip() - storage_key = body.storage_key.strip() - if not name: - raise HTTPException(status_code=400, detail="name is required") - if not storage_key: - raise HTTPException(status_code=400, detail="storage_key is required") - - group_id = DEFAULT_GROUP_ID if not MULTI_GROUP_ENABLED else (body.group_id or DEFAULT_GROUP_ID) - if not await group_exists(group_id): - raise HTTPException(status_code=400, detail=f"Group '{group_id}' does not exist") - - service_id = (body.service_id or "").strip() or storage_key_to_service_id(storage_key) - device_id = str(uuid.uuid4()) - db_fields = persist_fields(device_id, storage_key, body.fields) - - await insert_device( - device_id=device_id, - group_id=group_id, - name=name, - storage_key=storage_key, - service_id=service_id, - enabled=body.enabled, - verify_ssl=body.verify_ssl, - db_fields=db_fields, +class DeviceCredentialRevealRequest(BaseModel): + field: Optional[str] = Field( + None, + description="Reveal only this credential key when provided.", ) - await sync_service_tool_state(service_id) - return await route_get_device(device_id) -@router.put("/{device_id}", response_model=DeviceIntegration) -async def route_update_device(device_id: str, body: DeviceIntegrationUpdate): +@router.post("/{device_id}/credentials", response_model=DeviceCredentialResponse) +async def route_get_device_credentials( + device_id: str, + request: Request, + body: Optional[DeviceCredentialRevealRequest] = None, +): + current_user = require_user(request) row = await fetch_device(device_id) if row is None: raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail="Device not found") + db_fields: dict = json.loads(row["fields"] or "{}") + requested_field = (body.field or "").strip() if body else "" + resolved_fields = resolve_for_runtime(db_fields) + if requested_field: + if requested_field not in resolved_fields: + raise HTTPException( + status_code=http_status.HTTP_404_NOT_FOUND, + detail=f"Credential field '{requested_field}' not found", + ) + response_fields = {requested_field: resolved_fields[requested_field]} + revealed_keys = [requested_field] + else: + response_fields = resolved_fields + revealed_keys = sorted(resolved_fields.keys()) + + # Record the reveal action without ever logging the plaintext values. + await _emit_device_audit( + "device.credentials_reveal", + { + "action": "credentials_reveal", + "actor_id": current_user.id, + "actor_name": current_user.username, + "user_id": current_user.id, + "username": current_user.username, + "device_id": device_id, + "storage_key": row["storage_key"], + "field_keys": revealed_keys, + "ip": get_request_ip(request), + "user_agent": get_request_user_agent(request), + }, + ) + return DeviceCredentialResponse(fields=response_fields) - prior_fields: dict = json.loads(row["fields"] or "{}") - - stripped_name = body.name.strip() if body.name else "" - new_name = stripped_name or row["name"] - new_enabled = body.enabled if body.enabled is not None else bool(row["enabled"]) - new_ssl = body.verify_ssl if body.verify_ssl is not None else bool(row["verify_ssl"]) - if body.group_id and MULTI_GROUP_ENABLED and body.group_id != row["group_id"]: - if not await group_exists(body.group_id): - raise HTTPException(status_code=400, detail=f"Group '{body.group_id}' does not exist") - new_group_id = body.group_id - else: - new_group_id = row["group_id"] or DEFAULT_GROUP_ID +@router.post("", response_model=DeviceIntegration, status_code=http_status.HTTP_201_CREATED) +async def route_create_device(body: DeviceIntegrationCreate): + try: + return await create_device(body) + except ValueError as exc: + raise HTTPException(status_code=http_status.HTTP_400_BAD_REQUEST, detail=str(exc)) - new_fields = ( - persist_fields(device_id, row["storage_key"], body.fields, prior_db_fields=prior_fields) - if body.fields is not None - else prior_fields - ) - await update_device_row( - device_id, - name=new_name, - group_id=new_group_id, - enabled=new_enabled, - verify_ssl=new_ssl, - db_fields=new_fields, - ) - # Recompute ``service_id`` from the row's ``storage_key`` instead of - # trusting the stored column. Rows created before the descriptor- - # aware ``storage_key_to_service_id`` fix may carry a too-greedy - # value (e.g. ``onesig`` instead of ``onesig_v2_5_3_D20250710_api``); - # using the column directly would route this sync to the wrong key - # bucket and leave ``api_services[storage_key].enabled`` stale, - # which in turn keeps tools wrongly exposed to the LLM. - await sync_service_tool_state(storage_key_to_service_id(row["storage_key"])) - return await route_get_device(device_id) +@router.put("/{device_id}", response_model=DeviceIntegration) +async def route_update_device(device_id: str, body: DeviceIntegrationUpdate): + try: + return await update_device(device_id, body) + except DeviceNotFoundError: + raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail="Device not found") + except ValueError as exc: + raise HTTPException(status_code=http_status.HTTP_400_BAD_REQUEST, detail=str(exc)) @router.delete("/{device_id}", status_code=http_status.HTTP_204_NO_CONTENT) async def route_delete_device(device_id: str): - row = await fetch_device(device_id) - if row is None: + try: + await delete_device(device_id) + except DeviceNotFoundError: raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail="Device not found") - # Capture storage_key BEFORE deletion: once the row is gone the DB query - # inside sync_service_tool_state can no longer see it, so if this was the - # last instance for that storage_key the tool would never get disabled. - storage_key: str = row["storage_key"] - # Always derive service_id from the live storage_key — see comment in - # ``route_update_device`` for why we don't trust the stored column. - service_id: str = storage_key_to_service_id(storage_key) - db_fields: dict = json.loads(row["fields"] or "{}") - - delete_secrets(device_id, db_fields) - await delete_device_row(device_id) - # Per-device tool settings are cleaned up automatically via - # ON DELETE CASCADE on the device_tool_settings table. - await sync_service_tool_state(service_id, deleted_storage_keys=[storage_key]) # =========================================================================== @@ -356,102 +410,9 @@ async def route_update_device_tool( ) -class DeviceTestRequest(BaseModel): - """Optional body for ``POST /devices/{id}/test``. - - All fields are optional. When supplied, they take precedence over the - persisted device row so the user can validate unsaved edits (e.g. flip - the SSL toggle in the form and re-test before clicking 保存). - """ - base_url: Optional[str] = Field(None, description="Override the persisted base_url for this probe only") - verify_ssl: Optional[bool] = Field(None, description="Override the persisted verify_ssl for this probe only") - - @router.post("/{device_id}/test", response_model=DeviceTestResult) async def route_test_device(device_id: str, body: Optional[DeviceTestRequest] = None): - """Connectivity test: GET on the device's ``base_url``. - - HTTP 4xx → reachable (success); HTTP 5xx / connect error / timeout → failure. - - Optionally accepts a JSON body with ``base_url`` / ``verify_ssl`` overrides - so the WebUI can probe with the form's current (unsaved) values. - """ - row = await fetch_device(device_id) - if row is None: - raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail="Device not found") - - db_fields: dict = json.loads(row["fields"] or "{}") - resolved = resolve_for_runtime(db_fields) - persisted_base_url = (resolved.get("base_url") or "").strip() - - # Form overrides take priority over the DB row so the toggle on screen is - # what gets used for the probe. - override_base_url = (body.base_url.strip() if body and body.base_url else "") - base_url = override_base_url or persisted_base_url - - # Some providers (e.g. Sangfor SIP) store host + port instead of base_url. - # Fall back to constructing the URL from those fields so the connectivity - # test works without requiring a separate base_url field. - if not base_url: - host = (resolved.get("host") or "").strip() - port = (resolved.get("port") or "").strip() - if host: - # If the operator already typed a scheme into the host field - # (e.g. "http://10.1.2.3"), respect it instead of double-prefixing. - has_scheme = "://" in host - if has_scheme: - base_url = f"{host}:{port}" if port else host - else: - base_url = f"https://{host}:{port}" if port else f"https://{host}" - - if not base_url: - return DeviceTestResult( - success=False, - message="未配置设备地址（base_url 或 host），请先填写", - ) - - if body is not None and body.verify_ssl is not None: - verify_ssl = bool(body.verify_ssl) - else: - verify_ssl = bool(row["verify_ssl"]) - - result = await _probe(base_url, verify_ssl=verify_ssl) - await record_test_result( - device_id, - success=result.success, - message=result.message, - latency_ms=result.latency_ms, - ) - return result - - -async def _probe(base_url: str, *, verify_ssl: bool) -> DeviceTestResult: - """Single HTTP GET probe; uniformly returns a DeviceTestResult.""" - start = time.monotonic() - - def elapsed() -> int: - return int((time.monotonic() - start) * 1000) - try: - async with httpx.AsyncClient(verify=verify_ssl, timeout=10.0) as client: - resp = await client.get(base_url) - ms = elapsed() - return DeviceTestResult( - success=resp.status_code < 500, - message=f"HTTP {resp.status_code}，延迟 {ms}ms", - latency_ms=ms, - ) - except httpx.ConnectError: - return DeviceTestResult( - success=False, - message=f"无法连接到 {base_url}，请检查地址是否正确", - latency_ms=elapsed(), - ) - except httpx.TimeoutException: - return DeviceTestResult( - success=False, - message="连接超时（10s），请检查网络或设备地址", - latency_ms=elapsed(), - ) - except Exception as exc: - return DeviceTestResult(success=False, message=f"测试失败：{exc}") + return await test_device(device_id, body) + except DeviceNotFoundError: + raise HTTPException(status_code=http_status.HTTP_404_NOT_FOUND, detail="Device not found") diff --git a/flocks/server/routes/file.py b/flocks/server/routes/file.py index 2493cff67..3b8c969bb 100644 --- a/flocks/server/routes/file.py +++ b/flocks/server/routes/file.py @@ -4,8 +4,11 @@ Routes for file reading, searching, and listing """ +import mimetypes +from pathlib import Path from typing import List, Optional, Dict from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import FileResponse from pydantic import BaseModel from flocks.config.config import Config @@ -59,6 +62,31 @@ async def read_file(path: str = Query(..., description="File path")): raise HTTPException(status_code=500, detail=str(e)) +@router.get("/download", summary="Download file") +async def download_file(path: str = Query(..., description="File path")): + try: + cfg = await Config.get() + safe_path = await resolve_path_for_http_file_access(path, cfg) + target = Path(safe_path) + if not target.is_file(): + raise HTTPException(status_code=400, detail=f"Not a file: {path}") + return FileResponse( + path=str(target), + filename=target.name, + media_type=mimetypes.guess_type(str(target))[0] or "application/octet-stream", + ) + except HTTPException: + raise + except FileNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + except PermissionError: + log.warning("http_file.download.denied", {"path": path}) + raise HTTPException(status_code=403, detail="Access denied") + except Exception as e: + log.error("file.download.error", {"error": str(e), "path": path}) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/search", response_model=List[str], summary="Search files") async def search_files( query: str = Query(..., description="Search query"), diff --git a/flocks/server/routes/logs.py b/flocks/server/routes/logs.py index 5517257cd..c7a59044b 100644 --- a/flocks/server/routes/logs.py +++ b/flocks/server/routes/logs.py @@ -5,6 +5,7 @@ """ from collections import deque +from datetime import date, datetime from pathlib import Path from typing import List @@ -46,9 +47,9 @@ async def list_logs(): return LogListResponse(files=[], log_dir=str(log_dir)) files: List[LogFileInfo] = [] - for p in sorted(log_dir.glob("*.log"), key=lambda f: f.stat().st_mtime, reverse=True): + for p in sorted(_iter_log_files(log_dir), key=lambda f: f.stat().st_mtime, reverse=True): stat = p.stat() - files.append(LogFileInfo(name=p.name, size=stat.st_size, modified=stat.st_mtime)) + files.append(LogFileInfo(name=p.relative_to(log_dir).as_posix(), size=stat.st_size, modified=stat.st_mtime)) return LogListResponse(files=files, log_dir=str(log_dir)) @@ -65,15 +66,24 @@ async def read_latest_log( if not log_dir.is_dir(): raise HTTPException(status_code=404, detail="Log directory not found") - log_files = sorted(log_dir.glob("*.log"), key=lambda f: f.stat().st_mtime, reverse=True) + today_log = log_dir / date.today().isoformat() / "flocks.log" + if today_log.is_file(): + return _read_log_file(today_log, tail, filename=_relative_log_name(log_dir, today_log)) + + for day_dir in sorted(_iter_date_dirs(log_dir), reverse=True): + main_log = day_dir / "flocks.log" + if main_log.is_file(): + return _read_log_file(main_log, tail, filename=_relative_log_name(log_dir, main_log)) + + log_files = sorted(_iter_log_files(log_dir), key=lambda f: f.stat().st_mtime, reverse=True) if not log_files: raise HTTPException(status_code=404, detail="No log files found") - return _read_log_file(log_files[0], tail) + return _read_log_file(log_files[0], tail, filename=_relative_log_name(log_dir, log_files[0])) @router.get( - "/{filename}", + "/{filename:path}", response_model=LogContentResponse, summary="Read a specific log file", ) @@ -85,16 +95,64 @@ async def read_log( log_dir = get_log_dir() log_path = log_dir / filename - if not log_path.is_file() or not log_path.suffix == ".log": + if not log_path.is_file() or not _is_allowed_log_path(log_dir, log_path): raise HTTPException(status_code=404, detail=f"Log file not found: {filename}") if not log_path.resolve().is_relative_to(log_dir.resolve()): raise HTTPException(status_code=403, detail="Access denied") - return _read_log_file(log_path, tail) + return _read_log_file(log_path, tail, filename=filename) + + +def _is_date_dir(path: Path) -> bool: + try: + datetime.strptime(path.name, "%Y-%m-%d") + except ValueError: + return False + return path.is_dir() + +def _iter_date_dirs(log_dir: Path) -> List[Path]: + return [p for p in log_dir.iterdir() if _is_date_dir(p)] -def _read_log_file(path: Path, tail: int) -> LogContentResponse: + +def _is_allowed_log_path(log_dir: Path, path: Path) -> bool: + try: + relative = path.relative_to(log_dir) + except ValueError: + return False + parts = relative.parts + if len(parts) == 1: + return parts[0] in {"backend.log", "webui.log"} + if len(parts) == 2: + day, filename = parts + try: + datetime.strptime(day, "%Y-%m-%d") + except ValueError: + return False + return filename in {"flocks.log", "errors.log"} + return False + + +def _iter_log_files(log_dir: Path) -> List[Path]: + files = [] + for name in ("backend.log", "webui.log"): + path = log_dir / name + if path.is_file(): + files.append(path) + for day_dir in _iter_date_dirs(log_dir): + for name in ("flocks.log", "errors.log"): + path = day_dir / name + if path.is_file(): + files.append(path) + return files + + +def _relative_log_name(log_dir: Path, path: Path) -> str: + return path.relative_to(log_dir).as_posix() + + +def _read_log_file(path: Path, tail: int, filename: str | None = None) -> LogContentResponse: try: lines: deque[str] = deque(maxlen=tail) total = 0 @@ -109,7 +167,7 @@ def _read_log_file(path: Path, tail: int) -> LogContentResponse: content = "\n".join(lines) return LogContentResponse( - filename=path.name, + filename=filename or path.name, content=content, total_lines=total, truncated=truncated, diff --git a/flocks/server/routes/session.py b/flocks/server/routes/session.py index 2b6ad7fbd..a8684521d 100644 --- a/flocks/server/routes/session.py +++ b/flocks/server/routes/session.py @@ -22,6 +22,7 @@ from flocks.session.policy import SessionPolicy from flocks.utils.log import Log from flocks.utils.json_repair import parse_json_robust, repair_truncated_json +from flocks.utils.monitor import get_monitor from flocks.server.auth import require_user router = APIRouter() @@ -37,10 +38,6 @@ # ".exe"). _UPLOAD_SAFE_EXTS = frozenset({"png", "jpg", "jpeg", "gif", "webp", "bmp", "pdf"}) -# Import monitor for metrics endpoint -from flocks.utils.monitor import get_monitor - - # ============================================================================= # Request/Response Models - API Compatible (camelCase) # ============================================================================= @@ -98,8 +95,7 @@ class SessionResponse(BaseModel): """ Session response - Flocks compatible - Matches Flocks Session.Info format exactly. - No agent/model/provider at top level - these come from messages. + Matches Flocks Session.Info format. """ model_config = ConfigDict(populate_by_name=True, by_alias=True) @@ -115,6 +111,9 @@ class SessionResponse(BaseModel): permission: Optional[List[Dict[str, Any]]] = Field(None, description="Permission rules") revert: Optional[Dict[str, Any]] = Field(None, description="Revert state") category: str = Field("user", description="Session category: user or task") + provider: Optional[str] = Field(None, description="Pinned provider ID") + model: Optional[str] = Field(None, description="Pinned model ID") + model_pinned: bool = Field(False, description="Whether provider/model are pinned for this session") ownerUserID: Optional[str] = Field(None, description="Session owner user id") ownerUsername: Optional[str] = Field(None, description="Session owner username") canWrite: bool = Field(False, description="Whether current user can continue this session") @@ -125,9 +124,6 @@ class SessionResponse(BaseModel): def _session_to_response(session: SessionModel) -> SessionResponse: """ Convert SessionModel to SessionResponse - - Note: agent/model/provider are NOT included at session level. - They are retrieved from the latest user message in the session. """ current_user = get_current_auth_user() can_write = SessionPolicy.can_write(session, current_user) @@ -152,6 +148,9 @@ def _session_to_response(session: SessionModel) -> SessionResponse: revert=session.revert.model_dump(by_alias=True) if session.revert else None, permission=[p.model_dump() for p in session.permission] if session.permission else None, category=session.category, + provider=session.provider, + model=session.model, + model_pinned=session.model_pinned, ownerUserID=session.owner_user_id, ownerUsername=session.owner_username, canWrite=can_write, @@ -434,6 +433,7 @@ class TodoInfo(BaseModel): id: str = Field(..., description="Todo ID") content: str = Field(..., description="Task description") + activeForm: Optional[str] = Field(None, description="Active/progressive task description") status: str = Field(..., description="Status: pending, in_progress, completed, cancelled") priority: str = Field("medium", description="Priority: high, medium, low") @@ -446,7 +446,7 @@ class TodoInfo(BaseModel): ) async def get_session_todos(sessionID: str, request: Request) -> List[TodoInfo]: """Get session todos""" - from flocks.storage.storage import Storage + from flocks.session.features.todo import Todo _current_user = require_user(request) session = await _get_session_by_id_unfiltered(sessionID) if not session: @@ -456,10 +456,8 @@ async def get_session_todos(sessionID: str, request: Request) -> List[TodoInfo]: ) _require_session_read_access(session, _current_user) try: - todos = await Storage.read(["todo", sessionID]) - if todos is None: - return [] - return [TodoInfo(**todo) for todo in todos] + todos = await Todo.get(sessionID) + return [TodoInfo(**todo.model_dump(exclude_none=True)) for todo in todos] except Exception as e: log.warn("session.todo.read_error", {"sessionID": sessionID, "error": str(e)}) return [] @@ -473,8 +471,7 @@ async def get_session_todos(sessionID: str, request: Request) -> List[TodoInfo]: ) async def update_session_todos(sessionID: str, todos: List[TodoInfo], request: Request) -> List[TodoInfo]: """Update session todos""" - from flocks.storage.storage import Storage - from flocks.server.routes.event import publish_event + from flocks.session.features.todo import Todo, TodoInfo as SessionTodoInfo _current_user = require_user(request) session = await _get_session_by_id_unfiltered(sessionID) if not session: @@ -484,13 +481,10 @@ async def update_session_todos(sessionID: str, todos: List[TodoInfo], request: R ) _require_session_write_access(session, _current_user) try: - await Storage.write(["todo", sessionID], [t.model_dump() for t in todos]) - - await publish_event("todo.updated", { - "sessionID": sessionID, - "todos": [t.model_dump() for t in todos], - }) - + await Todo.update( + sessionID, + [SessionTodoInfo(**t.model_dump(exclude_none=True)) for t in todos], + ) return todos except Exception as e: log.error("session.todo.write_error", {"sessionID": sessionID, "error": str(e)}) @@ -569,6 +563,9 @@ class SessionUpdateRequest(BaseModel): title: Optional[str] = Field(None, description="New title") time: Optional[Dict[str, Any]] = Field(None, description="Time updates (archived)") + provider: Optional[str] = Field(None, description="Pinned provider ID") + model: Optional[str] = Field(None, description="Pinned model ID") + model_pinned: Optional[bool] = Field(None, description="Whether provider/model are pinned for this session") @router.patch( @@ -598,6 +595,12 @@ async def update_session( updates["title"] = request.title if request.time and request.time.get("archived") is not None: updates["archived"] = request.time["archived"] + if request.provider is not None: + updates["provider"] = request.provider + if request.model is not None: + updates["model"] = request.model + if request.model_pinned is not None: + updates["model_pinned"] = request.model_pinned session = await Session.update( project_id=existing.project_id, @@ -1187,7 +1190,12 @@ async def get_session_messages( _require_session_read_access(session, current_user) try: + from flocks.session.orphan_tools import abort_orphan_running_parts_in_messages + from flocks.session.core.status import SessionStatus + messages_with_parts = await Message.list_with_parts(sessionID, include_archived=True) + if sessionID not in SessionStatus.get_busy_session_ids(): + await abort_orphan_running_parts_in_messages(sessionID, messages_with_parts) if limit: messages_with_parts = messages_with_parts[-limit:] @@ -3365,7 +3373,7 @@ async def run_prompt_queue_item_now(sessionID: str, queueID: str) -> Dict[str, A async def send_session_message_async( sessionID: str, request: PromptRequest, - http_request: Request, + http_request: Request = None, ): """Send message asynchronously - returns 202 immediately, response via SSE""" import os @@ -3379,8 +3387,9 @@ async def send_session_message_async( status_code=status.HTTP_404_NOT_FOUND, detail=f"Session {sessionID} not found" ) - current_user = require_user(http_request) - _require_session_write_access(session, current_user) + if http_request is not None: + current_user = require_user(http_request) + _require_session_write_access(session, current_user) working_directory = session.directory or os.getcwd() @@ -3454,7 +3463,7 @@ async def send_session_command(sessionID: str, request: CommandRequest, http_req Side-effecting direct commands like /clear run without creating a chat message and instead update session state via callbacks. - LLM-based commands (/plan, /ask, /init, /compact, ...) are routed through + LLM-based commands (/init, /compact, ...) are routed through the normal session-loop pipeline. In both cases the user message (showing the raw slash command text, e.g. @@ -3847,5 +3856,3 @@ async def clear_session(sessionID: str, http_request: Request): except Exception as e: log.error("session.clear.error", {"sessionID": sessionID, "error": str(e)}) raise HTTPException(status_code=500, detail=f"Failed to clear session: {str(e)}") - - diff --git a/flocks/server/routes/user_defined_pages.py b/flocks/server/routes/user_defined_pages.py new file mode 100644 index 000000000..151cfcffe --- /dev/null +++ b/flocks/server/routes/user_defined_pages.py @@ -0,0 +1,428 @@ +"""user-defined custom pages API routes.""" + +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import time +import zipfile +from pathlib import Path +from typing import Any, Optional + +from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile, status +from fastapi.responses import FileResponse +from starlette.background import BackgroundTask +from pydantic import BaseModel, ConfigDict, Field + +from flocks.server.auth import require_admin, require_user +from flocks.user_defined_pages.builder import UserDefinedPagesBuilder +from flocks.user_defined_pages.api_runtime import UserDefinedPageApiRuntime +from flocks.user_defined_pages.models import UserDefinedPageBuildMeta, UserDefinedPageDetail, UserDefinedPageListItem, UserDefinedPageManifest +from flocks.user_defined_pages.store import UserDefinedPagesStore +from flocks.server.routes.event import publish_event +from flocks.utils.log import Log + +router = APIRouter() +log = Log.create(service="user-defined-pages-routes") + +MAX_IMPORT_ARCHIVE_BYTES = 10_000_000 +MAX_IMPORT_FILES = 500 +MAX_IMPORT_FILE_BYTES = 5_000_000 +MAX_IMPORT_TOTAL_BYTES = 50_000_000 +_IMPORT_SOURCE_SUFFIXES = {".tsx", ".ts", ".jsx", ".js", ".css", ".json"} +_IMPORT_API_SUFFIXES = {".py", ".yaml", ".yml"} +_IMPORT_DIST_FILES = {"dist/page.js", "dist/meta.json", "dist/api-meta.json"} + +_store = UserDefinedPagesStore() +_builder = UserDefinedPagesBuilder(_store) +_api_runtime = UserDefinedPageApiRuntime(_store) + + +class UserDefinedPageCreateRequest(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + id: str = Field(..., description="Page identifier") + title: str = Field(..., description="Navigation title") + icon: str = Field("LayoutDashboard", description="Lucide icon name") + order: int = Field(100, description="Navigation sort order") + + +class UserDefinedPageSaveRequest(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + manifest: Optional[dict[str, Any]] = Field(None, description="Manifest fields to merge") + sourcePath: Optional[str] = Field(None, description="Relative source path to write") + sourceContent: Optional[str] = Field(None, description="Source file content") + + +class UserDefinedPageSaveResponse(BaseModel): + model_config = ConfigDict(populate_by_name=True, by_alias=True) + + manifest: UserDefinedPageManifest + build: UserDefinedPageBuildMeta + + +async def _read_limited_upload(file: UploadFile) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > MAX_IMPORT_ARCHIVE_BYTES: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail="archive is too large", + ) + chunks.append(chunk) + return b"".join(chunks) + + +def _normalize_archive_member_name(name: str) -> str: + normalized = name.replace("\\", "/") + if normalized.startswith("/") or ".." in normalized.split("/"): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid archive path") + parts = [part for part in normalized.split("/") if part] + if any(part.startswith(".") for part in parts): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="hidden archive paths are not allowed") + return "/".join(parts) + + +def _validate_import_relative_path(relative_path: str) -> None: + if not relative_path: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid archive structure") + if relative_path == "manifest.json": + return + if relative_path.startswith("src/") and Path(relative_path).suffix in _IMPORT_SOURCE_SUFFIXES: + return + if relative_path.startswith("api/") and Path(relative_path).suffix in _IMPORT_API_SUFFIXES: + return + if relative_path.startswith("assets/"): + return + if relative_path in _IMPORT_DIST_FILES: + return + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"unsupported archive file: {relative_path}") + + +def _validate_manifest_entry(entry: str) -> str: + normalized = (entry or "").replace("\\", "/").lstrip("/") + parts = [part for part in normalized.split("/") if part] + if not parts or ".." in parts or any(part.startswith(".") for part in parts): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid manifest entry") + if not normalized.startswith("src/") or Path(normalized).suffix not in _IMPORT_SOURCE_SUFFIXES: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid manifest entry") + return normalized + + +def _normalize_import_manifest(extracted_root: Path, page_id: str) -> None: + manifest_path = extracted_root / "manifest.json" + if not manifest_path.is_file(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="manifest.json is required") + try: + raw = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = UserDefinedPageManifest.model_validate(raw) + except Exception as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"invalid manifest.json: {exc}") from exc + + entry = _validate_manifest_entry(manifest.entry) + normalized = manifest.model_copy( + update={ + "id": page_id, + "route": f"/user-defined-pages/{page_id}", + "entry": entry, + "updatedAt": int(time.time() * 1000), + } + ) + manifest_path.write_text( + json.dumps(normalized.model_dump(), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +@router.get("/user-defined-pages", response_model=list[UserDefinedPageListItem]) +async def list_user_defined_pages(enabled_only: bool = Query(False, alias="enabledOnly")): + return _store.list_pages(enabled_only=enabled_only) + + +@router.post("/user-defined-pages", response_model=UserDefinedPageDetail, status_code=status.HTTP_201_CREATED) +async def create_user_defined_page(req: UserDefinedPageCreateRequest, _admin: object = Depends(require_admin)): + try: + detail = _store.create_page( + page_id=req.id, + title=req.title, + icon=req.icon, + order=req.order, + ) + except FileExistsError as exc: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + try: + build = _builder.build(detail.manifest.id) + if build.status == "ready": + await publish_event("user_defined_pages.updated", {"id": detail.manifest.id, "hash": build.hash}) + elif build.status == "failed": + await publish_event( + "user_defined_pages.build_failed", + {"id": detail.manifest.id, "error": build.error or "build failed"}, + ) + except Exception as exc: + log.warning("user_defined_pages.create.build_failed", {"pageId": detail.manifest.id, "error": str(exc)}) + + await publish_event("user_defined_pages.nav_changed", {"id": detail.manifest.id}) + return _store.get_page(detail.manifest.id) + + +@router.get("/user-defined-pages/{page_id}", response_model=UserDefinedPageDetail) +async def get_user_defined_page(page_id: str): + try: + return _store.get_page(page_id) + except FileNotFoundError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + + +@router.put("/user-defined-pages/{page_id}", response_model=UserDefinedPageSaveResponse) +async def save_user_defined_page(page_id: str, req: UserDefinedPageSaveRequest, _admin: object = Depends(require_admin)): + nav_changed = False + try: + if req.manifest is not None: + _store.save_manifest(page_id, req.manifest) + nav_changed = True + if req.sourcePath is not None: + if req.sourceContent is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="sourceContent is required when sourcePath is provided", + ) + _store.save_source_file(page_id, req.sourcePath, req.sourceContent) + except FileNotFoundError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + build = UserDefinedPageBuildMeta(status="idle") + if req.sourcePath is not None: + rel = req.sourcePath.replace("\\", "/").lstrip("/") + if rel.startswith("api/"): + try: + routes = await _api_runtime.reload_page(page_id) + await publish_event("user_defined_pages.api_changed", {"id": page_id, "routes": routes}) + except HTTPException as exc: + await publish_event( + "user_defined_pages.api_failed", + {"id": page_id, "error": str(exc.detail)}, + ) + raise + except Exception as exc: + await publish_event( + "user_defined_pages.api_failed", + {"id": page_id, "error": str(exc)}, + ) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + else: + build = _builder.build(page_id) + if build.status == "ready": + await publish_event("user_defined_pages.updated", {"id": page_id, "hash": build.hash}) + nav_changed = True + else: + await publish_event( + "user_defined_pages.build_failed", + {"id": page_id, "error": build.error or "build failed"}, + ) + elif nav_changed: + await publish_event("user_defined_pages.nav_changed", {"id": page_id}) + + manifest = _store.get_page(page_id).manifest + return UserDefinedPageSaveResponse(manifest=manifest, build=build) + + +@router.post("/user-defined-pages/{page_id}/build", response_model=UserDefinedPageBuildMeta) +async def build_user_defined_page(page_id: str, _admin: object = Depends(require_admin)): + try: + build = _builder.build(page_id) + except FileNotFoundError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + + if build.status == "ready": + await publish_event("user_defined_pages.updated", {"id": page_id, "hash": build.hash}) + await publish_event("user_defined_pages.nav_changed", {"id": page_id}) + else: + await publish_event( + "user_defined_pages.build_failed", + {"id": page_id, "error": build.error or "build failed"}, + ) + return build + + +@router.get("/user-defined-pages/{page_id}/bundle.js") +async def get_user_defined_page_bundle(page_id: str, v: Optional[str] = Query(None)): + try: + bundle_path = _store.bundle_path(page_id) + if not bundle_path.is_file(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="bundle not found") + headers = {"Cache-Control": "no-cache"} if v else None + return FileResponse( + path=bundle_path, + media_type="application/javascript", + filename="page.js", + headers=headers, + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + +@router.get("/user-defined-pages/{page_id}/assets/{asset_path:path}") +async def get_user_defined_page_asset(page_id: str, asset_path: str): + try: + path = _store.asset_path(page_id, asset_path) + if not path.is_file(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="asset not found") + return FileResponse(path=path) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + +@router.get("/user-defined-pages/{page_id}/api") +async def list_user_defined_page_api_routes(page_id: str): + return await _api_runtime.list_routes(page_id) + + +@router.post("/user-defined-pages/{page_id}/api/reload") +async def reload_user_defined_page_api(page_id: str, _admin: object = Depends(require_admin)): + routes = await _api_runtime.reload_page(page_id) + await publish_event("user_defined_pages.api_changed", {"id": page_id, "routes": routes}) + return {"routes": routes} + + +@router.api_route( + "/user-defined-pages/{page_id}/api/{api_path:path}", + methods=["GET", "POST", "PUT", "PATCH", "DELETE"], +) +async def dispatch_user_defined_page_api(page_id: str, api_path: str, request: Request): + user = require_user(request) + return await _api_runtime.dispatch(page_id, api_path, request, user) + + +@router.get("/user-defined-pages/{page_id}/export") +async def export_user_defined_page(page_id: str, _admin: object = Depends(require_admin)): + page_path = _store.page_dir(page_id) + if not page_path.is_dir(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"page not found: {page_id}") + + fd, archive_path = tempfile.mkstemp(prefix=f"user-defined-page-{page_id}-", suffix=".zip") + os.close(fd) + try: + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for file_path in page_path.rglob("*"): + if not file_path.is_file(): + continue + arc_name = str(file_path.relative_to(page_path)).replace("\\", "/") + zf.write(file_path, arcname=f"{page_id}/{arc_name}") + except Exception: + if os.path.exists(archive_path): + os.unlink(archive_path) + raise + return FileResponse( + archive_path, + media_type="application/zip", + filename=f"{page_id}.zip", + background=BackgroundTask(lambda: os.path.exists(archive_path) and os.unlink(archive_path)), + ) + + +@router.post("/user-defined-pages/import") +async def import_user_defined_page( + file: UploadFile = File(...), + overwrite: bool = Query(False), + _admin: object = Depends(require_admin), +): + data = await _read_limited_upload(file) + if not data: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="empty archive") + try: + with tempfile.TemporaryDirectory(prefix="udp-import-") as tmpdir: + temp_root = Path(tmpdir) / "extract" + temp_root.mkdir(parents=True, exist_ok=True) + archive_path = Path(tmpdir) / "archive.zip" + archive_path.write_bytes(data) + with zipfile.ZipFile(archive_path) as zf: + members = [member for member in zf.infolist() if not member.is_dir()] + if not members: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="archive has no files") + if len(members) > MAX_IMPORT_FILES: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="archive has too many files") + names = [_normalize_archive_member_name(member.filename) for member in members] + root_parts = {name.split("/", 1)[0] for name in names} + if len(root_parts) != 1: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="archive must contain a single page root directory") + page_id = _store.validate_page_id(next(iter(root_parts))) + extracted_root = temp_root / page_id + total_uncompressed = 0 + for member, member_name in zip(members, names): + if "/" not in member_name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid archive structure") + rel_part = member_name.split("/", 1)[1] + _validate_import_relative_path(rel_part) + if member.file_size > MAX_IMPORT_FILE_BYTES: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="archive file is too large") + total_uncompressed += member.file_size + if total_uncompressed > MAX_IMPORT_TOTAL_BYTES: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="archive contents are too large") + target = (extracted_root / rel_part).resolve() + try: + target.relative_to(extracted_root.resolve()) + except ValueError: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid archive path") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(zf.read(member)) + _normalize_import_manifest(extracted_root, page_id) + target = _store.page_dir(page_id) + if target.exists(): + if not overwrite: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"page already exists: {page_id}") + shutil.rmtree(target) + shutil.move(str((temp_root / page_id).resolve()), str(target)) + try: + build = _builder.build(page_id) + if build.status == "ready": + await publish_event("user_defined_pages.updated", {"id": page_id, "hash": build.hash}) + else: + await publish_event( + "user_defined_pages.build_failed", + {"id": page_id, "error": build.error or "build failed"}, + ) + except Exception as exc: + log.warning("user_defined_pages.import.build_failed", {"pageId": page_id, "error": str(exc)}) + await publish_event("user_defined_pages.build_failed", {"id": page_id, "error": str(exc)}) + if _store.routes_path(page_id).is_file(): + try: + routes = await _api_runtime.reload_page(page_id) + await publish_event("user_defined_pages.api_changed", {"id": page_id, "routes": routes}) + except Exception as exc: + log.warning("user_defined_pages.import.api_reload_failed", {"pageId": page_id, "error": str(exc)}) + await publish_event("user_defined_pages.api_failed", {"id": page_id, "error": str(exc)}) + await publish_event("user_defined_pages.nav_changed", {"id": page_id}) + return _store.get_page(page_id) + except HTTPException: + raise + except zipfile.BadZipFile as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid zip archive") from exc + + +def reset_route_dependencies( + *, + store: Optional[UserDefinedPagesStore] = None, + builder: Optional[UserDefinedPagesBuilder] = None, + api_runtime: Optional[UserDefinedPageApiRuntime] = None, +) -> None: + """Test helper to inject isolated store/builder instances.""" + global _store, _builder, _api_runtime + _store = store or UserDefinedPagesStore() + _builder = builder or UserDefinedPagesBuilder(_store) + _api_runtime = api_runtime or UserDefinedPageApiRuntime(_store) diff --git a/flocks/server/routes/workflow.py b/flocks/server/routes/workflow.py index d36bc67af..a54631a4a 100644 --- a/flocks/server/routes/workflow.py +++ b/flocks/server/routes/workflow.py @@ -7,6 +7,7 @@ import asyncio import hashlib +import hmac import json import os import shutil @@ -15,7 +16,7 @@ from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Any, Dict, Literal -from fastapi import APIRouter, HTTPException, status, Query +from fastapi import APIRouter, HTTPException, Request, status, Query from pydantic import BaseModel, Field, ConfigDict import uuid @@ -57,6 +58,25 @@ from flocks.workflow.io import load_workflow, dump_workflow from flocks.workflow.tool_context import build_workflow_tool_context from flocks.workflow.tools import get_tool_registry +from flocks.workflow.triggers import ( + TriggerDefinition, + TriggerEvent, + build_trigger_event, + preview_trigger_mapping, + set_workflow_json_triggers, + workflow_json_declares_triggers, + workflow_trigger_definitions_from_json, +) +from flocks.workflow.triggers.dispatcher import evaluate_trigger_filter +from flocks.workflow.triggers.runtime import default_runtime as default_trigger_runtime +from flocks.workflow.triggers.compat import ( + kafka_trigger_to_legacy_config, + legacy_kafka_trigger_from_config, + legacy_schedule_trigger_from_config, + legacy_syslog_trigger_from_config, + schedule_trigger_to_legacy_config, + syslog_trigger_to_legacy_config, +) from flocks.config.config import Config from flocks.storage.storage import Storage from flocks.server.routes.event import publish_event @@ -65,8 +85,11 @@ router = APIRouter() +webhook_router = APIRouter() log = Log.create(service="workflow-routes") +_LEGACY_SINGLETON_TRIGGER_TYPES = frozenset({"schedule", "kafka", "syslog"}) + @dataclass class ActiveWorkflowExecution: @@ -153,6 +176,11 @@ class WorkflowExecutionResponse(BaseModel): duration: Optional[float] = Field(None, description="Duration (seconds)") executionLog: List[Dict[str, Any]] = Field(default_factory=list, description="Execution log") errorMessage: Optional[str] = Field(None, description="Error message") + triggerId: Optional[str] = Field(None, description="Trigger ID") + triggerType: Optional[str] = Field(None, description="Trigger type") + deliveryId: Optional[str] = Field(None, description="Trigger delivery ID") + attempt: Optional[int] = Field(None, description="Trigger attempt") + triggerSource: Optional[str] = Field(None, description="Trigger source") currentNodeId: Optional[str] = Field(None, description="Current running node ID") currentNodeType: Optional[str] = Field(None, description="Current running node type") currentPhase: Optional[str] = Field(None, description="Current execution phase") @@ -394,6 +422,151 @@ def _syslog_config_key(workflow_id: str) -> str: return f"{WORKFLOW_SYSLOG_CONFIG_PREFIX}{workflow_id}" +async def _read_legacy_trigger_defs(workflow_id: str) -> List[TriggerDefinition]: + triggers: List[TriggerDefinition] = [] + for key, converter in ( + (_kafka_config_key(workflow_id), legacy_kafka_trigger_from_config), + (f"workflow_poller_config/{workflow_id}", legacy_schedule_trigger_from_config), + (_syslog_config_key(workflow_id), legacy_syslog_trigger_from_config), + ): + try: + config = await Storage.read(key) + except Exception: + config = None + trigger = converter(config) + if trigger is not None: + triggers.append(trigger) + return triggers + + +async def _get_workflow_trigger_defs( + workflow_id: str, + workflow_data: Optional[Dict[str, Any]] = None, +) -> List[TriggerDefinition]: + data = workflow_data or _read_workflow_from_fs(workflow_id) + if not data: + return [] + workflow_json = data.get("workflowJson") or {} + triggers = workflow_trigger_definitions_from_json(workflow_json) + # Once the workflow JSON explicitly declares a trigger list, it becomes the + # single source of truth, even when the list is empty. + if workflow_json_declares_triggers(workflow_json): + return triggers + return await _read_legacy_trigger_defs(workflow_id) + + +def _trigger_to_api_dict(trigger: TriggerDefinition) -> Dict[str, Any]: + return trigger.model_dump(mode="json", by_alias=True, exclude_none=True) + + +def _replace_or_append_trigger( + triggers: List[TriggerDefinition], + trigger: TriggerDefinition, +) -> List[TriggerDefinition]: + updated = [existing for existing in triggers if existing.id != trigger.id] + updated.append(trigger) + return updated + + +def _disable_legacy_trigger_of_type( + workflow_id: str, + trigger_type: str, +) -> tuple[Optional[str], Optional[Dict[str, Any]]]: + now_ms = int(time.time() * 1000) + if trigger_type == "kafka": + return ( + _kafka_config_key(workflow_id), + {"workflowId": workflow_id, "enabled": False, "updatedAt": now_ms}, + ) + if trigger_type == "schedule": + return ( + f"workflow_poller_config/{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": now_ms}, + ) + if trigger_type == "syslog": + return ( + _syslog_config_key(workflow_id), + {"workflowId": workflow_id, "enabled": False, "updatedAt": now_ms}, + ) + return None, None + + +async def _sync_trigger_legacy_state(workflow_id: str, trigger: TriggerDefinition) -> Optional[Dict[str, Any]]: + if trigger.type == "kafka": + config = kafka_trigger_to_legacy_config(workflow_id, trigger) + await Storage.write(_kafka_config_key(workflow_id), config) + from flocks.ingest.kafka.manager import default_manager as _kafka_default_manager + + return await _kafka_default_manager.restart_workflow(workflow_id) + if trigger.type == "schedule": + config = schedule_trigger_to_legacy_config(workflow_id, trigger) + await Storage.write(f"workflow_poller_config/{workflow_id}", config) + from flocks.workflow.poller_manager import default_manager as _poller_default_manager + + return await _poller_default_manager.restart_workflow(workflow_id) + if trigger.type == "syslog": + config = syslog_trigger_to_legacy_config(workflow_id, trigger) + await Storage.write(_syslog_config_key(workflow_id), config) + from flocks.ingest.syslog.manager import default_manager as _syslog_default_manager + + return await _syslog_default_manager.restart_workflow(workflow_id) + return await default_trigger_runtime.get_trigger_status(workflow_id, trigger) + + +async def _remove_legacy_trigger_state(workflow_id: str, trigger: TriggerDefinition) -> None: + """Remove legacy trigger configs so deleted unified triggers do not reappear.""" + if trigger.type == "kafka": + try: + from flocks.ingest.kafka.manager import default_manager as _kafka_default_manager + + await _kafka_default_manager.stop_workflow(workflow_id) + except Exception: + pass + try: + await Storage.remove(_kafka_config_key(workflow_id)) + except Storage.NotFoundError: + pass + return + if trigger.type == "schedule": + try: + from flocks.workflow.poller_manager import default_manager as _poller_default_manager + + await _poller_default_manager.stop_workflow(workflow_id) + except Exception: + pass + try: + await Storage.remove(f"workflow_poller_config/{workflow_id}") + except Storage.NotFoundError: + pass + return + if trigger.type == "syslog": + try: + from flocks.ingest.syslog.manager import default_manager as _syslog_default_manager + + await _syslog_default_manager.stop_workflow(workflow_id) + except Exception: + pass + try: + await Storage.remove(_syslog_config_key(workflow_id)) + except Storage.NotFoundError: + pass + + +async def _persist_workflow_triggers( + workflow_id: str, + workflow_data: Dict[str, Any], + triggers: List[TriggerDefinition], +) -> Dict[str, Any]: + workflow_json = workflow_data.get("workflowJson") or {} + updated_json = set_workflow_json_triggers(workflow_json, triggers) + data = dict(workflow_data) + data["workflowJson"] = updated_json + data["updatedAt"] = int(time.time() * 1000) + is_global = data.get("source") == "global" + _write_workflow_to_fs(workflow_id, updated_json, data, data.get("markdownContent"), global_store=is_global) + return data + + async def _run_workflow_execution_task( *, workflow_id: str, @@ -1124,6 +1297,8 @@ async def workflow_center_releases(workflow_id: str): async def get_workflow_history( workflow_id: str, limit: int = Query(50, ge=1, le=100, description="Max results"), + trigger_id: Optional[str] = Query(None, alias="triggerId"), + trigger_type: Optional[str] = Query(None, alias="triggerType"), ): """ Get workflow execution history @@ -1131,7 +1306,8 @@ async def get_workflow_history( Returns list of recent executions for this workflow. """ try: - if not _read_workflow_from_fs(workflow_id): + data = _read_workflow_from_fs(workflow_id) + if not data: raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") # 单次查询批量读取所有 execution 记录，避免 N 次单独 read 导致超长耗时 @@ -1143,6 +1319,10 @@ async def get_workflow_history( continue if exec_data.get("workflowId") != workflow_id: continue + if trigger_id and exec_data.get("triggerId") != trigger_id: + continue + if trigger_type and exec_data.get("triggerType") != trigger_type: + continue executions.append(WorkflowExecutionResponse(**exec_data)) except Exception as e: log.warning("workflow.history.skip", {"key": _key, "error": str(e)}) @@ -1408,6 +1588,38 @@ def _strip_execution_only_comments(value: Any) -> Any: } +class TriggerEventPayloadRequest(BaseModel): + """Sample event payload for trigger preview/testing.""" + + model_config = ConfigDict(populate_by_name=True) + + body: Any = None + headers: Dict[str, Any] = Field(default_factory=dict) + query: Dict[str, Any] = Field(default_factory=dict) + path_params: Dict[str, Any] = Field(default_factory=dict, alias="pathParams") + + +class TriggerPreviewResponse(BaseModel): + """Preview result for trigger mapping and filtering.""" + + model_config = ConfigDict(populate_by_name=True, by_alias=True) + + triggerId: str + triggerType: str + matched: bool + inputs: Dict[str, Any] = Field(default_factory=dict) + filterError: Optional[str] = None + + +class TriggerSaveResponse(BaseModel): + """Persisted trigger definition with runtime status.""" + + model_config = ConfigDict(populate_by_name=True, by_alias=True) + + trigger: Dict[str, Any] + status: Optional[Dict[str, Any]] = None + + class WorkflowPollerConfigRequest(BaseModel): """Per-workflow background poller configuration.""" @@ -1582,6 +1794,292 @@ async def list_workflow_services(): raise HTTPException(status_code=500, detail=f"Failed to list services: {str(e)}") +def _find_trigger_or_404(triggers: List[TriggerDefinition], trigger_id: str) -> TriggerDefinition: + trigger = next((item for item in triggers if item.id == trigger_id), None) + if trigger is None: + raise HTTPException(status_code=404, detail=f"Trigger not found: {trigger_id}") + return trigger + + +def _validate_trigger_type_constraints(triggers: List[TriggerDefinition]) -> None: + singleton_ids_by_type: Dict[str, List[str]] = {} + for trigger in triggers: + if trigger.type not in _LEGACY_SINGLETON_TRIGGER_TYPES: + continue + singleton_ids_by_type.setdefault(trigger.type, []).append(trigger.id or "") + + duplicates = { + trigger_type: trigger_ids + for trigger_type, trigger_ids in singleton_ids_by_type.items() + if len(trigger_ids) > 1 + } + if not duplicates: + return + + first_type = sorted(duplicates)[0] + trigger_ids = [trigger_id for trigger_id in duplicates[first_type] if trigger_id] + detail = ( + f"Only one {first_type} trigger is supported per workflow; " + f"found: {', '.join(trigger_ids) or 'multiple triggers'}" + ) + raise HTTPException(status_code=409, detail=detail) + + +@router.get("/workflow/{workflow_id}/triggers") +async def list_workflow_triggers(workflow_id: str): + """List unified triggers for a workflow with runtime status.""" + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + triggers = await _get_workflow_trigger_defs(workflow_id, data) + statuses = { + item.get("triggerId"): item + for item in await default_trigger_runtime.get_workflow_trigger_statuses( + workflow_id, + set_workflow_json_triggers(data.get("workflowJson") or {}, triggers), + ) + } + return [ + { + "trigger": _trigger_to_api_dict(trigger), + "status": statuses.get(trigger.id), + } + for trigger in triggers + ] + + +@router.post("/workflow/{workflow_id}/triggers", response_model=TriggerSaveResponse) +async def create_workflow_trigger(workflow_id: str, trigger: TriggerDefinition): + """Create or replace a unified trigger definition.""" + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + existing = await _get_workflow_trigger_defs(workflow_id, data) + updated = _replace_or_append_trigger(existing, trigger) + _validate_trigger_type_constraints(updated) + persisted = await _persist_workflow_triggers(workflow_id, data, updated) + await default_trigger_runtime.restart_workflow(workflow_id, persisted.get("workflowJson") or {}) + status = await default_trigger_runtime.get_trigger_status(workflow_id, trigger) + return TriggerSaveResponse(trigger=_trigger_to_api_dict(trigger), status=status) + + +@router.put("/workflow/{workflow_id}/triggers/{trigger_id}", response_model=TriggerSaveResponse) +async def update_workflow_trigger(workflow_id: str, trigger_id: str, trigger: TriggerDefinition): + """Update a unified trigger definition.""" + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + existing = await _get_workflow_trigger_defs(workflow_id, data) + _find_trigger_or_404(existing, trigger_id) + updated_trigger = trigger.model_copy(update={"id": trigger_id}) + updated = _replace_or_append_trigger(existing, updated_trigger) + _validate_trigger_type_constraints(updated) + persisted = await _persist_workflow_triggers(workflow_id, data, updated) + await default_trigger_runtime.restart_workflow(workflow_id, persisted.get("workflowJson") or {}) + status = await default_trigger_runtime.get_trigger_status(workflow_id, updated_trigger) + return TriggerSaveResponse(trigger=_trigger_to_api_dict(updated_trigger), status=status) + + +@router.delete("/workflow/{workflow_id}/triggers/{trigger_id}") +async def delete_workflow_trigger(workflow_id: str, trigger_id: str): + """Delete a unified trigger definition.""" + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + existing = await _get_workflow_trigger_defs(workflow_id, data) + trigger = _find_trigger_or_404(existing, trigger_id) + remaining = [item for item in existing if item.id != trigger_id] + persisted = await _persist_workflow_triggers(workflow_id, data, remaining) + await _remove_legacy_trigger_state(workflow_id, trigger) + await default_trigger_runtime.restart_workflow(workflow_id, persisted.get("workflowJson") or {}) + return {"ok": True, "triggerId": trigger_id} + + +@router.get("/workflow/{workflow_id}/triggers/{trigger_id}/status") +async def get_workflow_trigger_status(workflow_id: str, trigger_id: str): + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = _find_trigger_or_404(triggers, trigger_id) + return await default_trigger_runtime.get_trigger_status(workflow_id, trigger) + + +@router.post("/workflow/{workflow_id}/triggers/{trigger_id}/preview-mapping", response_model=TriggerPreviewResponse) +async def preview_workflow_trigger_mapping( + workflow_id: str, + trigger_id: str, + payload: TriggerEventPayloadRequest, +): + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = _find_trigger_or_404(triggers, trigger_id) + event = build_trigger_event( + workflow_id=workflow_id, + trigger=trigger, + body=payload.body, + headers=payload.headers, + query=payload.query, + path_params=payload.path_params, + ) + matched, filter_error = evaluate_trigger_filter(trigger, event) + return TriggerPreviewResponse( + triggerId=trigger.id or trigger_id, + triggerType=trigger.type, + matched=matched, + inputs=preview_trigger_mapping(trigger, event), + filterError=filter_error, + ) + + +@router.post("/workflow/{workflow_id}/triggers/{trigger_id}/test") +async def test_workflow_trigger( + workflow_id: str, + trigger_id: str, + payload: TriggerEventPayloadRequest, +): + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + workflow_json = data.get("workflowJson") or {} + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = _find_trigger_or_404(triggers, trigger_id) + event = build_trigger_event( + workflow_id=workflow_id, + trigger=trigger, + body=payload.body, + headers=payload.headers, + query=payload.query, + path_params=payload.path_params, + ) + result = await default_trigger_runtime.dispatch_event( + workflow_id=workflow_id, + workflow_json=workflow_json, + trigger=trigger, + event=event, + ) + return { + "ok": True, + "trigger": _trigger_to_api_dict(trigger), + **result, + } + + +@router.get("/workflow-trigger-plugins") +async def list_workflow_trigger_plugins(): + return default_trigger_runtime.list_plugin_specs() + + +def _resolve_trigger_secret(secret_ref: Optional[str]) -> Optional[str]: + if not secret_ref: + return None + try: + from flocks.security import get_secret_manager + + return get_secret_manager().get(secret_ref) + except Exception: + return None + + +def _normalize_hmac_signature(signature: Optional[str]) -> Optional[str]: + if not signature: + return None + value = signature.strip() + if value.lower().startswith("sha256="): + return value.split("=", 1)[1].strip() + return value + + +def _authorize_webhook_trigger( + trigger: TriggerDefinition, + headers: Dict[str, str], + query: Dict[str, str], + *, + raw_body: bytes, +) -> None: + auth = trigger.auth + if auth is None or auth.type in {"none", ""}: + return + if auth.type == "api_key": + expected = auth.apiKey or _resolve_trigger_secret(auth.secretRef) + if not expected: + raise HTTPException(status_code=401, detail="Webhook trigger API key is not configured") + header_name = (auth.headerName or "x-api-key").lower() + actual = headers.get(header_name) or query.get(auth.queryParam or "api_key") + if actual != expected: + raise HTTPException(status_code=401, detail="Invalid webhook API key") + return + if auth.type == "hmac": + expected = _resolve_trigger_secret(auth.secretRef) + if not expected: + raise HTTPException(status_code=401, detail="Webhook trigger secret is not configured") + signature = _normalize_hmac_signature(headers.get((auth.headerName or "x-flocks-signature").lower())) + expected_signature = hmac.new( + expected.encode("utf-8"), + raw_body, + hashlib.sha256, + ).hexdigest() + if not signature or not hmac.compare_digest(signature, expected_signature): + raise HTTPException(status_code=401, detail="Invalid webhook signature") + return + raise HTTPException(status_code=400, detail=f"Unsupported webhook auth type: {auth.type}") + + +@webhook_router.post("/webhook/workflows/{workflow_id}/{trigger_id}") +async def invoke_workflow_webhook_trigger( + workflow_id: str, + trigger_id: str, + request: Request, +): + """Invoke a webhook/custom_webhook trigger and dispatch the workflow.""" + data = _read_workflow_from_fs(workflow_id) + if not data: + raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") + workflow_json = data.get("workflowJson") or {} + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = _find_trigger_or_404(triggers, trigger_id) + if trigger.type not in {"webhook", "custom_webhook"}: + raise HTTPException(status_code=400, detail=f"Trigger is not a webhook trigger: {trigger_id}") + if not trigger.enabled: + raise HTTPException(status_code=403, detail=f"Trigger is disabled: {trigger_id}") + + headers = {key.lower(): value for key, value in request.headers.items()} + query = {key: value for key, value in request.query_params.items()} + raw_body = await request.body() + _authorize_webhook_trigger(trigger, headers, query, raw_body=raw_body) + + try: + body = json.loads(raw_body.decode("utf-8")) + except Exception: + body = raw_body.decode("utf-8", errors="replace") + + event = build_trigger_event( + workflow_id=workflow_id, + trigger=trigger, + body=body, + headers=headers, + query=query, + path_params={"workflow_id": workflow_id, "trigger_id": trigger_id}, + raw=body, + source=(trigger.source or {}).get("path") or str(request.url.path), + ) + result = await default_trigger_runtime.dispatch_event( + workflow_id=workflow_id, + workflow_json=workflow_json, + trigger=trigger, + event=event, + ) + return { + "ok": True, + "matched": result.get("matched", True), + "executed": result.get("executed", False), + "inputs": result.get("inputs", {}), + "deliveryId": event.source.deliveryId, + } + + @router.post("/workflow/{workflow_id}/kafka-config") async def save_kafka_config(workflow_id: str, req: KafkaConfigRequest): """ @@ -1593,7 +2091,8 @@ async def save_kafka_config(workflow_id: str, req: KafkaConfigRequest): instead of falsely claiming the consumer is running. """ try: - if not _read_workflow_from_fs(workflow_id): + data = _read_workflow_from_fs(workflow_id) + if not data: raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") config = { @@ -1608,6 +2107,32 @@ async def save_kafka_config(workflow_id: str, req: KafkaConfigRequest): "updatedAt": int(time.time() * 1000), } await Storage.write(_kafka_config_key(workflow_id), config) + unified_trigger = TriggerDefinition.model_validate( + { + "id": "kafka-default", + "type": "kafka", + "enabled": req.enabled, + "source": { + "inputBroker": req.inputBroker or "", + "inputTopic": req.inputTopic or "", + "inputGroupId": req.inputGroupId or "", + "autoOffsetReset": req.autoOffsetReset, + }, + "mapping": { + req.inputKey or "kafka_message": "$.body", + }, + "inputs": _strip_execution_only_comments(req.inputs), + "updatedAt": config["updatedAt"], + } + ) + triggers = await _get_workflow_trigger_defs(workflow_id, data) + updated_triggers = _replace_or_append_trigger(triggers, unified_trigger) + _validate_trigger_type_constraints(updated_triggers) + await _persist_workflow_triggers( + workflow_id, + data, + updated_triggers, + ) from flocks.ingest.kafka.manager import default_manager as _kafka_default_manager @@ -1634,6 +2159,13 @@ async def get_kafka_config(workflow_id: str): """ try: config = await Storage.read(_kafka_config_key(workflow_id)) + if config is None: + data = _read_workflow_from_fs(workflow_id) + if data: + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = next((item for item in triggers if item.type == "kafka"), None) + if trigger is not None: + config = kafka_trigger_to_legacy_config(workflow_id, trigger) return config # None / null if not configured except Exception as e: log.error("workflow.kafka_config.get.error", {"id": workflow_id, "error": str(e)}) @@ -1661,7 +2193,8 @@ async def get_kafka_status(workflow_id: str): async def save_workflow_poller_config(workflow_id: str, req: WorkflowPollerConfigRequest): """Save background poller configuration for a workflow.""" try: - if not _read_workflow_from_fs(workflow_id): + data = _read_workflow_from_fs(workflow_id) + if not data: raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") config = { @@ -1674,6 +2207,31 @@ async def save_workflow_poller_config(workflow_id: str, req: WorkflowPollerConfi "updatedAt": int(time.time() * 1000), } await Storage.write(f"workflow_poller_config/{workflow_id}", config) + unified_trigger = TriggerDefinition.model_validate( + { + "id": "schedule-default", + "type": "schedule", + "enabled": req.enabled, + "source": { + "mode": "interval", + "intervalSeconds": req.intervalSeconds, + }, + "runtime": { + "timeoutSeconds": req.timeoutSeconds, + "noOverlap": req.noOverlap, + }, + "inputs": req.inputs, + "updatedAt": config["updatedAt"], + } + ) + triggers = await _get_workflow_trigger_defs(workflow_id, data) + updated_triggers = _replace_or_append_trigger(triggers, unified_trigger) + _validate_trigger_type_constraints(updated_triggers) + await _persist_workflow_triggers( + workflow_id, + data, + updated_triggers, + ) from flocks.workflow.poller_manager import default_manager as _poller_default_manager @@ -1696,7 +2254,15 @@ async def save_workflow_poller_config(workflow_id: str, req: WorkflowPollerConfi async def get_workflow_poller_config(workflow_id: str): """Get saved poller configuration for a workflow.""" try: - return await Storage.read(f"workflow_poller_config/{workflow_id}") + config = await Storage.read(f"workflow_poller_config/{workflow_id}") + if config is None: + data = _read_workflow_from_fs(workflow_id) + if data: + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = next((item for item in triggers if item.type == "schedule"), None) + if trigger is not None: + config = schedule_trigger_to_legacy_config(workflow_id, trigger) + return config except Exception as e: log.error("workflow.poller_config.get.error", {"id": workflow_id, "error": str(e)}) raise HTTPException(status_code=500, detail=f"Failed to get poller config: {str(e)}") @@ -1718,7 +2284,8 @@ async def get_workflow_poller_status(workflow_id: str): async def run_workflow_poller_once(workflow_id: str): """Trigger one immediate poller execution for a workflow.""" try: - if not _read_workflow_from_fs(workflow_id): + data = _read_workflow_from_fs(workflow_id) + if not data: raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") from flocks.workflow.poller_manager import default_manager as _poller_default_manager @@ -1744,7 +2311,8 @@ async def save_syslog_config(workflow_id: str, req: SyslogConfigRequest): instead of falsely claiming "Listening". """ try: - if not _read_workflow_from_fs(workflow_id): + data = _read_workflow_from_fs(workflow_id) + if not data: raise HTTPException(status_code=404, detail=f"Workflow not found: {workflow_id}") config = { @@ -1758,6 +2326,31 @@ async def save_syslog_config(workflow_id: str, req: SyslogConfigRequest): "updatedAt": int(time.time() * 1000), } await Storage.write(_syslog_config_key(workflow_id), config) + unified_trigger = TriggerDefinition.model_validate( + { + "id": "syslog-default", + "type": "syslog", + "enabled": req.enabled, + "source": { + "protocol": req.protocol, + "host": req.host, + "port": req.port, + "format": req.msg_format, + }, + "mapping": { + req.input_key or "syslog_message": "$.body", + }, + "updatedAt": config["updatedAt"], + } + ) + triggers = await _get_workflow_trigger_defs(workflow_id, data) + updated_triggers = _replace_or_append_trigger(triggers, unified_trigger) + _validate_trigger_type_constraints(updated_triggers) + await _persist_workflow_triggers( + workflow_id, + data, + updated_triggers, + ) from flocks.ingest.syslog.manager import default_manager as _syslog_default_manager @@ -1782,6 +2375,13 @@ async def get_syslog_config(workflow_id: str): """Get saved syslog configuration for a workflow.""" try: config = await Storage.read(_syslog_config_key(workflow_id)) + if config is None: + data = _read_workflow_from_fs(workflow_id) + if data: + triggers = await _get_workflow_trigger_defs(workflow_id, data) + trigger = next((item for item in triggers if item.type == "syslog"), None) + if trigger is not None: + config = syslog_trigger_to_legacy_config(workflow_id, trigger) return config except Exception as e: log.error("workflow.syslog_config.get.error", {"id": workflow_id, "error": str(e)}) diff --git a/flocks/server/routes/workspace.py b/flocks/server/routes/workspace.py index 42019bd71..51aed456d 100644 --- a/flocks/server/routes/workspace.py +++ b/flocks/server/routes/workspace.py @@ -57,6 +57,7 @@ # Upload size limit read at request time so env-var changes take effect # without restarting the process. _DEFAULT_MAX_UPLOAD_MB = 100 +_DEFAULT_MAX_READ_BYTES = 2 * 1024 * 1024 _ALLOWED_UPLOAD_EXTENSIONS = { ".txt", ".md", ".json", ".yaml", ".yml", ".xml", ".csv", ".pdf", ".doc", ".docx", ".html", ".htm", ".ppt", ".pptx", ".xls", ".xlsx", @@ -68,6 +69,10 @@ def _max_upload_bytes() -> int: return int(os.getenv("FLOCKS_WORKSPACE_MAX_UPLOAD_MB", str(_DEFAULT_MAX_UPLOAD_MB))) * 1024 * 1024 +def _max_read_bytes() -> int: + return int(os.getenv("FLOCKS_WORKSPACE_MAX_READ_BYTES", str(_DEFAULT_MAX_READ_BYTES))) + + # ─── helpers ──────────────────────────────────────────────────────────────── def _get_manager() -> WorkspaceManager: @@ -144,6 +149,16 @@ def _dir_stats_sync(root: Path): return file_count, dir_count, total_size +def _read_text_preview_sync(path: Path, max_bytes: int) -> tuple[str, bool]: + """Read at most ``max_bytes`` from a text file for safe preview.""" + with path.open("rb") as handle: + data = handle.read(max_bytes + 1) + truncated = len(data) > max_bytes + if truncated: + data = data[:max_bytes] + return data.decode("utf-8", errors="replace"), truncated + + # ─── directory operations ─────────────────────────────────────────────────── @router.get("/tree", response_model=WorkspaceNode, summary="List directory tree") @@ -314,11 +329,22 @@ async def read_file( status_code=400, detail="Binary file — use /download endpoint instead", ) + max_read_bytes = _max_read_bytes() try: - content = target.read_text(encoding="utf-8", errors="replace") + content, truncated = await asyncio.to_thread( + _read_text_preview_sync, + target, + max_read_bytes, + ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - return {"path": path, "content": content} + return { + "path": path, + "content": content, + "truncated": truncated, + "size": target.stat().st_size, + "preview_limit_bytes": max_read_bytes, + } class FileWriteRequest(BaseModel): @@ -470,11 +496,22 @@ async def read_memory_file( raise HTTPException(status_code=404, detail=f"Memory file not found: {path}") if not target.is_file(): raise HTTPException(status_code=400, detail=f"Not a file: {path}") + max_read_bytes = _max_read_bytes() try: - content = target.read_text(encoding="utf-8", errors="replace") + content, truncated = await asyncio.to_thread( + _read_text_preview_sync, + target, + max_read_bytes, + ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - return {"path": path, "content": content} + return { + "path": path, + "content": content, + "truncated": truncated, + "size": target.stat().st_size, + "preview_limit_bytes": max_read_bytes, + } # ─── stats ────────────────────────────────────────────────────────────────── diff --git a/flocks/session/orphan_tools.py b/flocks/session/orphan_tools.py new file mode 100644 index 000000000..dd97c724b --- /dev/null +++ b/flocks/session/orphan_tools.py @@ -0,0 +1,100 @@ +"""Recovery helpers for tool calls left running by interrupted processes.""" + +import time +from typing import Iterable, Optional + +from flocks.session.message import Message, MessageWithParts, ToolPart, ToolStateError +from flocks.session.session import SessionInfo +from flocks.storage.storage import Storage +from flocks.utils.log import Log + + +log = Log.create(service="session.orphan_tools") + + +INTERRUPTED_TOOL_ERROR = "Interrupted by server restart" + + +def _build_interrupted_error_state(state: object, now_ms: int) -> ToolStateError: + """Create the terminal error state used for recovered orphaned tools.""" + time_info = getattr(state, "time", {}) or {} + start_ms = time_info.get("start", now_ms) + + return ToolStateError( + status="error", + input=getattr(state, "input", {}), + error=INTERRUPTED_TOOL_ERROR, + metadata=getattr(state, "metadata", None), + time={"start": start_ms, "end": now_ms}, + ) + + +async def abort_orphan_running_parts_in_messages( + session_id: str, + messages_with_parts: Iterable[MessageWithParts], +) -> int: + """Mark running tool parts as interrupted using preloaded message parts.""" + now_ms = int(time.time() * 1000) + repaired = 0 + + for msg_with_parts in messages_with_parts: + message_id = msg_with_parts.info.id + for part in msg_with_parts.parts: + if not isinstance(part, ToolPart): + continue + state = part.state + if getattr(state, "status", None) != "running": + continue + + part.state = _build_interrupted_error_state(state, now_ms) + await Message.store_part(session_id, message_id, part) + repaired += 1 + + if repaired: + log.info("session.orphan_tools.aborted", { + "session_id": session_id, + "count": repaired, + }) + return repaired + + +async def abort_orphan_running_parts(session_id: str) -> int: + """Mark persisted running tool parts as interrupted errors.""" + messages_with_parts = await Message.list_with_parts(session_id) + return await abort_orphan_running_parts_in_messages(session_id, messages_with_parts) + + +async def abort_orphan_running_parts_for_sessions( + session_ids: Iterable[str], + *, + skip_busy: bool = False, +) -> int: + """Best-effort recovery for a known set of sessions.""" + total = 0 + for session_id in dict.fromkeys(session_ids): + try: + if skip_busy: + from flocks.session.core.status import SessionStatus + + if session_id in SessionStatus.get_busy_session_ids(): + continue + total += await abort_orphan_running_parts(session_id) + except Exception as exc: + log.warn("session.orphan_tools.session_failed", { + "session_id": session_id, + "error": str(exc), + }) + return total + + +async def abort_all_orphan_running_parts(*, limit: Optional[int] = None) -> int: + """Best-effort startup recovery for all persisted sessions.""" + entries = await Storage.list_entries(prefix="session:", model=SessionInfo) + session_ids = [ + session.id + for _, session in entries + if getattr(session, "status", None) != "deleted" + ] + if limit is not None: + session_ids = session_ids[:limit] + return await abort_orphan_running_parts_for_sessions(session_ids, skip_busy=True) diff --git a/flocks/session/prompt.py b/flocks/session/prompt.py index 3896400e6..d36a961e3 100644 --- a/flocks/session/prompt.py +++ b/flocks/session/prompt.py @@ -992,6 +992,70 @@ def _print_system_prompts_for_debug( print(f"\n--- prompt[{idx}] ---\n{prompt}\n", file=sys.stderr) print("=== end system_prompt ===\n", file=sys.stderr) + @classmethod + def _build_minimal_environment(cls, session_directory: Optional[str]) -> str: + """Build the small runtime environment block used by system subagents.""" + working_dir = session_directory or os.getcwd() + today = datetime.now().strftime("%A %b %d, %Y") + return "\n".join([ + "## Environment", + f"Current working directory: {working_dir}", + f"Platform: {platform.system().lower()}", + f"Today's date: {today}", + ]) + + @classmethod + async def _is_builtin_system_subagent_session( + cls, + *, + session_id: str, + agent_name: str, + ) -> bool: + """Return true for built-in system subagents running as child sessions.""" + try: + from flocks.agent.registry import Agent + from flocks.session.session import Session + + agent = await Agent.get(agent_name) + if not agent: + return False + if agent.mode != "subagent" or "system" not in (agent.tags or []): + return False + + builtin_agents_dir = Path(__file__).resolve().parents[1] / "agent" / "agents" + name_candidates = { + agent.name, + agent.name.replace("-", "_"), + agent_name, + agent_name.replace("-", "_"), + } + if not any((builtin_agents_dir / name / "agent.yaml").exists() for name in name_candidates): + return False + + session = await Session.get_by_id(session_id) + return bool(session and session.parent_id) + except Exception as exc: + log.debug("prompt.subagent_minimal_check_failed", { + "session_id": session_id, + "agent_name": agent_name, + "error": str(exc), + }) + return False + + @classmethod + async def _build_subagent_minimal_prompts( + cls, + *, + session_directory: Optional[str], + agent_prompt: Optional[str], + ) -> List[str]: + """Build minimal system prompts for built-in system subagents.""" + prompts = [ + cls._normalize_prompt_text(agent_prompt), + cls._build_minimal_environment(session_directory), + ] + return [prompt for prompt in prompts if prompt] + @classmethod async def build_system_prompts( cls, @@ -1021,8 +1085,25 @@ async def build_system_prompts( construction below so this method reads as an ordered list of prompt layers. """ - normalized_tool_names = tuple(sorted(prompt_tool_names)) vcs = "git" if session_directory else None + if await cls._is_builtin_system_subagent_session( + session_id=session_id, + agent_name=agent_name, + ): + prompts = await cls._build_subagent_minimal_prompts( + session_directory=session_directory, + agent_prompt=agent_prompt, + ) + cls._print_system_prompts_for_debug( + session_id=session_id, + agent_name=agent_name, + provider_id=provider_id, + model_id=model_id, + prompts=prompts, + ) + return prompts + + normalized_tool_names = tuple(sorted(prompt_tool_names)) runtime_day = datetime.now().strftime("%Y-%m-%d") custom_signature = SystemPrompt.custom_signature(directory=session_directory) memory_guidance = cls._build_memory_guidance_prompt( diff --git a/flocks/session/prompt/anthropic-20250930.txt b/flocks/session/prompt/anthropic-20250930.txt index 138b5305f..a8ada5ede 100644 --- a/flocks/session/prompt/anthropic-20250930.txt +++ b/flocks/session/prompt/anthropic-20250930.txt @@ -69,7 +69,7 @@ For example, if the user asks you how to approach something, you should do your Prioritize technical accuracy and truthfulness over validating the user's beliefs. Focus on facts and problem-solving, providing direct, objective technical info without any unnecessary superlatives, praise, or emotional validation. It is best for the user if Claude honestly applies the same rigorous standards to all ideas and disagrees when necessary, even if it may not be what the user wants to hear. Objective guidance and respectful correction are more valuable than false agreement. Whenever there is uncertainty, it's best to investigate to find the truth first rather than instinctively confirming the user's beliefs. # Task Management -You have access to the TodoWrite tools to help you manage and plan tasks. Use these tools VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. +You have access to the `todo` tool to help you manage and plan tasks. Use `todo(action="write")` VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. These tools are also EXTREMELY helpful for planning tasks, and for breaking down larger complex tasks into smaller steps. If you do not use this tool when planning, you may forget to do important tasks - and that is unacceptable. It is critical that you mark todos as completed as soon as you are done with a task. Do not batch up multiple tasks before marking them as completed. @@ -78,7 +78,7 @@ Examples: user: Analyze our web server logs for potential SQL injection attempts -assistant: I'm going to use the TodoWrite tool to write the following items to the todo list: +assistant: I'm going to use `todo(action="write")` to write the following items to the todo list: - Read and parse web server access logs - Identify SQL injection patterns (UNION, OR 1=1, etc.) - Correlate with application error logs @@ -86,7 +86,7 @@ assistant: I'm going to use the TodoWrite tool to write the following items to t I'm now going to read the log files using the Read tool. -Found 43 suspicious requests with SQL injection indicators. I'm going to use the TodoWrite tool to add analysis tasks for each pattern. +Found 43 suspicious requests with SQL injection indicators. I'm going to use `todo(action="write")` to add analysis tasks for each pattern. marking the first todo as in_progress @@ -101,7 +101,7 @@ In the above example, the assistant completes all the security analysis tasks, i user: Help me create a YARA rule to detect ransomware behavior -assistant: I'll help you create a YARA rule for ransomware detection. Let me first use the TodoWrite tool to plan this task. +assistant: I'll help you create a YARA rule for ransomware detection. Let me first use `todo(action="write")` to plan this task. Adding the following todos to the todo list: 1. Research common ransomware indicators (file encryption, ransom notes, extensions) 2. Identify behavioral patterns and strings @@ -145,7 +145,7 @@ Assistant knowledge cutoff is January 2025. IMPORTANT: Assist with defensive security tasks only. Refuse to create, modify, or improve code that may be used maliciously. Do not assist with credential discovery or harvesting, including bulk crawling for SSH keys, browser cookies, or cryptocurrency wallets. Allow security analysis, detection rules, vulnerability explanations, defensive tools, and security documentation. -IMPORTANT: Always use the TodoWrite tool to plan and track tasks throughout the conversation. +IMPORTANT: Always use `todo(action="write")` to plan and track tasks throughout the conversation. # Security References diff --git a/flocks/session/prompt/anthropic.txt b/flocks/session/prompt/anthropic.txt index 301e12df7..655551871 100644 --- a/flocks/session/prompt/anthropic.txt +++ b/flocks/session/prompt/anthropic.txt @@ -16,7 +16,7 @@ CRITICAL: NEVER include ``, ``, or any XML-based tool cal Prioritize technical accuracy and truthfulness over validating the user's beliefs. Focus on facts and problem-solving, providing direct, objective technical info without any unnecessary superlatives, praise, or emotional validation. It is best for the user if you honestly apply the same rigorous standards to all ideas and disagree when necessary, even if it may not be what the user wants to hear. Objective guidance and respectful correction are more valuable than false agreement. Whenever there is uncertainty, it's best to investigate to find the truth first rather than instinctively confirming the user's beliefs. # Task Management -You have access to the TodoWrite tools to help you manage and plan tasks. Use these tools VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. +You have access to the `todo` tool to help you manage and plan tasks. Use `todo(action="write")` VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. These tools are also EXTREMELY helpful for planning tasks, and for breaking down larger complex tasks into smaller steps. If you do not use this tool when planning, you may forget to do important tasks - and that is unacceptable. It is critical that you mark todos as completed as soon as you are done with a task. Do not batch up multiple tasks before marking them as completed. @@ -25,7 +25,7 @@ Examples: user: Analyze this suspicious login pattern in our auth logs -assistant: I'm going to use the TodoWrite tool to write the following items to the todo list: +assistant: I'm going to use `todo(action="write")` to write the following items to the todo list: - Read and parse the authentication logs - Identify anomalous login patterns - Check for brute force indicators @@ -33,7 +33,7 @@ assistant: I'm going to use the TodoWrite tool to write the following items to t I'm now going to read the log files using the Read tool. -Found 127 failed login attempts from 15 different IPs. I'm going to use the TodoWrite tool to add analysis tasks for each suspicious pattern. +Found 127 failed login attempts from 15 different IPs. I'm going to use `todo(action="write")` to add analysis tasks for each suspicious pattern. marking the first todo as in_progress @@ -47,7 +47,7 @@ In the above example, the assistant completes all the security analysis tasks, i user: Help me create a detection rule for detecting suspicious PowerShell commands -assistant: I'll help you create a PowerShell detection rule. Let me first use the TodoWrite tool to plan this task. +assistant: I'll help you create a PowerShell detection rule. Let me first use `todo(action="write")` to plan this task. Adding the following todos to the todo list: 1. Research common malicious PowerShell patterns 2. Identify detection points (command line arguments, encoding, obfuscation) @@ -84,7 +84,7 @@ user: Find all places where user input is processed without validation assistant: [Uses the Task tool to comprehensively search for input validation gaps] -IMPORTANT: Always use the TodoWrite tool to plan and track tasks throughout the conversation. +IMPORTANT: Always use `todo(action="write")` to plan and track tasks throughout the conversation. # Security References diff --git a/flocks/session/prompt/plan-reminder-anthropic.txt b/flocks/session/prompt/plan-reminder-anthropic.txt deleted file mode 100644 index 28f1e629d..000000000 --- a/flocks/session/prompt/plan-reminder-anthropic.txt +++ /dev/null @@ -1,67 +0,0 @@ - -# Plan Mode - System Reminder - -Plan mode is active. The user indicated that they do not want you to execute yet -- you MUST NOT make any edits (with the exception of the plan file mentioned below), run any non-readonly tools (including changing configs or making commits), or otherwise make any changes to the system. This supersedes any other instructions you have received. - ---- - -## Plan File Info - -No plan file exists yet. You should create your plan at `/Users/aidencline/.claude/plans/happy-waddling-feigenbaum.md` using the Write tool. - -You should build your plan incrementally by writing to or editing this file. NOTE that this is the only file you are allowed to edit - other than this you are only allowed to take READ-ONLY actions. - -**Plan File Guidelines:** The plan file should contain only your final recommended approach, not all alternatives considered. Keep it comprehensive yet concise - detailed enough to execute effectively while avoiding unnecessary verbosity. - ---- - -## Enhanced Planning Workflow - -### Phase 1: Initial Understanding - -**Goal:** Gain a comprehensive understanding of the user's request by reading through code and asking them questions. Critical: In this phase you should only use the Explore subagent type. - -1. Understand the user's request thoroughly - -2. **Launch up to 3 Explore agents IN PARALLEL** (single message, multiple tool calls) to efficiently explore the codebase. Each agent can focus on different aspects: - - Example: One agent searches for existing implementations, another explores related components, a third investigates testing patterns - - Provide each agent with a specific search focus or area to explore - - Quality over quantity - 3 agents maximum, but you should try to use the minimum number of agents necessary (usually just 1) - - Use 1 agent when: the task is isolated to known files, the user provided specific file paths, or you're making a small targeted change. Use multiple agents when: the scope is uncertain, multiple areas of the codebase are involved, or you need to understand existing patterns before planning. - - Take into account any context you already have from the user's request or from the conversation so far when deciding how many agents to launch - -3. Use AskUserQuestion tool to clarify ambiguities in the user request up front. - -### Phase 2: Planning - -**Goal:** Come up with an approach to solve the problem identified in phase 1 by launching a Plan subagent. - -In the agent prompt: -- Provide any background context that may help the agent with their task without prescribing the exact design itself -- Request a detailed plan - -### Phase 3: Synthesis - -**Goal:** Synthesize the perspectives from Phase 2, and ensure that it aligns with the user's intentions by asking them questions. - -1. Collect all agent responses -2. Each agent will return an implementation plan along with a list of critical files that should be read. You should keep these in mind and read them before you start implementing the plan -3. Use AskUserQuestion to ask the users questions about trade offs. - -### Phase 4: Final Plan - -Once you have all the information you need, ensure that the plan file has been updated with your synthesized recommendation including: -- Recommended approach with rationale -- Key insights from different perspectives -- Critical files that need modification - -### Phase 5: Call ExitPlanMode - -At the very end of your turn, once you have asked the user questions and are happy with your final plan file - you should always call ExitPlanMode to indicate to the user that you are done planning. - -This is critical - your turn should only end with either asking the user a question or calling ExitPlanMode. Do not stop unless it's for these 2 reasons. - ---- - -**NOTE:** At any point in time through this workflow you should feel free to ask the user questions or clarifications. Don't make large assumptions about user intent. The goal is to present a well researched plan to the user, and tie any loose ends before implementation begins. - diff --git a/flocks/session/runner.py b/flocks/session/runner.py index 577714d89..df2f11145 100644 --- a/flocks/session/runner.py +++ b/flocks/session/runner.py @@ -1103,8 +1103,20 @@ async def device_asset_prompt_factory() -> Optional[str]: # Convert messages to chat format with error handling try: + queued_user_message_ids = self._get_queued_user_message_ids(messages) + if queued_user_message_ids: + self._invalidate_chat_context_cache() + previous_queued_user_ids = getattr(self, "_queued_user_message_ids", None) + self._queued_user_message_ids = queued_user_message_ids chat_messages_started_at = time.perf_counter() - chat_messages = await self._to_chat_messages(messages, system_prompts) + try: + chat_messages = await self._to_chat_messages(messages, system_prompts) + finally: + if previous_queued_user_ids is None: + if hasattr(self, "_queued_user_message_ids"): + delattr(self, "_queued_user_message_ids") + else: + self._queued_user_message_ids = previous_queued_user_ids self._log_perf( "runner.process_step.chat_messages_ready", chat_messages_started_at, @@ -1141,67 +1153,6 @@ async def device_asset_prompt_factory() -> Optional[str]: "session_id": self.session.id, }) - # Add reminder wrapping for queued user messages (matching Flocks logic) - # This reminds the AI to address new user messages while continuing tasks - if self._step > 1: - # Find last finished assistant message - last_finished = None - for msg in reversed(messages): - if msg.role == MessageRole.ASSISTANT and hasattr(msg, 'finish') and msg.finish: - if msg.finish not in ("tool-calls", "unknown"): - last_finished = msg - break - - # Wrap queued user messages with reminder - if last_finished: - from flocks.session.prompt_strings import SYNTHETIC_MESSAGE_MARKERS - for chat_msg in chat_messages: - if chat_msg.role != "user": - continue - - content = chat_msg.content - - if isinstance(content, str): - if any(marker in content for marker in SYNTHETIC_MESSAGE_MARKERS): - continue - chat_msg.content = ( - "\n" - "The user sent the following message:\n" - f"{content}\n\n" - "Please address this message and continue with your tasks.\n" - "" - ) - elif isinstance(content, list): - # Multimodal user content (e.g. image_url blocks). - # Naively f-stringing the whole list would call - # ``str(list)`` and serialize every image block — base64 - # data and all — into plain text, which both blows up - # the token count AND makes vision-capable models - # respond with "I see only base64 text". Wrap *only* - # the first text block instead, leaving image blocks - # untouched. If there is no text block at all (rare — - # an image-only turn), skip wrapping entirely. - first_text_idx: Optional[int] = None - for idx, block in enumerate(content): - if isinstance(block, dict) and block.get("type") == "text": - first_text_idx = idx - break - if first_text_idx is None: - continue - text_val = content[first_text_idx].get("text") or "" - if any(marker in text_val for marker in SYNTHETIC_MESSAGE_MARKERS): - continue - content[first_text_idx] = { - "type": "text", - "text": ( - "\n" - "The user sent the following message:\n" - f"{text_val}\n\n" - "Please address this message and continue with your tasks.\n" - "" - ), - } - # Add max steps warning if this is the last step (matching Flocks) if is_last_step: from flocks.session.prompt_strings import PROMPT_MAX_STEPS @@ -1275,6 +1226,20 @@ async def device_asset_prompt_factory() -> Optional[str]: assistant_msg=assistant_msg, ) + if getattr(self, "_llm_call_aborted", False): + await Message.update( + self.session.id, + assistant_msg.id, + error=self._build_message_aborted_error(), + finish="error", + ) + await self._record_usage_if_available(result.usage, message_id=assistant_msg.id) + return StepResult( + action="stop", + content=result.content, + usage=result.usage, + ) + # Detect empty response: some models (e.g. MiniMax) occasionally # return 0 chunks with finish_reason=stop after tool execution, # producing no text and no tool calls. Treat this as a transient @@ -1888,6 +1853,90 @@ def _message_conversion_cache_key( self._provider_capability_key(), ) + @staticmethod + def _build_message_aborted_error() -> Dict[str, Any]: + message = "Message generation was interrupted before completion." + return { + "name": "MessageAbortedError", + "message": message, + "data": { + "message": message, + }, + } + + @staticmethod + def _message_role_value(msg: MessageInfo) -> Optional[str]: + return msg.role if isinstance(msg.role, str) else getattr(msg.role, "value", None) + + def _get_queued_user_message_ids( + self, + messages: List[MessageInfo], + ) -> set[str]: + if self._step <= 1: + return set() + + last_finished_index: Optional[int] = None + for idx, msg in enumerate(messages): + role = self._message_role_value(msg) + finish = getattr(msg, "finish", None) + if role == "assistant" and finish and finish not in ("tool-calls", "unknown"): + last_finished_index = idx + + if last_finished_index is None: + return set() + + queued_user_ids: set[str] = set() + seen_turn_root_user = False + for msg in messages[last_finished_index + 1:]: + if self._message_role_value(msg) != "user": + continue + if not seen_turn_root_user: + seen_turn_root_user = True + continue + queued_user_ids.add(msg.id) + return queued_user_ids + + def _invalidate_chat_context_cache(self) -> None: + self._static_cache.pop("chat_context_cache", None) + + @staticmethod + def _wrap_queued_user_text(content: str) -> str: + from flocks.session.prompt_strings import SYNTHETIC_MESSAGE_MARKERS + + if not content or any(marker in content for marker in SYNTHETIC_MESSAGE_MARKERS): + return content + return ( + "\n" + "The user sent the following message:\n" + f"{content}\n\n" + "Please address this message and continue with your tasks.\n" + "" + ) + + def _wrap_queued_user_blocks( + self, + blocks: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + first_text_idx: Optional[int] = None + for idx, block in enumerate(blocks): + if isinstance(block, dict) and block.get("type") == "text": + first_text_idx = idx + break + if first_text_idx is None: + return blocks + + text_val = blocks[first_text_idx].get("text") or "" + wrapped_text = self._wrap_queued_user_text(text_val) + if wrapped_text == text_val: + return blocks + + wrapped_blocks = copy.deepcopy(blocks) + wrapped_blocks[first_text_idx] = { + "type": "text", + "text": wrapped_text, + } + return wrapped_blocks + @staticmethod def _clone_cached_chat_messages(payloads: List[Dict[str, Any]]) -> List[ChatMessage]: return [ChatMessage.model_validate(copy.deepcopy(payload)) for payload in payloads] @@ -1975,6 +2024,7 @@ async def _to_chat_messages( ctx_window_tokens = self._get_context_window_tokens() tool_result_refs: List[Dict[str, Any]] = [] turn_index = 0 + queued_user_message_ids: set[str] = set(getattr(self, "_queued_user_message_ids", set()) or set()) active_model = Provider.resolve_model(self.provider_id, self.model_id) active_interleaved = ( getattr(active_model.capabilities, "interleaved", None) @@ -2047,7 +2097,8 @@ async def _to_chat_messages( for idx, msg in enumerate(messages): if idx < resume_message_index: continue - if msg.role == MessageRole.USER or (isinstance(msg.role, str) and msg.role == "user"): + role = msg.role if isinstance(msg.role, str) else msg.role.value + if role == "user": turn_index += 1 is_latest_user_turn = msg.id == last_user_msg_id # Get message parts @@ -2057,14 +2108,18 @@ async def _to_chat_messages( # Fallback: use text content only content = await Message.get_text_content(msg) if content.strip(): + normalized_content = _expand_workflow_node_ref(content) + if role == "user" and msg.id in queued_user_message_ids: + normalized_content = self._wrap_queued_user_text(normalized_content) chat_messages.append(ChatMessage( - role=msg.role if isinstance(msg.role, str) else msg.role.value, - content=_expand_workflow_node_ref(content), + role=role, + content=normalized_content, )) continue # Build message content from parts if msg.role == MessageRole.USER or (isinstance(msg.role, str) and msg.role == "user"): + is_queued_user_turn = msg.id in queued_user_message_ids user_content_parts = [] user_content_blocks: list[dict[str, Any]] = [] for part in parts: @@ -2123,14 +2178,19 @@ async def _to_chat_messages( block.get("type") == "image" for block in user_content_blocks ): + if is_queued_user_turn: + user_content_blocks = self._wrap_queued_user_blocks(user_content_blocks) chat_messages.append(ChatMessage( role="user", content=user_content_blocks, )) elif user_content_parts: + user_text = "\n\n".join(user_content_parts) + if is_queued_user_turn: + user_text = self._wrap_queued_user_text(user_text) chat_messages.append(ChatMessage( role="user", - content="\n\n".join(user_content_parts), + content=user_text, )) elif msg.role == MessageRole.ASSISTANT or (isinstance(msg.role, str) and msg.role == "assistant"): @@ -2439,6 +2499,7 @@ def _build_llm_response_payload( session_id=self.session.id, assistant_message=assistant_msg, agent=agent, + abort_event=self._external_abort or self._abort, permission_callback=self._handle_permission, text_delta_callback=self.callbacks.on_text_delta, reasoning_delta_callback=self.callbacks.on_reasoning_delta, @@ -2574,6 +2635,7 @@ def _build_llm_response_payload( } llm_before_enabled = False llm_after_enabled = False + self._llm_call_aborted = False try: llm_before_enabled = await HookPipeline.has_stage_handlers( HookStage.LLM_BEFORE, @@ -2612,6 +2674,7 @@ def _build_llm_response_payload( llm_call_started_at = time.perf_counter() first_chunk_logged = False + aborted_during_stream = False try: async for chunk in _iter_with_chunk_timeout( provider.chat_stream( @@ -2648,6 +2711,7 @@ def _build_llm_response_payload( # Check for abort if self.is_aborted: + aborted_during_stream = True break # Determine event type from chunk. A single chunk may carry any @@ -2812,6 +2876,11 @@ def _build_llm_response_payload( await processor.process_event(FinishEvent( finish_reason=processor.get_finish_reason() )) + + # Foreground subagent tool-calls are launched concurrently during + # streaming so sibling subagents can start in the same assistant turn. + # Drain them here before exposing tool results to the next loop step. + await processor.drain_parallel_tool_calls() # Get processed content content = processor.get_text_content() @@ -2849,6 +2918,7 @@ def _build_llm_response_payload( assistant_msg.id, content=content, ) + self._llm_call_aborted = aborted_during_stream # Note: Tools were already executed synchronously during streaming # Build tool call list for result diff --git a/flocks/session/session_loop.py b/flocks/session/session_loop.py index 6b2e1f455..3423216c7 100644 --- a/flocks/session/session_loop.py +++ b/flocks/session/session_loop.py @@ -188,6 +188,27 @@ async def _publish_runtime_event( "error": str(exc), }) + @classmethod + async def _publish_session_status( + cls, + callbacks: "LoopCallbacks", + session_id: str, + status: str, + ) -> None: + if not callbacks.event_publish_callback: + return + try: + await callbacks.event_publish_callback("session.status", { + "sessionID": session_id, + "status": {"type": status}, + }) + except Exception as exc: + log.debug("loop.session_status.publish_failed", { + "session_id": session_id, + "status": status, + "error": str(exc), + }) + @classmethod async def _publish_session_notice( cls, @@ -341,11 +362,14 @@ async def run( # Set status to busy SessionStatus.set(session_id, SessionStatusBusy()) + await cls._publish_session_status(callbacks or LoopCallbacks(), session_id, "busy") # Mark orphaned running tool parts as error (e.g. from server restart). # Wrapped in try/except so cleanup failures never block the session loop. try: - await cls._abort_orphan_running_parts(session_id) + from flocks.session.orphan_tools import abort_orphan_running_parts + + await abort_orphan_running_parts(session_id) except Exception as exc: log.warn("loop.orphan_cleanup_failed", { "session_id": session_id, @@ -382,6 +406,7 @@ async def run( # Set status to idle SessionStatus.set(session_id, SessionStatusIdle()) + await cls._publish_session_status(callbacks or LoopCallbacks(), session_id, "idle") # Touch session (update timestamp) await Session.touch(session.project_id, session_id) @@ -394,57 +419,6 @@ async def run( except Exception as exc: log.warn("loop.idle.event_error", {"error": str(exc)}) - @classmethod - async def _abort_orphan_running_parts(cls, session_id: str) -> None: - """Mark any tool parts stuck in 'running' status as error. - - When the server restarts while a synchronous tool (e.g. delegate_task) - is executing, the tool part stays 'running' in storage forever. On the - next session loop start we know nothing is actually executing yet, so - any 'running' parts are orphans. - """ - import time as _time - from flocks.session.message import ( - ToolPart, ToolStateError, - ) - - messages = await Message.list(session_id) - now_ms = int(_time.time() * 1000) - repaired = 0 - - for msg in messages: - parts = await Message.parts(msg.id, session_id) - for part in parts: - if not isinstance(part, ToolPart): - continue - state = part.state - if getattr(state, "status", None) != "running": - continue - - time_info = getattr(state, "time", {}) or {} - start_ms = time_info.get("start", now_ms) - - error_state = ToolStateError( - status="error", - input=getattr(state, "input", {}), - error="Interrupted by server restart", - time={"start": start_ms, "end": now_ms}, - ) - # Preserve metadata (e.g. sessionId) so the card still works - meta = getattr(state, "metadata", None) - if meta: - error_state.metadata = meta - - part.state = error_state - await Message.store_part(session_id, msg.id, part) - repaired += 1 - - if repaired: - log.info("loop.orphan_parts_aborted", { - "session_id": session_id, - "count": repaired, - }) - @staticmethod async def _resolve_model( session: Any, diff --git a/flocks/session/streaming/stream_processor.py b/flocks/session/streaming/stream_processor.py index d895faded..e6047ef2b 100644 --- a/flocks/session/streaming/stream_processor.py +++ b/flocks/session/streaming/stream_processor.py @@ -96,6 +96,7 @@ def __init__( session_id: str, assistant_message: MessageInfo, agent: AgentInfo, + abort_event: Optional[asyncio.Event] = None, permission_callback: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None, text_delta_callback: Optional[Callable[[str], Awaitable[None]]] = None, reasoning_delta_callback: Optional[Callable[[str], Awaitable[None]]] = None, @@ -112,6 +113,7 @@ def __init__( self.session_id = session_id self.assistant_message = assistant_message self.agent = agent + self.abort_event = abort_event or asyncio.Event() self.permission_callback = permission_callback self.text_delta_callback = text_delta_callback self.reasoning_delta_callback = reasoning_delta_callback @@ -154,6 +156,12 @@ def __init__( # Flag: model emitted text-embedded tool calls ( XML) that were extracted and executed self._text_tool_calls_executed = False + + # Foreground subagents are long-running but independent. Start them as + # soon as their tool-call arrives so later sibling tool-calls in the same + # model response can launch too; the runner drains these before the step + # returns. + self._parallel_tool_tasks: Dict[str, asyncio.Task[None]] = {} async def process_event(self, event: StreamEvent) -> None: """ @@ -193,7 +201,10 @@ async def process_event(self, event: StreamEvent) -> None: pass # Input is complete elif event_type == "tool-call": - await self._handle_tool_call(event) + if self._should_run_tool_call_parallel(event): + self._start_parallel_tool_call(event) + else: + await self._handle_tool_call(event) elif event_type == "text-start": await self._handle_text_start(event) @@ -343,7 +354,7 @@ async def _handle_reasoning_end(self, event: ReasoningEndEvent) -> None: """Handle reasoning block end""" if event.id in self.reasoning_parts: part = self.reasoning_parts[event.id] - part.text = part.text.rstrip() + part.text = part.text.strip() if event.metadata: self._merge_reasoning_metadata(part, event.metadata) @@ -652,6 +663,22 @@ def _make_metadata_cb( import copy _finished = [False] + _pending_tasks: set[asyncio.Task[Any]] = set() + + def _track_task(coro: Awaitable[None]) -> None: + task = asyncio.create_task(coro) + _pending_tasks.add(task) + + def _cleanup(done_task: asyncio.Task[Any]) -> None: + _pending_tasks.discard(done_task) + try: + done_task.result() + except asyncio.CancelledError: + pass + except Exception as exc: + log.debug("stream.metadata_task.error", {"error": str(exc)}) + + task.add_done_callback(_cleanup) def _cb(metadata: Dict[str, Any]): if _finished[0]: @@ -667,6 +694,8 @@ def _cb(metadata: Dict[str, Any]): state_dict["title"] = snapshot["title"] if self.event_publish_callback: async def _safe_publish(): + if _finished[0]: + return try: await self.event_publish_callback( "message.part.updated", @@ -682,9 +711,11 @@ async def _safe_publish(): } }, ) + except asyncio.CancelledError: + return except Exception as exc: log.debug("stream.metadata_publish.error", {"error": str(exc)}) - asyncio.create_task(_safe_publish()) + _track_task(_safe_publish()) # Persist updated running state so metadata (e.g. sessionId) # survives page reload / session switch @@ -713,11 +744,18 @@ async def _persist_running_metadata(): self.assistant_message.id, part, ) + except asyncio.CancelledError: + return except Exception as exc: log.debug("stream.metadata_persist.error", {"error": str(exc)}) - asyncio.create_task(_persist_running_metadata()) + _track_task(_persist_running_metadata()) + + def _mark_finished() -> None: + _finished[0] = True + for task in list(_pending_tasks): + task.cancel() - _cb.mark_finished = lambda: _finished.__setitem__(0, True) + _cb.mark_finished = _mark_finished return _cb ctx = ToolContext( @@ -725,23 +763,25 @@ async def _persist_running_metadata(): message_id=self.assistant_message.id, agent=self.agent.name, call_id=tool_call_id, + abort_event=self.abort_event, permission_callback=self.permission_callback, extra=sandbox_meta["extra"], metadata_callback=_make_metadata_cb(), event_publish_callback=self.event_publish_callback, ) - - result = await ToolRegistry.execute( - tool_name=tool_name, - ctx=ctx, - **tool_input - ) - - # Mark metadata callback as finished so pending async persist - # tasks won't overwrite the upcoming completed/error state cb = ctx._metadata_callback - if cb and hasattr(cb, 'mark_finished'): - cb.mark_finished() + try: + result = await ToolRegistry.execute( + tool_name=tool_name, + ctx=ctx, + **tool_input + ) + finally: + # Mark metadata callback as finished so pending async + # running-state updates cannot overwrite completed, + # errored, or interrupted tool state. + if cb and hasattr(cb, 'mark_finished'): + cb.mark_finished() # Hook pipeline: tool.execute.after try: @@ -867,6 +907,67 @@ async def _persist_running_metadata(): except Exception as e: log.error("stream.tool_end_callback.error", {"error": str(e)}) + except asyncio.CancelledError: + interrupt_msg = "Tool execution was interrupted" + log.info("stream.tool_call.cancelled", { + "tool_call_id": tool_call_id, + "tool_name": tool_name, + }) + try: + if tool_span_ctx is not None: + tool_span_ctx.end( + output=interrupt_msg, + metadata={"success": False}, + level="ERROR", + status_message="tool_cancelled", + ) + except Exception as _span_err: + log.debug("stream.tool_span.cancel_end_failed", {"error": str(_span_err)}) + + tool_state.status = "error" + tool_state.error = interrupt_msg + + try: + tool_end_time = int(datetime.now().timestamp() * 1000) + error_state = ToolStateError( + status="error", + input=tool_input, + error=interrupt_msg, + time={"start": tool_start_time if 'tool_start_time' in locals() else tool_end_time, "end": tool_end_time}, + ) + + error_part = ToolPart( + id=tool_state.part_id, + sessionID=self.session_id, + messageID=self.assistant_message.id, + type="tool", + callID=tool_call_id, + tool=tool_name, + state=error_state, + ) + await Message.store_part(self.session_id, self.assistant_message.id, error_part) + + if self.event_publish_callback: + await self.event_publish_callback("message.part.updated", { + "part": { + "id": tool_state.part_id, + "messageID": self.assistant_message.id, + "sessionID": self.session_id, + "type": "tool", + "callID": tool_call_id, + "tool": tool_name, + "state": { + "status": "error", + "input": tool_input, + "error": interrupt_msg, + "time": {"start": tool_start_time if 'tool_start_time' in locals() else tool_end_time, "end": tool_end_time}, + } + } + }) + except Exception as store_e: + log.error("stream.tool_call.cancelled_update_failed", {"error": str(store_e)}) + + raise except Exception as e: log.error("stream.tool_call.error", { "tool_call_id": tool_call_id, @@ -1200,7 +1301,7 @@ async def _handle_text_end(self, event: TextEndEvent) -> None: "session_id": self.session_id, "reason": "found XML tool-call markup but could not parse any valid tool calls", }) - + # Update time if self.current_text_part.time: self.current_text_part.time.end = int(datetime.now().timestamp() * 1000) @@ -1227,6 +1328,50 @@ async def _handle_text_end(self, event: TextEndEvent) -> None: self.current_text_part = None + def _should_run_tool_call_parallel(self, event: ToolCallEvent) -> bool: + """Return true for independent foreground subagent tool-calls.""" + if event.tool_name not in {"delegate_task", "task"}: + return False + tool_input = event.input if isinstance(event.input, dict) else {} + if tool_input.get("run_in_background") is True: + return False + return bool( + tool_input.get("subagent_type") + or tool_input.get("category") + or tool_input.get("session_id") + ) + + def _start_parallel_tool_call(self, event: ToolCallEvent) -> None: + if event.tool_call_id in self._parallel_tool_tasks: + return + + async def _run() -> None: + await self._handle_tool_call(event) + + task = asyncio.create_task(_run(), name=f"tool:{event.tool_name}:{event.tool_call_id}") + self._parallel_tool_tasks[event.tool_call_id] = task + + def _cleanup(done_task: asyncio.Task[None]) -> None: + try: + done_task.result() + except asyncio.CancelledError: + pass + except Exception as exc: + log.error("stream.parallel_tool_call.error", { + "tool_call_id": event.tool_call_id, + "tool_name": event.tool_name, + "error": str(exc), + }) + + task.add_done_callback(_cleanup) + + async def drain_parallel_tool_calls(self) -> None: + """Wait for foreground parallel tool-calls before returning the step.""" + if not self._parallel_tool_tasks: + return + tasks = list(self._parallel_tool_tasks.values()) + await asyncio.gather(*tasks, return_exceptions=True) + @staticmethod def _compute_visible_delta(previous: str, current: str) -> str: if current.startswith(previous): diff --git a/flocks/skill/installer.py b/flocks/skill/installer.py index e340c33cb..7f1a42e28 100644 --- a/flocks/skill/installer.py +++ b/flocks/skill/installer.py @@ -2,11 +2,13 @@ Skill Installer Handles: -1. Installing skills from external sources (GitHub, raw URL, local path, clawhub) +1. Installing skills from external sources (GitHub, raw URL, local path, clawhub, + skills.sh, SafeSkill) 2. Installing a skill's declared tool dependencies (brew, npm, uv, pip, go) Source scheme routing: - safeskill: → SafeSkill registry API (reserved for future) + safeskill: → SafeSkill CLI staging import + skills-sh: → skills.sh registry via GitHub source clawhub: → clawhub.com registry API github:/ → GitHub raw download https://... → Direct HTTP download @@ -75,12 +77,27 @@ def _resolve_install_root(scope: str) -> Path: return _user_skills_root() +def _normalize_github_repo_path(repo_path: str) -> str: + """Normalize GitHub web paths to owner/repo[/skill-dir].""" + parts = repo_path.strip("/").split("/") + if len(parts) < 4 or parts[2] not in {"blob", "tree"}: + return repo_path.strip("/") + + owner, repo, view_kind, _branch, *subpath_parts = parts + if view_kind == "blob" and subpath_parts[-1:] == ["SKILL.md"]: + subpath_parts = subpath_parts[:-1] + + subpath = "/".join(subpath_parts) + return f"{owner}/{repo}/{subpath}".rstrip("/") + + def _resolve_source(source: str) -> dict: """ Parse source string into a typed dict with keys: kind, value. Supported kinds: - safeskill – reserved, future SafeSkill registry + skills_sh – skills.sh registry + safeskill – SafeSkill CLI staging import clawhub – clawhub.com registry github – GitHub raw download url – arbitrary HTTPS URL @@ -88,16 +105,33 @@ def _resolve_source(source: str) -> dict: """ source = source.strip() + if source.startswith(("skills-sh:", "skills.sh:")): + prefix = "skills-sh:" if source.startswith("skills-sh:") else "skills.sh:" + return {"kind": "skills_sh", "value": source[len(prefix):]} + if source.startswith("safeskill:"): return {"kind": "safeskill", "value": source[len("safeskill:"):]} + if source.startswith("safeskill://"): + return {"kind": "safeskill", "value": source} + if source.startswith("clawhub:"): return {"kind": "clawhub", "value": source[len("clawhub:"):]} if source.startswith("github:"): - return {"kind": "github", "value": source[len("github:"):]} + return { + "kind": "github", + "value": _normalize_github_repo_path(source[len("github:"):]), + } if source.startswith(("http://", "https://")): + skills_sh_match = re.match( + r"https?://(?:www\.)?skills\.sh/?(?P.*)$", + source, + ) + if skills_sh_match: + return {"kind": "skills_sh", "value": skills_sh_match.group("id")} + # Detect GitHub URLs and handle them specially gh_match = re.match( r"https?://github\.com/([^/]+/[^/]+)(?:/tree/[^/]+)?(/.*)?$", @@ -106,7 +140,8 @@ def _resolve_source(source: str) -> dict: if gh_match: repo = gh_match.group(1).rstrip("/") subpath = (gh_match.group(2) or "").strip("/") - return {"kind": "github", "value": f"{repo}/{subpath}" if subpath else repo} + repo_path = f"{repo}/{subpath}" if subpath else repo + return {"kind": "github", "value": _normalize_github_repo_path(repo_path)} return {"kind": "url", "value": source} if source.startswith(("/", "./", "../", "~/")): @@ -135,6 +170,7 @@ async def install_from_source( cls, source: str, scope: str = "global", + yes: bool = False, ) -> SkillInstallResult: """ Install a skill from an external source. @@ -143,6 +179,8 @@ async def install_from_source( source: Source string (URL, GitHub, clawhub:, local path …) scope: "global" → ~/.flocks/plugins/skills/ "project" → .flocks/plugins/skills/ (cwd) + yes: When True, pass -y to downstream CLIs (e.g. `skills add`) + so installs can run non-interactively. Returns: SkillInstallResult @@ -151,9 +189,11 @@ async def install_from_source( kind = resolved["kind"] value = resolved["value"] - log.info("skill.install.start", {"source": source, "kind": kind, "scope": scope}) + log.info("skill.install.start", {"source": source, "kind": kind, "scope": scope, "yes": yes}) - if kind == "safeskill": + if kind == "skills_sh": + return await cls._install_from_skills_sh(value, scope, yes=yes) + elif kind == "safeskill": return await cls._install_from_safeskill(value, scope) elif kind == "clawhub": return await cls._install_from_clawhub(value, scope) @@ -170,16 +210,272 @@ async def install_from_source( ) @classmethod - async def _install_from_safeskill(cls, name: str, scope: str) -> SkillInstallResult: - """Reserved for future SafeSkill registry integration.""" + async def _install_from_skills_sh( + cls, + identifier: str, + scope: str, + yes: bool = False, + ) -> SkillInstallResult: + """Install a skills.sh skill via npx staging, then GitHub fallback.""" + normalized = cls._normalize_skills_sh_identifier(identifier) + if not normalized: + return SkillInstallResult( + success=False, + error="skills.sh skill identifier is required, e.g. skills-sh:owner/repo/skill", + ) + + staged = await cls._install_from_skills_sh_cli(normalized, scope, yes=yes) + if staged.success: + return staged + + if normalized.count("/") >= 2: + result = await cls._install_from_github(normalized, scope) + if result.success: + return result + + resolved = await cls._resolve_skills_sh_github_identifier(normalized) + if not resolved: + return SkillInstallResult( + success=False, + error=( + f"Could not resolve skills.sh skill {identifier!r}. " + "Use an identifier like skills-sh:owner/repo/skill-name." + ), + ) + return await cls._install_from_github(resolved, scope) + + @classmethod + async def _install_from_skills_sh_cli( + cls, + identifier: str, + scope: str, + yes: bool = False, + ) -> SkillInstallResult: + """Run `npx skills add` in staging so default agent-dir installs are imported.""" + npx = shutil.which("npx") + if not npx: + return SkillInstallResult( + success=False, + error="npx is not available for skills.sh CLI install", + ) + + with tempfile.TemporaryDirectory(prefix="flocks-skills-sh-") as tmp: + staging = Path(tmp) + env = os.environ.copy() + env["HOME"] = str(staging) + env["XDG_CONFIG_HOME"] = str(staging / ".config") + cmd = [npx, "-y", "skills", "add", identifier] + if yes: + cmd.append("-y") + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=str(staging), + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout_b, stderr_b = await proc.communicate() + output = ( + stdout_b.decode(errors="replace") + + stderr_b.decode(errors="replace") + ).strip() + if proc.returncode != 0: + return SkillInstallResult( + success=False, + error=output or f"skills.sh CLI failed with exit {proc.returncode}", + ) + + imported = cls._import_staged_skill_dirs(staging, scope) + if not imported: + return SkillInstallResult( + success=False, + error=( + "skills.sh CLI completed but no SKILL.md files were found " + "in staged agent skill directories." + ), + ) + + Skill.clear_cache() + names = ", ".join(name for name, _ in imported) return SkillInstallResult( - success=False, - error=( - "SafeSkill registry is not yet available. " - "Use a GitHub URL or clawhub: instead." - ), + success=True, + skill_name=imported[0][0], + location=str(imported[0][1]), + message=f"Imported skills.sh skill(s) into Flocks: {names}", + ) + + @classmethod + async def _install_from_safeskill(cls, source: str, scope: str) -> SkillInstallResult: + """Run SafeSkill CLI in a staging directory and import its agent output.""" + npx = shutil.which("npx") + if not npx: + return SkillInstallResult( + success=False, + error="npx is required for safeskill installs. Install Node.js/npm first.", + ) + + source = source.strip() + if not source: + return SkillInstallResult( + success=False, + error=( + "safeskill source is required, e.g. " + "safeskill:safeskill://official/acme/code-review" + ), + ) + + with tempfile.TemporaryDirectory(prefix="flocks-safeskill-") as tmp: + staging = Path(tmp) + cmd = [ + npx, + "-y", + "@safeskill/cli", + "add", + source, + "--copy", + "-y", + "-a", + "universal", + ] + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=str(staging), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout_b, stderr_b = await proc.communicate() + output = ( + stdout_b.decode(errors="replace") + + stderr_b.decode(errors="replace") + ).strip() + if proc.returncode != 0: + return SkillInstallResult( + success=False, + error=output or f"SafeSkill CLI failed with exit {proc.returncode}", + ) + + imported = cls._import_staged_skill_dirs(staging, scope) + if not imported: + return SkillInstallResult( + success=False, + error=( + "SafeSkill CLI completed but no SKILL.md files were found " + "in the staging agent directories." + ), + ) + + Skill.clear_cache() + names = ", ".join(name for name, _ in imported) + return SkillInstallResult( + success=True, + skill_name=imported[0][0], + location=str(imported[0][1]), + message=f"Imported SafeSkill skill(s) into Flocks: {names}", ) + @staticmethod + def _normalize_skills_sh_identifier(identifier: str) -> str: + """Normalize skills.sh prefixes and URLs to owner/repo/skill-path.""" + value = identifier.strip().strip("/") + for prefix in ("skills-sh/", "skills.sh/", "skils-sh/", "skils.sh/"): + if value.startswith(prefix): + value = value[len(prefix):] + break + if value.startswith("https://www.skills.sh/"): + value = value[len("https://www.skills.sh/"):] + if value.startswith("http://www.skills.sh/"): + value = value[len("http://www.skills.sh/"):] + if value.startswith("https://skills.sh/"): + value = value[len("https://skills.sh/"):] + if value.startswith("http://skills.sh/"): + value = value[len("http://skills.sh/"):] + return value.strip("/") + + @classmethod + async def _resolve_skills_sh_github_identifier(cls, identifier: str) -> Optional[str]: + """Resolve a skills.sh detail page to a GitHub owner/repo[/skill] path.""" + try: + import httpx + except ImportError: + return None + + normalized = cls._normalize_skills_sh_identifier(identifier) + if normalized.count("/") < 2: + return None + + try: + async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client: + resp = await client.get(f"https://skills.sh/{normalized}") + if resp.status_code != 200: + return None + except Exception: + return None + + install_match = re.search( + r"npx\s+skills\s+add\s+(?Phttps?://github\.com/[^\s<]+|[^\s<]+)" + r"(?:\s+--skill\s+(?P[^\s<]+))?", + resp.text, + flags=re.IGNORECASE, + ) + if install_match: + repo = install_match.group("repo").strip().strip("\"'") + skill = (install_match.group("skill") or "").strip().strip("\"'") + repo = cls._github_repo_slug(repo) + if repo: + return f"{repo}/{skill}" if skill else repo + + parts = normalized.split("/", 2) + repo = f"{parts[0]}/{parts[1]}" + skill_path = parts[2] + return f"{repo}/{skill_path}" + + @staticmethod + def _github_repo_slug(value: str) -> Optional[str]: + value = value.strip().strip("/") + if value.startswith("https://github.com/"): + value = value[len("https://github.com/"):] + elif value.startswith("http://github.com/"): + value = value[len("http://github.com/"):] + parts = value.split("/") + if len(parts) >= 2: + return f"{parts[0]}/{parts[1]}" + return None + + @classmethod + def _import_staged_skill_dirs(cls, staging: Path, scope: str) -> List[tuple[str, Path]]: + """Copy staged SafeSkill agent directories into Flocks skill storage.""" + install_root = _resolve_install_root(scope) + imported: List[tuple[str, Path]] = [] + seen: set[Path] = set() + candidate_roots = [ + staging / ".agents" / "skills", + staging / ".claude" / "skills", + staging / ".cursor" / "skills", + staging / "skills", + ] + for root in candidate_roots: + if not root.exists(): + continue + for skill_md in root.rglob("SKILL.md"): + skill_dir = skill_md.parent.resolve() + if skill_dir in seen: + continue + seen.add(skill_dir) + try: + content = skill_md.read_text(encoding="utf-8") + except Exception: + continue + data = Skill._parse_frontmatter(content) + name = (data.get("name") or skill_dir.name).strip() + if not name or not Skill._is_valid_name(name): + continue + dest = install_root / name + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(skill_dir, dest) + imported.append((name, dest / "SKILL.md")) + return imported + @classmethod async def _install_from_clawhub(cls, name: str, scope: str) -> SkillInstallResult: """Download a skill from clawhub.ai registry (ZIP bundle).""" @@ -211,7 +507,14 @@ async def _install_from_clawhub(cls, name: str, scope: str) -> SkillInstallResul success=False, error=f"clawhub returned HTTP {resp.status_code} for skill '{name}'", ) - content_type = resp.headers.get("content-type", "") + content_type = "" + headers = getattr(resp, "headers", None) + if headers is not None: + header_value = headers.get("content-type", "") + if isinstance(header_value, str): + content_type = header_value + elif asyncio.iscoroutine(header_value): + header_value.close() if "text/html" in content_type: return SkillInstallResult( success=False, @@ -320,10 +623,16 @@ async def _install_from_github(cls, repo_path: str, scope: str) -> SkillInstallR # Candidate directory paths to try (in order) if subpath: - candidate_paths = [subpath, f"skills/{subpath}"] + candidate_paths = [ + subpath, + f"skills/{subpath}", + f".agents/skills/{subpath}", + f".claude/skills/{subpath}", + ] else: candidate_paths = [""] + errors: List[str] = [] async with httpx.AsyncClient( timeout=30, follow_redirects=True, @@ -336,12 +645,65 @@ async def _install_from_github(cls, repo_path: str, scope: str) -> SkillInstallR ) if result.success: return result + if result.error: + errors.append(result.error) + + # Unauthenticated GitHub Contents API can return 403 rate-limit + # errors while raw.githubusercontent.com still works. In that case + # install the SKILL.md directly instead of reporting a misleading + # "directory not found" error. + for branch in ("main", "master"): + for dir_path in candidate_paths: + result = await cls._download_github_skill_md_raw( + client, owner, repo, branch, dir_path, scope + ) + if result.success: + return result + if result.error: + errors.append(result.error) + if errors and any("GitHub API 403" in error for error in errors): + return SkillInstallResult( + success=False, + error=( + f"GitHub API returned 403 for {owner}/{repo}. " + "This is usually an unauthenticated API rate-limit or access issue, " + "and raw SKILL.md fallback also failed. " + f"Last error: {errors[-1]}" + ), + ) return SkillInstallResult( success=False, error=f"Could not find a skill directory in GitHub repo: {owner}/{repo}", ) + @classmethod + async def _download_github_skill_md_raw( + cls, + client: Any, + owner: str, + repo: str, + branch: str, + dir_path: str, + scope: str, + ) -> SkillInstallResult: + """Download SKILL.md through raw.githubusercontent.com as API fallback.""" + raw_path = f"{dir_path.strip('/')}/SKILL.md" if dir_path else "SKILL.md" + url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{raw_path}" + resp = await client.get(url) + if resp.status_code != 200: + return SkillInstallResult( + success=False, + error=f"Raw GitHub HTTP {resp.status_code} for {url}", + ) + hint = Path(dir_path).name if dir_path else repo + result = cls._save_skill_content(resp.text, scope, skill_name_hint=hint) + if result.success: + result.message = ( + f"{result.message} (installed from raw GitHub SKILL.md fallback)" + ) + return result + @classmethod async def _download_github_dir( cls, @@ -497,7 +859,14 @@ async def _install_from_url( success=False, error=f"HTTP {resp.status_code} fetching {url}", ) - content_type = resp.headers.get("content-type", "") + content_type = "" + headers = getattr(resp, "headers", None) + if headers is not None: + header_value = headers.get("content-type", "") + if isinstance(header_value, str): + content_type = header_value + elif asyncio.iscoroutine(header_value): + header_value.close() if "text/html" in content_type: return SkillInstallResult( success=False, diff --git a/flocks/task/background.py b/flocks/task/background.py index ca1c60e0e..e5a62d0ef 100644 --- a/flocks/task/background.py +++ b/flocks/task/background.py @@ -35,6 +35,7 @@ class BackgroundTask: agent: str parent_session_id: Optional[str] = None parent_message_id: Optional[str] = None + parent_call_id: Optional[str] = None parent_agent: Optional[str] = None parent_model: Optional[dict] = None model: Optional[dict] = None @@ -47,6 +48,9 @@ class BackgroundTask: completed_at: Optional[int] = None last_activity_at: Optional[int] = None # 最近一次有交互的时间戳（ms），用于不活跃超时检测 allow_user_questions: bool = True + provider_id: Optional[str] = None + model_id: Optional[str] = None + completion_injected: bool = False @dataclass @@ -57,6 +61,7 @@ class LaunchInput: parent_session_id: Optional[str] parent_message_id: Optional[str] parent_agent: Optional[str] + parent_call_id: Optional[str] = None parent_model: Optional[dict] = None model: Optional[dict] = None model_pinned: bool = False @@ -72,6 +77,7 @@ class ResumeInput: parent_session_id: Optional[str] parent_message_id: Optional[str] parent_agent: Optional[str] + parent_call_id: Optional[str] = None parent_model: Optional[dict] = None @@ -97,6 +103,7 @@ async def launch(self, input_data: LaunchInput) -> BackgroundTask: agent=input_data.agent, parent_session_id=input_data.parent_session_id, parent_message_id=input_data.parent_message_id, + parent_call_id=input_data.parent_call_id, parent_agent=input_data.parent_agent, parent_model=input_data.parent_model, model=input_data.model, @@ -117,6 +124,7 @@ async def resume(self, input_data: ResumeInput) -> BackgroundTask: agent="continue", parent_session_id=input_data.parent_session_id, parent_message_id=input_data.parent_message_id, + parent_call_id=input_data.parent_call_id, parent_agent=input_data.parent_agent, parent_model=input_data.parent_model, session_id=input_data.session_id, @@ -133,6 +141,13 @@ async def run_existing_session( agent: str, *, allow_user_questions: bool = True, + parent_session_id: Optional[str] = None, + parent_message_id: Optional[str] = None, + parent_call_id: Optional[str] = None, + parent_agent: Optional[str] = None, + parent_model: Optional[dict] = None, + provider_id: Optional[str] = None, + model_id: Optional[str] = None, ) -> BackgroundTask: """Run the session loop on an already-created session. @@ -147,8 +162,15 @@ async def run_existing_session( description=description, prompt="", agent=agent, + parent_session_id=parent_session_id, + parent_message_id=parent_message_id, + parent_call_id=parent_call_id, + parent_agent=parent_agent, + parent_model=parent_model, session_id=session_id, allow_user_questions=allow_user_questions, + provider_id=provider_id, + model_id=model_id, ) self._tasks[task_id] = task handle = asyncio.create_task(self._run_existing_session(task, session_id)) @@ -167,6 +189,8 @@ async def _run_existing_session(self, task: BackgroundTask, session_id: str) -> session_id, callbacks, allow_user_questions=task.allow_user_questions, + provider_id=task.provider_id, + model_id=task.model_id, ) output = "" if result.last_message: @@ -174,15 +198,175 @@ async def _run_existing_session(self, task: BackgroundTask, session_id: str) -> task.output = output task.status = "completed" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) except asyncio.CancelledError: task.status = "cancelled" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) raise except Exception as exc: log.error("background.existing_session.error", {"error": str(exc), "task_id": task.id}) task.error = str(exc) task.status = "error" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) + + async def _inject_parent_completion(self, task: BackgroundTask) -> None: + """Inject completed background task output into the parent context.""" + if task.completion_injected or not task.parent_session_id: + return + + state = task.status + if state == "completed": + body_tag = "task_result" + body = task.output or "(no output)" + summary = f"Background task completed: {task.description}" + elif state == "cancelled": + body_tag = "task_error" + body = task.error or "Task was cancelled." + summary = f"Background task cancelled: {task.description}" + else: + body_tag = "task_error" + body = task.error or "Task failed without an error message." + summary = f"Background task failed: {task.description}" + + content = ( + f'\n' + f" {summary}\n" + f" <{body_tag}>{body}\n" + "" + ) + try: + await Message.create( + session_id=task.parent_session_id, + role=MessageRole.USER, + content=content, + agent=task.parent_agent or "rex", + model=task.parent_model, + synthetic=True, + part_metadata={ + "kind": "background_task_result", + "task_id": task.id, + "session_id": task.session_id, + "status": state, + }, + ) + await self._update_parent_tool_part(task) + task.completion_injected = True + self._schedule_parent_resume(task) + except Exception as exc: + log.warn("background.inject_completion_failed", { + "task_id": task.id, + "parent_session_id": task.parent_session_id, + "error": str(exc), + }) + + def _schedule_parent_resume(self, task: BackgroundTask) -> None: + """Kick the parent session so Rex consumes injected background results.""" + if not task.parent_session_id: + return + if task.status not in ("completed", "error"): + return + if SessionLoop.is_running(task.parent_session_id): + log.info("background.parent_resume.already_running", { + "task_id": task.id, + "parent_session_id": task.parent_session_id, + }) + return + + async def _run_parent() -> None: + try: + from flocks.server.routes.event import publish_event + from flocks.session.session_loop import LoopCallbacks + + model = task.parent_model or {} + result = await SessionLoop.run( + session_id=task.parent_session_id, + provider_id=model.get("providerID"), + model_id=model.get("modelID"), + agent_name=task.parent_agent, + callbacks=LoopCallbacks(event_publish_callback=publish_event), + ) + log.info("background.parent_resume.completed", { + "task_id": task.id, + "parent_session_id": task.parent_session_id, + "result_action": result.action, + }) + except Exception as exc: + log.warn("background.parent_resume.failed", { + "task_id": task.id, + "parent_session_id": task.parent_session_id, + "error": str(exc), + }) + + asyncio.create_task(_run_parent()) + + async def _update_parent_tool_part(self, task: BackgroundTask) -> None: + """Mark the original background launch tool part with child completion.""" + if not task.parent_session_id or not task.parent_message_id or not task.parent_call_id: + return + + try: + from flocks.session.message import ToolPart, ToolStateCompleted, ToolStateError + + parts = await Message.parts(task.parent_message_id, task.parent_session_id) + target = next( + ( + part for part in parts + if isinstance(part, ToolPart) + and getattr(part, "callID", None) == task.parent_call_id + ), + None, + ) + if target is None: + return + + previous_state = getattr(target, "state", None) + previous_input = getattr(previous_state, "input", {}) or {} + previous_metadata = dict(getattr(previous_state, "metadata", {}) or {}) + previous_metadata.update({ + "sessionId": task.session_id, + "taskId": task.id, + "status": task.status, + "background": True, + }) + start_time = None + previous_time = getattr(previous_state, "time", None) + if isinstance(previous_time, dict): + start_time = previous_time.get("start") + end_time = int(datetime.now().timestamp() * 1000) + + if task.status == "completed": + state = ToolStateCompleted( + status="completed", + input=previous_input, + output=task.output or "", + title=task.description, + metadata=previous_metadata, + time={"start": start_time or end_time, "end": end_time}, + ) + else: + state = ToolStateError( + status="error", + input=previous_input, + error=task.error or f"Background task {task.status}", + metadata=previous_metadata, + time={"start": start_time or end_time, "end": end_time}, + ) + + await Message.update_part( + session_id=task.parent_session_id, + message_id=task.parent_message_id, + part_id=target.id, + state=state, + ) + except Exception as exc: + log.warn("background.update_parent_tool_part_failed", { + "task_id": task.id, + "parent_session_id": task.parent_session_id, + "parent_message_id": task.parent_message_id, + "error": str(exc), + }) async def wait_for(self, task_id: str, timeout_ms: Optional[int] = None) -> Optional[BackgroundTask]: handle = self._task_handles.get(task_id) @@ -414,15 +598,18 @@ async def _run_task(self, task: BackgroundTask, input_data: LaunchInput) -> None task.output = output task.status = "completed" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) except asyncio.CancelledError: task.status = "cancelled" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) raise except Exception as exc: log.error("background.task.error", {"error": str(exc), "task_id": task.id}) task.error = str(exc) task.status = "error" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) async def _run_resume(self, task: BackgroundTask, input_data: ResumeInput) -> None: async with self._semaphore: @@ -451,15 +638,18 @@ async def _run_resume(self, task: BackgroundTask, input_data: ResumeInput) -> No task.output = output task.status = "completed" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) except asyncio.CancelledError: task.status = "cancelled" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) raise except Exception as exc: log.error("background.resume.error", {"error": str(exc), "task_id": task.id}) task.error = str(exc) task.status = "error" task.completed_at = int(datetime.now().timestamp() * 1000) + await self._inject_parent_completion(task) def _create_manager() -> BackgroundManager: diff --git a/flocks/tool/__init__.py b/flocks/tool/__init__.py index 96280035d..232388155 100644 --- a/flocks/tool/__init__.py +++ b/flocks/tool/__init__.py @@ -13,9 +13,8 @@ P1 Tools: - webfetch: Fetch web content -- todoread/todowrite: TODO list management +- todo: TODO list management - question: User interaction -- plan_enter/plan_exit: Plan mode switching P2 Tools: - task: Subagent execution diff --git a/flocks/tool/agent/delegate_task.py b/flocks/tool/agent/delegate_task.py index 27ccf30a4..9cffcdd16 100644 --- a/flocks/tool/agent/delegate_task.py +++ b/flocks/tool/agent/delegate_task.py @@ -19,7 +19,6 @@ CATEGORY_PROMPT_APPENDS, CATEGORY_DESCRIPTIONS, ) -from flocks.task.background import get_background_manager, LaunchInput, ResumeInput from flocks.session.session import Session from flocks.session.message import Message, MessageRole from flocks.session.session_loop import SessionLoop @@ -38,21 +37,37 @@ async def _subagent_session_permissions(agent_name: str) -> list: from flocks.agent.registry import Agent from flocks.session.session import PermissionRule as SessionPermissionRule - agent = await Agent.get(agent_name) + def deny_nested_delegation() -> list: + return [ + SessionPermissionRule(permission="delegate_task", action="deny", pattern="*"), + SessionPermissionRule(permission="task", action="deny", pattern="*"), + ] + + try: + agent = await Agent.get(agent_name) + except Exception as exc: + log.debug("delegate_task.subagent_permission_agent_load_failed", { + "agent": agent_name, + "error": str(exc), + }) + agent = None rules: list = [] if agent_name != "prometheus": rules.append(SessionPermissionRule(permission="question", action="deny", pattern="*")) - if agent and agent.permission: - for rule in agent.permission: - level = rule.level.value if hasattr(rule.level, "value") else str(rule.level) + agent_permissions = getattr(agent, "permission", None) + if agent and agent_permissions: + for rule in agent_permissions: + raw_level = getattr(rule, "level", None) or getattr(rule, "action", None) or "allow" + level = raw_level.value if hasattr(raw_level, "value") else str(raw_level) rules.append( SessionPermissionRule( - permission=rule.permission or "*", + permission=getattr(rule, "permission", None) or "*", action=level, - pattern=rule.pattern or "*", + pattern=getattr(rule, "pattern", None) or "*", ) ) + rules.extend(deny_nested_delegation()) return rules if agent_name == "prometheus": @@ -63,6 +78,7 @@ async def _subagent_session_permissions(agent_name: str) -> list: ]) elif not rules: rules.append(SessionPermissionRule(permission="question", action="deny", pattern="*")) + rules.extend(deny_nested_delegation()) return rules @@ -223,6 +239,7 @@ def _derive_task_description( return f"continue task {session_id}" return "delegate task" + # ------------------------------------------------------------------ # Tool definition # ------------------------------------------------------------------ @@ -238,12 +255,16 @@ def _derive_task_description( - Provide a clear description (3-5 words) - Provide detailed prompt with context - Pass session_id to continue a previous agent with full context -- run_in_background=false: (default) waits for completion and returns results inline +- Background subagent execution is disabled. Do not set run_in_background=true. +- Foreground execution is always used: the tool waits for completion and returns results inline. +- For independent parallel work needed this turn, emit multiple sibling + foreground delegate_task/task tool calls in the same assistant response. + The runtime executes them concurrently and the webui renders each as its + own DelegateTaskCard. REQUIRED: prompt. LOAD_SKILLS is optional and defaults to []. DESCRIPTION is optional and will be auto-derived when omitted. -RUN_IN_BACKGROUND defaults to false (sync). if true, need: returns task_id immediately, collect results later with background_output USE EITHER subagent_type OR category — NEVER both simultaneously. """ @@ -268,15 +289,9 @@ def _derive_task_description( ToolParameter( name="prompt", type=ParameterType.STRING, - description="REQUIRED. Full detailed prompt for the agent.", + description="Full detailed prompt for the subagent.", required=True, ), - ToolParameter( - name="run_in_background", - type=ParameterType.BOOLEAN, - description="Optional. true=async (returns task_id immediately), false=sync (blocks until done). Defaults to false.", - required=False, - ), ToolParameter( name="category", type=ParameterType.STRING, @@ -301,21 +316,43 @@ def _derive_task_description( description="Optional command name for tracking", required=False, ), + ToolParameter( + name="model", + type=ParameterType.STRING, + description="Optional model override (provider/model or model)", + required=False, + ), ], ) async def delegate_task_tool( ctx: ToolContext, - prompt: str, + prompt: Optional[str] = None, load_skills: Optional[List[str]] = None, description: Optional[str] = None, - run_in_background: Optional[bool] = False, + # Internal-only: not exposed in the public schema. The registry rejects + # `run_in_background=True` at the schema layer for any caller, but legacy + # in-process call paths (e.g. `task.py` alias) may still pass it through. + # This guard is the second line of defense. + run_in_background: bool = False, category: Optional[str] = None, subagent_type: Optional[str] = None, session_id: Optional[str] = None, command: Optional[str] = None, + model: Optional[str] = None, ) -> ToolResult: - if run_in_background is None: - run_in_background = False + if run_in_background: + return ToolResult( + success=False, + error=( + "Background subagent execution is disabled. " + "Use foreground delegate_task/task calls; emit multiple sibling calls " + "in the same assistant turn for parallel work." + ), + ) + + if not prompt: + return ToolResult(success=False, error="prompt is required") + load_skills = [str(name).strip() for name in (load_skills or []) if str(name).strip()] description = _derive_task_description(description, prompt, subagent_type, category, session_id) if category and subagent_type: @@ -351,30 +388,10 @@ async def delegate_task_tool( category_configs = {**DEFAULT_CATEGORIES, **(cfg.categories or {})} category_prompt_append = None category_model = None + explicit_model = _parse_model(model) agent_to_use: Optional[str] = None if session_id: - if run_in_background: - manager = get_background_manager() - task = await manager.resume( - ResumeInput( - session_id=session_id, - prompt=prompt, - parent_session_id=ctx.session_id, - parent_message_id=ctx.message_id, - parent_agent=ctx.agent, - ) - ) - ctx.metadata({"title": f"Continue: {description}", "metadata": {"sessionId": task.session_id}}) - output = ( - "Background task continued.\n\n" - f"Task ID: {task.id}\n" - f"Description: {task.description}\n" - f"Agent: {task.agent}\n" - f"Status: {task.status}\n\n" - f'\nsession_id: {task.session_id}\n' - ) - return ToolResult(success=True, output=output, title=description, metadata={"sessionId": task.session_id}) # Sync continuation session = await Session.get_by_id(session_id) if not session: @@ -406,7 +423,7 @@ async def delegate_task_tool( if not config: available = ", ".join(category_configs.keys()) return ToolResult(success=False, error=f'Unknown category "{category}". Available: {available}') - raw_model = _parse_model(config.get("model") if isinstance(config, dict) else getattr(config, "model", None)) + raw_model = explicit_model or _parse_model(config.get("model") if isinstance(config, dict) else getattr(config, "model", None)) category_model = _validate_category_model(raw_model, category) if raw_model and not category_model: log.info("delegate_task.using_parent_model", { @@ -434,6 +451,7 @@ async def delegate_task_tool( error=f'Agent "{subagent_type}" cannot be delegated to (it may be a primary agent or restricted).', ) agent_to_use = subagent_type + category_model = explicit_model system_parts = [] if skill_result["content"]: @@ -443,38 +461,12 @@ async def delegate_task_tool( system_content = "\n\n".join(system_parts) if system_parts else "" full_prompt = f"{system_content}\n\n{prompt}" if system_content else prompt - if run_in_background: - manager = get_background_manager() - task = await manager.launch( - LaunchInput( - description=description, - prompt=full_prompt, - agent=agent_to_use, - parent_session_id=ctx.session_id, - parent_message_id=ctx.message_id, - parent_agent=ctx.agent, - model=category_model, - model_pinned=False, - category=category, - ) - ) - ctx.metadata({"title": description, "metadata": {"sessionId": task.session_id}}) - output = ( - "Background task launched successfully.\n\n" - f"Task ID: {task.id}\n" - f"Description: {task.description}\n" - f"Agent: {task.agent}\n" - f"Status: {task.status}\n\n" - f'\nsession_id: {task.session_id}\n' - ) - return ToolResult(success=True, output=output, title=description, metadata={"sessionId": task.session_id}) - # Sync execution parent_session = await Session.get_by_id(ctx.session_id) if not parent_session: return ToolResult(success=False, error="Parent session not found") - created = await Session.create( + create_kwargs = dict( project_id=parent_session.project_id, directory=parent_session.directory, title=f"{description} (@{agent_to_use} subagent)", @@ -483,6 +475,13 @@ async def delegate_task_tool( permission=await _subagent_session_permissions(agent_to_use), category="task", ) + if category_model and category_model.get("providerID") and category_model.get("modelID"): + create_kwargs.update( + provider=category_model["providerID"], + model=category_model["modelID"], + model_pinned=bool(explicit_model), + ) + created = await Session.create(**create_kwargs) await Message.create( session_id=created.id, role=MessageRole.USER, diff --git a/flocks/tool/agent/task.py b/flocks/tool/agent/task.py index 48e5c7242..b934de335 100644 --- a/flocks/tool/agent/task.py +++ b/flocks/tool/agent/task.py @@ -1,136 +1,32 @@ -""" -Task Tool - Subagent execution - -Launches specialized subagents for complex, multi-step tasks. -Supports both synchronous (blocking) and background (async) execution. +"""Compatibility alias for delegate_task. -Model resolution priority for child sessions: - 1. Explicit ``model`` param (WebUI override, format: "provider/model" or "model") - 2. Agent-specific model from AgentInfo.model (set in flocks.json agent config) - 3. Parent session's pinned model/provider - 4. Global default LLM (``default_models.llm`` in config) - 5. Environment / hardcoded fallback +The runtime keeps ``task`` as a registered tool name for workflow/backward +compatibility, but all scheduling behavior lives in ``delegate_task``. +Background subagent execution is disabled; run synchronously and emit +multiple sibling tool calls in one assistant turn for parallel work. """ -from typing import Optional, Dict, Tuple +from __future__ import annotations + +from typing import Optional +from flocks.tool.agent.delegate_task import delegate_task_tool from flocks.tool.registry import ( - ToolRegistry, ToolCategory, ToolParameter, ParameterType, ToolResult, ToolContext + ParameterType, + ToolCategory, + ToolContext, + ToolParameter, + ToolRegistry, + ToolResult, ) -from flocks.agent.registry import is_delegatable -from flocks.task.background import get_background_manager, LaunchInput, ResumeInput -from flocks.session.session import Session -from flocks.session.message import Message, MessageRole -from flocks.session.session_loop import SessionLoop -from flocks.tool.subagent_result import format_sync_subagent_result -from flocks.utils.log import Log - - -log = Log.create(service="tool.task") - - -# ------------------------------------------------------------------ -# Model resolution helpers -# ------------------------------------------------------------------ - -async def _resolve_child_model( - agent_name: str, - parent_session, - model_override: Optional[str] = None, -) -> Tuple[Optional[str], Optional[str], str]: - """Resolve ``(provider_id, model_id, source)`` for a child subagent session. - - Priority: - 1. ``model_override`` — explicit param from WebUI (format "provider/model" or "model") - 2. Agent model override from Storage (set via WebUI) - 3. Agent-specific model from ``AgentInfo.model`` - 4. Parent session's pinned model/provider - 5. Global default LLM from config - """ - provider: Optional[str] = None - model: Optional[str] = None - source = "unknown" - - # 1. Explicit override (WebUI) - if model_override: - if "/" in model_override: - provider, model = model_override.split("/", 1) - else: - model = model_override - source = "explicit" - - # 2. Agent model override from Storage (set via WebUI) - if not model: - try: - from flocks.storage.storage import Storage - overrides = await Storage.read("agent/model_overrides") - if isinstance(overrides, dict) and agent_name in overrides: - override = overrides[agent_name] - override_provider = override.get('providerID') - override_model = override.get('modelID') - if override_provider and override_model: - provider = override_provider - model = override_model - source = "agent_override" - except Exception: - pass - - # 3. Agent-level model from registry - if not model: - try: - from flocks.agent.registry import Agent - agent_info = await Agent.get(agent_name) - if agent_info and agent_info.model: - provider = provider or agent_info.model.provider_id - model = agent_info.model.model_id - source = "agent" - except Exception: - pass - # 4. Inherit only explicit parent pins - if (not model or not provider) and parent_session and Session.has_pinned_model(parent_session): - provider = provider or getattr(parent_session, "provider", None) - model = model or getattr(parent_session, "model", None) - if provider and model: - source = "parent_session" - # 5. Global default LLM - if not model or not provider: - try: - from flocks.config.config import Config - default_llm = await Config.resolve_default_llm() - if default_llm: - provider = provider or default_llm.get("provider_id") - model = model or default_llm.get("model_id") - if provider and model and source == "unknown": - source = "config" - except Exception: - pass +DESCRIPTION = """Compatibility alias for delegate_task. - return provider, model, source - - -def _model_dict(provider: Optional[str], model: Optional[str]) -> Optional[Dict[str, str]]: - """Build the ``model`` dict expected by ``LaunchInput``.""" - if not provider and not model: - return None - d: Dict[str, str] = {} - if provider: - d["providerID"] = provider - if model: - d["modelID"] = model - return d - - -# ------------------------------------------------------------------ -# Tool definition -# ------------------------------------------------------------------ - -DESCRIPTION = """Launch a new agent to handle complex, multi-step tasks autonomously. - -Routing (important): -- If the user did NOT explicitly request the `task` tool, prefer `delegate_task` for spawning sub-agents (category or subagent_type). -- Use `task` only when the user clearly asks to use the `task` tool. +Use delegate_task directly for new prompts. Workflows may continue using task; +it accepts the same single-subagent shape and forwards it to delegate_task. +Background subagent execution is disabled; run synchronously and emit multiple +sibling tool calls in one assistant turn for parallel work. """ @@ -142,226 +38,76 @@ def _model_dict(provider: Optional[str], model: Optional[str]) -> Optional[Dict[ ToolParameter( name="description", type=ParameterType.STRING, - description="A short (3-5 words) description of the task", - required=True, + description="Optional short task description (3-5 words)", + required=False, ), ToolParameter( name="prompt", type=ParameterType.STRING, - description="The task for the agent to perform", + description="Detailed prompt for the subagent.", required=True, ), ToolParameter( name="subagent_type", type=ParameterType.STRING, - description="The type of specialized agent to use, must be a delegatable agent", - required=True, + description="Delegatable agent name. Mutually exclusive with category.", + required=False, ), ToolParameter( - name="run_in_background", - type=ParameterType.BOOLEAN, - description="true=async (returns task_id, collect with background_output), false=sync (waits for result)", + name="category", + type=ParameterType.STRING, + description="Delegate category. Mutually exclusive with subagent_type.", required=False, ), + ToolParameter( + name="load_skills", + type=ParameterType.ARRAY, + description="Optional skill names to inject into the delegated agent", + required=False, + default=[], + ), ToolParameter( name="session_id", type=ParameterType.STRING, - description="Existing task session to continue (preserves full context)", + description="Existing subagent session to continue", + required=False, + ), + ToolParameter( + name="command", + type=ParameterType.STRING, + description="Optional command name for tracking", required=False, ), ToolParameter( name="model", type=ParameterType.STRING, - description="Optional model override for the subagent (format: 'provider/model' or 'model')", + description="Optional model override (provider/model or model)", required=False, ), ], ) async def task_tool( ctx: ToolContext, - description: str, - prompt: str, - subagent_type: str, - run_in_background: Optional[bool] = False, + description: Optional[str] = None, + prompt: Optional[str] = None, + subagent_type: Optional[str] = None, + category: Optional[str] = None, + load_skills: Optional[list] = None, + run_in_background: bool = False, session_id: Optional[str] = None, + command: Optional[str] = None, model: Optional[str] = None, ) -> ToolResult: - if not description: - return ToolResult(success=False, error="description is required") - if not prompt: - return ToolResult(success=False, error="prompt is required") - - normalized = subagent_type.lower() if subagent_type else "" - if not normalized: - return ToolResult(success=False, error="subagent_type is required") - - if not is_delegatable(normalized): - return ToolResult( - success=False, - error=f'Agent "{subagent_type}" cannot be delegated to (it may be a primary agent or restricted).', - ) - - await ctx.ask( - permission="task", - patterns=[normalized], - always=["*"], - metadata={"description": description, "subagent_type": normalized}, - ) - - # Resolve parent session once (needed for model inheritance + session creation) - parent_session = await Session.get_by_id(ctx.session_id) - - # Resolve effective model for the child agent - child_provider, child_model, child_source = await _resolve_child_model( - normalized, parent_session, model_override=model, + """Forward legacy task calls to delegate_task.""" + return await delegate_task_tool( + ctx=ctx, + prompt=prompt, + load_skills=load_skills, + description=description, + run_in_background=run_in_background, + category=category, + subagent_type=subagent_type, + session_id=session_id, + command=command, + model=model, ) - child_model_pinned = ( - child_source in {"explicit", "parent_session"} - and bool(child_provider and child_model) - ) - - log.info("task.model_resolved", { - "subagent": normalized, - "provider": child_provider, - "model": child_model, - "source": child_source, - "model_pinned": child_model_pinned, - "override": model, - "parent_provider": getattr(parent_session, "provider", None) if parent_session else None, - "parent_model": getattr(parent_session, "model", None) if parent_session else None, - }) - - # --- Background resume of existing session --- - if session_id and run_in_background: - manager = get_background_manager() - task = await manager.resume( - ResumeInput( - session_id=session_id, - prompt=prompt, - parent_session_id=ctx.session_id, - parent_message_id=ctx.message_id, - parent_agent=ctx.agent, - ) - ) - ctx.metadata({"title": f"Continue: {description}", "metadata": {"sessionId": task.session_id}}) - output = ( - "Background task continued.\n\n" - f"Task ID: {task.id}\n" - f"Description: {task.description}\n" - f"Agent: {task.agent}\n" - f"Status: {task.status}\n\n" - f'Use `background_output` with task_id="{task.id}" to check progress.\n\n' - f"\nsession_id: {task.session_id}\n" - ) - return ToolResult(success=True, output=output, title=description, metadata={"sessionId": task.session_id}) - - # --- Sync continue of existing session --- - if session_id: - session = await Session.get_by_id(session_id) - if not session: - return ToolResult(success=False, error=f"Session {session_id} not found") - await Message.create( - session_id=session.id, - role=MessageRole.USER, - content=prompt, - agent=session.agent or normalized, - ) - from flocks.session.session_loop import LoopCallbacks as _LoopCbs - result = await SessionLoop.run( - session.id, - callbacks=_LoopCbs(event_publish_callback=ctx.event_publish_callback), - ) - ctx.metadata({"title": f"Continue: {description}", "metadata": {"sessionId": session.id}}) - return await format_sync_subagent_result( - description=description, - session_id=session.id, - loop_result=result, - metadata={"sessionId": session.id}, - ) - - # --- Background launch (new session) --- - if run_in_background: - manager = get_background_manager() - task = await manager.launch( - LaunchInput( - description=description, - prompt=prompt, - agent=normalized, - parent_session_id=ctx.session_id, - parent_message_id=ctx.message_id, - parent_agent=ctx.agent, - model=_model_dict(child_provider, child_model) if child_model_pinned else None, - model_pinned=child_model_pinned, - ) - ) - ctx.metadata({"title": description, "metadata": {"sessionId": task.session_id}}) - output = ( - "Background task launched successfully.\n\n" - f"Task ID: {task.id}\n" - f"Description: {task.description}\n" - f"Agent: {task.agent}\n" - f"Status: {task.status}\n\n" - f'Use `background_output` with task_id="{task.id}" to check progress.\n\n' - f"\nsession_id: {task.session_id}\n" - ) - return ToolResult(success=True, output=output, title=description, metadata={"sessionId": task.session_id}) - - # --- Sync launch (new session) --- - if not parent_session: - return ToolResult(success=False, error="Parent session not found") - - try: - create_kwargs = dict( - project_id=parent_session.project_id, - directory=parent_session.directory, - title=f"{description} (@{normalized} subagent)", - parent_id=parent_session.id, - agent=normalized, - permission=[{"permission": "question", "action": "deny", "pattern": "*"}], - category="task", - ) - if child_model_pinned: - create_kwargs.update( - model=child_model, - provider=child_provider, - model_pinned=True, - ) - created = await Session.create(**create_kwargs) - await Message.create( - session_id=created.id, - role=MessageRole.USER, - content=prompt, - agent=normalized, - ) - - from flocks.session.features.activity_forwarder import ActivityForwarder - - forwarder = ActivityForwarder( - parent_ctx=ctx, - child_session_id=created.id, - description=description, - ) - ctx.metadata({"title": description, "metadata": {"sessionId": created.id, "status": "running"}}) - - result = await SessionLoop.run( - created.id, - callbacks=forwarder.build_callbacks( - event_publish_callback=ctx.event_publish_callback, - ), - ) - tool_result = await format_sync_subagent_result( - description=description, - session_id=created.id, - loop_result=result, - metadata=forwarder.final_metadata, - ) - result_status = "completed" if tool_result.success else "error" - ctx.metadata({ - "title": description, - "metadata": {**forwarder.final_metadata, "status": result_status}, - }) - return tool_result - - except Exception as e: - log.error("task.execute.error", {"error": str(e)}) - return ToolResult(success=False, error=f"Task execution failed: {str(e)}", title=description) diff --git a/flocks/tool/catalog.py b/flocks/tool/catalog.py index 2511f930f..c9b0b9af1 100644 --- a/flocks/tool/catalog.py +++ b/flocks/tool/catalog.py @@ -44,22 +44,14 @@ class ToolCatalogMetadata(BaseModel): "schedule_task_update": ["scheduled-task", "task-management"], "schedule_task_delete": ["scheduled-task", "task-management"], "schedule_task_rerun": ["scheduled-task", "task-management"], - "todowrite": ["task-management", "progress-tracking"], - "todoread": ["task-management", "progress-tracking"], + "todo": ["task-management", "progress-tracking"], "run_workflow": ["workflow", "execution"], "run_workflow_node": ["workflow", "execution"], "question": ["user-interaction", "clarification"], "flocks_skills": ["skill", "management"], "skill_load": ["knowledge", "skill"], "tool_search": ["tool-discovery", "capability-search"], - "session_list": ["session", "history"], - "session_get": ["session", "history"], - "session_create": ["session", "management"], - "session_update": ["session", "management"], - "session_delete": ["session", "management"], - "session_archive": ["session", "management"], - "background_output": ["background-task", "process"], - "background_cancel": ["background-task", "process"], + "session_manage": ["session", "history", "management"], "memory_search": ["memory", "search"], "memory_get": ["memory", "context"], "memory_write": ["memory", "context"], @@ -67,8 +59,6 @@ class ToolCatalogMetadata(BaseModel): "add_provider": ["provider", "configuration"], "add_model": ["model", "configuration"], "run_slash_command": ["slash-command", "orchestration"], - "plan_enter": ["planning", "mode"], - "plan_exit": ["planning", "mode"], "ssh_host_cmd": ["security", "remote-execution"], "ssh_run_script": ["security", "remote-execution"], "channel_message": ["messaging", "channel"], diff --git a/flocks/tool/credential_context.py b/flocks/tool/credential_context.py index a38a1049f..9cac8a8f1 100644 --- a/flocks/tool/credential_context.py +++ b/flocks/tool/credential_context.py @@ -19,7 +19,7 @@ from contextlib import asynccontextmanager from contextvars import ContextVar -from typing import Any, AsyncIterator, Dict, Optional +from typing import Any, AsyncIterator, Dict, NamedTuple, Optional # ContextVars – per-coroutine, so concurrent calls don't interfere. @@ -33,11 +33,20 @@ "device_config_override", default=None ) -# The service_id this config override belongs to (scoping guard) +# The service_id this config override belongs to (scoping guard). +# Matched by get_config_override() for the bare service_id alias. _config_override_service: ContextVar[Optional[str]] = ContextVar( "device_config_override_service", default=None ) +# The storage_key for the same override (versioned name, e.g. "sangfor_af_v8_0_48"). +# Many handlers set SERVICE_ID to the full storage_key, not the bare service_id, +# so we keep both to avoid a key mismatch causing credential lookup to silently +# fall back to the global default config (wrong IP, wrong credentials). +_config_override_storage_key: ContextVar[Optional[str]] = ContextVar( + "device_config_override_storage_key", default=None +) + # The currently active device_id (for logging / introspection) _active_device_id: ContextVar[Optional[str]] = ContextVar( "active_device_id", default=None @@ -50,6 +59,18 @@ ) +# --------------------------------------------------------------------------- +# Internal result type for _build_overrides +# --------------------------------------------------------------------------- + +class _DeviceOverrides(NamedTuple): + """Resolved credential and config data for a single device call.""" + secret_ovr: Optional[Dict[str, str]] + config_ovr: Optional[Dict[str, Any]] + service_id: Optional[str] # bare alias e.g. "sangfor_af" + storage_key: Optional[str] # versioned e.g. "sangfor_af_v8_0_48" + + # --------------------------------------------------------------------------- # Read helpers (called from SecretManager / ConfigWriter) # --------------------------------------------------------------------------- @@ -61,11 +82,32 @@ def get_secret_override(secret_id: str) -> Optional[str]: def get_config_override(service_id: str) -> Optional[Dict[str, Any]]: - """Return device-specific config dict if an override is active for *service_id*.""" - expected = _config_override_service.get() - if expected is None or expected != service_id: + """Return device-specific config dict if an override is active for *service_id*. + + Device handlers reference their provider config under two different naming + conventions: + + * **Bare service_id** (e.g. ``"sangfor_af"``) — produced by + ``storage_key_to_service_id()`` and stored in ``_config_override_service``. + * **Versioned storage_key** (e.g. ``"sangfor_af_v8_0_48"``) — the full key + stored in the DB and in ``_config_override_storage_key``. + + Both are checked so that a handler whose ``SERVICE_ID`` uses either form + still resolves the correct per-device config rather than silently falling + back to the global default (wrong IP / wrong credentials). + """ + override = _config_override.get() + if override is None: return None - return _config_override.get() + expected_service = _config_override_service.get() + expected_storage = _config_override_storage_key.get() + # Match on bare service_id alias OR full versioned storage_key — whichever + # the calling handler's SERVICE_ID constant happens to use. + matches_service = expected_service is not None and service_id == expected_service + matches_storage = expected_storage is not None and service_id == expected_storage + if matches_service or matches_storage: + return override + return None def get_active_device_id() -> Optional[str]: @@ -90,8 +132,9 @@ async def activate_device_credentials(device_id: str) -> AsyncIterator[bool]: Yields ``True`` if activation succeeded, ``False`` if the device was not found / disabled (caller may still continue with default credentials). """ - secret_ovr, config_ovr, service_id = await _build_overrides(device_id) - if secret_ovr is None and config_ovr is None or service_id is None: + ovr = await _build_overrides(device_id) + secret_ovr, config_ovr, service_id, storage_key = ovr + if (secret_ovr is None and config_ovr is None) or service_id is None: yield False return @@ -105,6 +148,10 @@ async def activate_device_credentials(device_id: str) -> AsyncIterator[bool]: # the previous coroutine's value visible). verify_ssl = bool((config_ovr or {}).get("verify_ssl", False)) t5 = _verify_ssl_override.set(verify_ssl) + # Also store the raw storage_key so handlers whose SERVICE_ID includes the + # version suffix (e.g. "sangfor_af_v8_0_48") can still match the override + # via get_config_override() without re-querying the DB. + t6 = _config_override_storage_key.set(storage_key or None) try: yield True finally: @@ -113,24 +160,32 @@ async def activate_device_credentials(device_id: str) -> AsyncIterator[bool]: _config_override_service.reset(t3) _active_device_id.reset(t4) _verify_ssl_override.reset(t5) + _config_override_storage_key.reset(t6) -async def _build_overrides( - device_id: str, -) -> tuple[Optional[Dict[str, str]], Optional[Dict[str, Any]], Optional[str]]: +async def _build_overrides(device_id: str) -> _DeviceOverrides: """Build secret and config override dicts for *device_id*. - Returns (None, None, None) when the device is not found or disabled. - Third element is the service_id used to scope the config override. + Returns a :class:`_DeviceOverrides` named-tuple. All fields are ``None`` + when the device is not found or disabled. + + * ``service_id`` — bare alias derived by ``storage_key_to_service_id`` + (e.g. ``"sangfor_af"``). + * ``storage_key`` — full versioned key as stored in the DB + (e.g. ``"sangfor_af_v8_0_48"``). + + Both keys are propagated to ContextVars so :func:`get_config_override` + can match whichever form a handler's ``SERVICE_ID`` uses. """ + _null = _DeviceOverrides(None, None, None, None) try: from flocks.tool.device.store import get_device_credentials creds = await get_device_credentials(device_id) except Exception: - return None, None, None + return _null if creds is None: - return None, None, None + return _null storage_key: str = creds.get("storage_key", "") service_id: str = creds.get("service_id", "") @@ -182,7 +237,12 @@ async def _build_overrides( config_ovr["verify_ssl"] = verify_ssl config_ovr["ssl_verify"] = verify_ssl - return secret_ovr or None, config_ovr or None, service_id or None + return _DeviceOverrides( + secret_ovr=secret_ovr or None, + config_ovr=config_ovr or None, + service_id=service_id or None, + storage_key=storage_key or None, + ) def _load_credential_fields(storage_key: str) -> list[Dict[str, Any]]: diff --git a/flocks/tool/device/__init__.py b/flocks/tool/device/__init__.py index e243dd44e..975f76c4a 100644 --- a/flocks/tool/device/__init__.py +++ b/flocks/tool/device/__init__.py @@ -11,10 +11,15 @@ DeviceGroup, DeviceGroupCreate, DeviceGroupUpdate, + DeviceCredentialResponse, DeviceIntegration, DeviceIntegrationCreate, DeviceIntegrationUpdate, + DeviceTemplate, + DeviceTestRequest, DeviceTestResult, + CustomDeviceTemplateCreate, + CustomDeviceToolCreate, ) from .startup import device_startup from .store import get_device_credentials @@ -28,10 +33,15 @@ "DeviceGroup", "DeviceGroupCreate", "DeviceGroupUpdate", + "DeviceCredentialResponse", "DeviceIntegration", "DeviceIntegrationCreate", "DeviceIntegrationUpdate", + "DeviceTemplate", + "DeviceTestRequest", "DeviceTestResult", + "CustomDeviceTemplateCreate", + "CustomDeviceToolCreate", # Entry points "device_startup", "get_device_credentials", diff --git a/flocks/tool/device/intake.py b/flocks/tool/device/intake.py new file mode 100644 index 000000000..1acbc34a5 --- /dev/null +++ b/flocks/tool/device/intake.py @@ -0,0 +1,326 @@ +"""Device lifecycle orchestration. + +Routes should stay thin: this module owns persistence, secret handling, tool +state sync, and connectivity probing for device integrations. +""" +from __future__ import annotations + +import asyncio +import json +import time +import uuid +from typing import Optional + +import httpx + +from flocks.storage.storage import Storage +from flocks.tool.device.models import ( + DEFAULT_GROUP_ID, + MULTI_GROUP_ENABLED, + DeviceIntegration, + DeviceIntegrationCreate, + DeviceIntegrationUpdate, + DeviceTestRequest, + DeviceTestResult, +) +from flocks.tool.device.secrets import delete_secrets, persist_fields, resolve_for_runtime +from flocks.tool.device.store import ( + delete_device_row, + ensure_default_group, + fetch_device, + group_exists, + insert_device, + list_devices, + record_test_result, + row_to_device, + storage_key_to_service_id, + update_device_row, +) +from flocks.tool.device.sync import sync_service_tool_state + +_AUTO_INSTANCE_IGNORED_KEY = "device.auto_instance_ignored_storage_keys" +_AUTO_INSTANCE_LOCKS: dict[int, asyncio.Lock] = {} + + +class DeviceIntakeError(Exception): + status_code = 400 + + +class DeviceNotFoundError(DeviceIntakeError): + status_code = 404 + + +async def _load_auto_instance_ignored_storage_keys() -> set[str]: + raw = await Storage.get(_AUTO_INSTANCE_IGNORED_KEY) + if isinstance(raw, list): + return {str(item) for item in raw if item} + if isinstance(raw, dict): + values = raw.get("storage_keys") + if isinstance(values, list): + return {str(item) for item in values if item} + return set() + + +async def _save_auto_instance_ignored_storage_keys(storage_keys: set[str]) -> None: + await Storage.set(_AUTO_INSTANCE_IGNORED_KEY, sorted(storage_keys)) + + +async def _remember_auto_instance_ignore(storage_key: str) -> None: + if not storage_key: + return + ignored = await _load_auto_instance_ignored_storage_keys() + if storage_key in ignored: + return + ignored.add(storage_key) + await _save_auto_instance_ignored_storage_keys(ignored) + + +async def _forget_auto_instance_ignore(storage_key: str) -> None: + if not storage_key: + return + ignored = await _load_auto_instance_ignored_storage_keys() + if storage_key not in ignored: + return + ignored.remove(storage_key) + await _save_auto_instance_ignored_storage_keys(ignored) + + +def _user_device_template_storage_keys(*, refresh_templates: bool = False) -> set[str]: + from flocks.tool.device.plugin_index import list_device_templates + + return { + template.storage_key + for template in list_device_templates(refresh=refresh_templates) + if template.source == "global" and template.installed + } + + +def _auto_instance_lock() -> asyncio.Lock: + loop = asyncio.get_running_loop() + loop_id = id(loop) + lock = _AUTO_INSTANCE_LOCKS.get(loop_id) + if lock is None: + lock = asyncio.Lock() + _AUTO_INSTANCE_LOCKS[loop_id] = lock + return lock + + +async def ensure_user_device_instances(*, refresh_templates: bool = False) -> int: + """Create default device rows for user-level device plugin templates. + + Device templates discovered under ``~/.flocks/plugins/tools/device`` are + installable product definitions. The device access page's left pane shows + concrete ``device_integrations`` rows, so a user-created local template + would otherwise appear only in the right-side picker until the user manually + added an instance. Auto-provision one editable instance per user-level + template when no instance for the same storage_key exists yet. + """ + async with _auto_instance_lock(): + return await _ensure_user_device_instances_unlocked( + refresh_templates=refresh_templates, + ) + + +async def _ensure_user_device_instances_unlocked(*, refresh_templates: bool = False) -> int: + await ensure_default_group() + existing_storage_keys = {device.storage_key for device in await list_devices()} + ignored_storage_keys = await _load_auto_instance_ignored_storage_keys() + user_template_storage_keys = _user_device_template_storage_keys( + refresh_templates=refresh_templates, + ) + created = 0 + + from flocks.tool.device.plugin_index import list_device_templates + + for template in list_device_templates(refresh=False): + if template.source != "global" or not template.installed: + continue + if template.storage_key in existing_storage_keys: + continue + if template.storage_key in ignored_storage_keys: + continue + if template.storage_key not in user_template_storage_keys: + continue + + device_id = str(uuid.uuid4()) + await insert_device( + device_id=device_id, + group_id=DEFAULT_GROUP_ID, + name=template.name, + storage_key=template.storage_key, + service_id=template.service_id, + enabled=True, + verify_ssl=False, + db_fields={}, + ) + existing_storage_keys.add(template.storage_key) + created += 1 + await sync_service_tool_state(template.service_id) + + return created + + +async def create_device(body: DeviceIntegrationCreate) -> DeviceIntegration: + name = body.name.strip() + storage_key = body.storage_key.strip() + if not name: + raise ValueError("name is required") + if not storage_key: + raise ValueError("storage_key is required") + + group_id = DEFAULT_GROUP_ID if not MULTI_GROUP_ENABLED else (body.group_id or DEFAULT_GROUP_ID) + if not await group_exists(group_id): + raise ValueError(f"Group '{group_id}' does not exist") + + service_id = (body.service_id or "").strip() or storage_key_to_service_id(storage_key) + device_id = str(uuid.uuid4()) + db_fields = persist_fields(device_id, storage_key, body.fields) + + await insert_device( + device_id=device_id, + group_id=group_id, + name=name, + storage_key=storage_key, + service_id=service_id, + enabled=body.enabled, + verify_ssl=body.verify_ssl, + db_fields=db_fields, + ) + await _forget_auto_instance_ignore(storage_key) + await sync_service_tool_state(service_id) + + row = await fetch_device(device_id) + if row is None: + raise RuntimeError(f"created device '{device_id}' was not persisted") + return row_to_device(row) + + +async def update_device(device_id: str, body: DeviceIntegrationUpdate) -> DeviceIntegration: + row = await fetch_device(device_id) + if row is None: + raise DeviceNotFoundError("Device not found") + + prior_fields: dict = json.loads(row["fields"] or "{}") + + stripped_name = body.name.strip() if body.name else "" + new_name = stripped_name or row["name"] + new_enabled = body.enabled if body.enabled is not None else bool(row["enabled"]) + new_ssl = body.verify_ssl if body.verify_ssl is not None else bool(row["verify_ssl"]) + + if body.group_id and MULTI_GROUP_ENABLED and body.group_id != row["group_id"]: + if not await group_exists(body.group_id): + raise ValueError(f"Group '{body.group_id}' does not exist") + new_group_id = body.group_id + else: + new_group_id = row["group_id"] or DEFAULT_GROUP_ID + + new_fields = ( + persist_fields(device_id, row["storage_key"], body.fields, prior_db_fields=prior_fields) + if body.fields is not None + else prior_fields + ) + + await update_device_row( + device_id, + name=new_name, + group_id=new_group_id, + enabled=new_enabled, + verify_ssl=new_ssl, + db_fields=new_fields, + ) + await sync_service_tool_state(storage_key_to_service_id(row["storage_key"])) + + updated = await fetch_device(device_id) + if updated is None: + raise DeviceNotFoundError("Device not found") + return row_to_device(updated) + + +async def delete_device(device_id: str) -> None: + row = await fetch_device(device_id) + if row is None: + raise DeviceNotFoundError("Device not found") + + storage_key: str = row["storage_key"] + service_id: str = storage_key_to_service_id(storage_key) + db_fields: dict = json.loads(row["fields"] or "{}") + + if storage_key in _user_device_template_storage_keys(refresh_templates=True): + await _remember_auto_instance_ignore(storage_key) + delete_secrets(device_id, db_fields) + await delete_device_row(device_id) + await sync_service_tool_state(service_id, deleted_storage_keys=[storage_key]) + + +async def test_device( + device_id: str, + body: Optional[DeviceTestRequest] = None, +) -> DeviceTestResult: + row = await fetch_device(device_id) + if row is None: + raise DeviceNotFoundError("Device not found") + + db_fields: dict = json.loads(row["fields"] or "{}") + resolved = resolve_for_runtime(db_fields) + persisted_base_url = (resolved.get("base_url") or "").strip() + + override_base_url = (body.base_url.strip() if body and body.base_url else "") + base_url = override_base_url or persisted_base_url + + if not base_url: + host = (resolved.get("host") or "").strip() + port = (resolved.get("port") or "").strip() + if host: + has_scheme = "://" in host + if has_scheme: + base_url = f"{host}:{port}" if port else host + else: + base_url = f"https://{host}:{port}" if port else f"https://{host}" + + if not base_url: + return DeviceTestResult( + success=False, + message="未配置设备地址（base_url 或 host），请先填写", + ) + + verify_ssl = bool(body.verify_ssl) if body is not None and body.verify_ssl is not None else bool(row["verify_ssl"]) + + result = await _probe(base_url, verify_ssl=verify_ssl) + await record_test_result( + device_id, + success=result.success, + message=result.message, + latency_ms=result.latency_ms, + ) + return result + + +async def _probe(base_url: str, *, verify_ssl: bool) -> DeviceTestResult: + start = time.monotonic() + + def elapsed() -> int: + return int((time.monotonic() - start) * 1000) + + try: + async with httpx.AsyncClient(verify=verify_ssl, timeout=10.0) as client: + resp = await client.get(base_url) + ms = elapsed() + return DeviceTestResult( + success=resp.status_code < 500, + message=f"HTTP {resp.status_code}，延迟 {ms}ms", + latency_ms=ms, + ) + except httpx.ConnectError: + return DeviceTestResult( + success=False, + message=f"无法连接到 {base_url}，请检查地址是否正确", + latency_ms=elapsed(), + ) + except httpx.TimeoutException: + return DeviceTestResult( + success=False, + message="连接超时（10s），请检查网络或设备地址", + latency_ms=elapsed(), + ) + except Exception as exc: + return DeviceTestResult(success=False, message=f"测试失败：{exc}") diff --git a/flocks/tool/device/models.py b/flocks/tool/device/models.py index 5b945673c..0072653ce 100644 --- a/flocks/tool/device/models.py +++ b/flocks/tool/device/models.py @@ -2,7 +2,7 @@ from __future__ import annotations import os -from typing import Dict, Optional +from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, Field @@ -12,10 +12,10 @@ # Feature flags # --------------------------------------------------------------------------- -#: Flip to True (or set env FLOCKS_DEVICE_MULTI_GROUP=1) to unlock multi-group -#: routes. The data layer is already multi-group ready; this is UI/API gating only. +#: Multi-group (多机房) is on by default. Set env FLOCKS_DEVICE_MULTI_GROUP=0 +#: to fall back to single-room mode. MULTI_GROUP_ENABLED: bool = ( - os.environ.get("FLOCKS_DEVICE_MULTI_GROUP", "").lower() in {"1", "true", "yes"} + os.environ.get("FLOCKS_DEVICE_MULTI_GROUP", "1").lower() not in {"0", "false", "no"} ) DEFAULT_GROUP_ID = "default-room" @@ -151,7 +151,64 @@ class DeviceIntegrationUpdate(BaseModel): fields: Optional[Dict[str, str]] = None +class DeviceCredentialResponse(BaseModel): + fields: Dict[str, str] = Field(default_factory=dict) + + class DeviceTestResult(BaseModel): success: bool message: str latency_ms: Optional[int] = None + + +class DeviceTestRequest(BaseModel): + """Optional body for ``POST /devices/{id}/test``.""" + + base_url: Optional[str] = Field( + None, + description="Override the persisted base_url for this probe only", + ) + verify_ssl: Optional[bool] = Field( + None, + description="Override the persisted verify_ssl for this probe only", + ) + + +class DeviceTemplate(BaseModel): + plugin_id: str + storage_key: str + service_id: str + name: str + version: Optional[str] = None + vendor: Optional[str] = None + description: Optional[str] = None + description_cn: Optional[str] = None + credential_schema: List[Dict[str, Any]] = Field(default_factory=list) + tool_count: int = 0 + installed: bool + state: Literal["available", "installed", "updateAvailable", "localOnly", "broken"] + source: Literal["bundled", "project", "global"] + + +class CustomDeviceToolCreate(BaseModel): + name: str + description: str + description_cn: Optional[str] = None + category: Optional[str] = None + inputSchema: Optional[Dict[str, Any]] = None + parameters: Optional[List[Dict[str, Any]]] = None + handler: Dict[str, Any] + response: Optional[Dict[str, Any]] = None + requires_confirmation: Optional[bool] = None + + +class CustomDeviceTemplateCreate(BaseModel): + plugin_id: str + name: str + vendor: Optional[str] = None + service_id: str + version: Optional[str] = None + description: Optional[str] = None + description_cn: Optional[str] = None + credential_fields: List[Dict[str, Any]] = Field(default_factory=list) + tools: List[CustomDeviceToolCreate] = Field(default_factory=list) diff --git a/flocks/tool/device/plugin_index.py b/flocks/tool/device/plugin_index.py new file mode 100644 index 000000000..202805def --- /dev/null +++ b/flocks/tool/device/plugin_index.py @@ -0,0 +1,396 @@ +"""Runtime index for device plugin templates. + +The device access page consumes this module instead of hand-maintaining a +frontend catalog. The only source of device identity is plugin metadata in +``_provider.yaml`` with ``integration_type: device``. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Dict, Iterable, Optional + +import yaml + +from flocks.config.api_versioning import ( + ApiServiceDescriptor, + derive_storage_key, + discover_api_service_descriptors, +) +from flocks.hub import catalog as hub_catalog +from flocks.hub import local as hub_local +from flocks.tool.device.models import CustomDeviceTemplateCreate, DeviceTemplate +from flocks.tool.registry import ToolRegistry +from flocks.tool.schema.api_service_schema import _build_api_service_credential_schema +from flocks.tool.tool_loader import TOOL_TYPE_DEVICE, extract_provider_version +from flocks.utils.log import Log + +log = Log.create(service="device.plugin-index") + +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$") +_SAFE_SERVICE_ID = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def list_device_templates(*, refresh: bool = False) -> list[DeviceTemplate]: + """Return device templates from Hub catalog plus local descriptor discovery.""" + if refresh: + _refresh_device_plugin_runtime() + + by_key: dict[str, DeviceTemplate] = {} + for entry in hub_catalog.list_catalog(plugin_type="device"): + root = _catalog_entry_root(entry) + if root is None: + by_key[f"broken:{entry.id}"] = DeviceTemplate( + plugin_id=entry.id, + storage_key=entry.id, + service_id=entry.id, + name=entry.name or entry.id, + version=entry.version, + description=entry.description, + description_cn=entry.descriptionCn, + credential_schema=[], + tool_count=0, + installed=False, + state="broken", + source=_source_from_entry(entry), + ) + continue + + template = _template_from_plugin_root( + plugin_id=entry.id, + root=root, + state=_normalize_state(entry.state), + installed=_entry_installed(entry), + source=_source_from_entry(entry), + fallback_name=entry.name, + fallback_description=entry.description, + fallback_description_cn=entry.descriptionCn, + fallback_version=entry.installedVersion or entry.version, + ) + if template is not None: + by_key[template.storage_key] = template + + # Defensive pass for project/user device plugins that may not have Hub + # records yet. This also explicitly exercises the descriptor API so version + # and storage_key stay aligned with the runtime credential namespace. + for descriptor in discover_api_service_descriptors(refresh=False): + root = descriptor.provider_yaml.parent + provider = _read_provider_yaml(root) + if _integration_type(provider) != "device": + continue + if descriptor.storage_key in by_key: + continue + plugin_id = root.name + source = _source_from_path(root) + by_key[descriptor.storage_key] = _template_from_descriptor( + plugin_id=plugin_id, + descriptor=descriptor, + provider=provider, + state="localOnly", + installed=True, + source=source, + ) + + return sorted(by_key.values(), key=_sort_key) + + +def create_custom_device_template(body: CustomDeviceTemplateCreate) -> DeviceTemplate: + """Create a user-level device plugin package and return its new template.""" + plugin_id = body.plugin_id.strip() + service_id = body.service_id.strip() + if not _SAFE_ID.match(plugin_id): + raise ValueError("plugin_id must contain only letters, numbers, '_' or '-'") + if not _SAFE_SERVICE_ID.match(service_id): + raise ValueError("service_id must be a valid identifier") + if not body.name.strip(): + raise ValueError("name is required") + if not body.tools: + raise ValueError("at least one tool is required") + + target = hub_local.install_dir("device", plugin_id, "global") + if target.exists() and hub_local.has_install_payload("device", target): + raise FileExistsError(f"device plugin '{plugin_id}' already exists") + + provider_yaml = _custom_provider_yaml(body) + target.mkdir(parents=True, exist_ok=True) + _write_yaml(target / "_provider.yaml", provider_yaml) + + seen_tools: set[str] = set() + for tool in body.tools: + tool_name = tool.name.strip() + if not _SAFE_SERVICE_ID.match(tool_name): + raise ValueError(f"tool name '{tool.name}' must be a valid identifier") + if tool_name in seen_tools: + raise ValueError(f"duplicate tool name '{tool_name}'") + seen_tools.add(tool_name) + _write_yaml(target / f"{tool_name}.yaml", _custom_tool_yaml(tool.model_dump(exclude_none=True), service_id)) + + record = hub_local.make_record( + plugin_type="device", + plugin_id=plugin_id, + version=body.version or "0.0.0", + source="custom", + install_path=target, + scope="global", + ) + hub_local.save_installed_record(record) + + _refresh_device_plugin_runtime() + + for template in list_device_templates(refresh=False): + if template.plugin_id == plugin_id: + return template + raise RuntimeError(f"created device plugin '{plugin_id}' was not indexed") + + +def _refresh_device_plugin_runtime() -> None: + """Refresh both device descriptors and runtime tool registrations.""" + discover_api_service_descriptors(refresh=True) + try: + ToolRegistry.refresh_plugin_tools() + except Exception as exc: # pragma: no cover - defensive runtime isolation + log.warn("device.templates.tool_refresh_failed", {"error": str(exc)}) + + +def _template_from_plugin_root( + *, + plugin_id: str, + root: Path, + state: str, + installed: bool, + source: str, + fallback_name: Optional[str] = None, + fallback_description: Optional[str] = None, + fallback_description_cn: Optional[str] = None, + fallback_version: Optional[str] = None, +) -> Optional[DeviceTemplate]: + provider = _read_provider_yaml(root) + if _integration_type(provider) != "device": + return None + service_id = str(provider.get("service_id") or provider.get("name") or "").strip() + if not service_id: + return None + version = extract_provider_version(provider) or fallback_version + descriptor = ApiServiceDescriptor( + service_id=service_id, + version=version, + storage_key=derive_storage_key(service_id, version), + provider_yaml=root / "_provider.yaml", + ) + return _template_from_descriptor( + plugin_id=plugin_id, + descriptor=descriptor, + provider=provider, + state=state, + installed=installed, + source=source, + fallback_name=fallback_name, + fallback_description=fallback_description, + fallback_description_cn=fallback_description_cn, + ) + + +def _template_from_descriptor( + *, + plugin_id: str, + descriptor: ApiServiceDescriptor, + provider: Dict[str, Any], + state: str, + installed: bool, + source: str, + fallback_name: Optional[str] = None, + fallback_description: Optional[str] = None, + fallback_description_cn: Optional[str] = None, +) -> DeviceTemplate: + name = _template_name(provider, descriptor, fallback_name, plugin_id) + description = _optional_str(provider.get("description")) or fallback_description + description_cn = _optional_str(provider.get("description_cn")) or fallback_description_cn + return DeviceTemplate( + plugin_id=plugin_id, + storage_key=descriptor.storage_key, + service_id=descriptor.service_id, + name=name, + version=descriptor.version, + vendor=_optional_str(provider.get("vendor")), + description=description, + description_cn=description_cn, + credential_schema=[ + field.model_dump(mode="json") + for field in _build_api_service_credential_schema(descriptor.storage_key, provider) + ], + tool_count=_tool_count(descriptor.storage_key, descriptor.provider_yaml.parent), + installed=installed, + state=state, # type: ignore[arg-type] + source=source, # type: ignore[arg-type] + ) + + +def _template_name( + provider: Dict[str, Any], + descriptor: ApiServiceDescriptor, + fallback_name: Optional[str], + plugin_id: str, +) -> str: + display_name = _optional_str(provider.get("display_name")) + if display_name: + return display_name + + raw_name = str(provider.get("name") or fallback_name or plugin_id).strip() or plugin_id + raw_identity = raw_name.lower() + identity_values = { + plugin_id.lower(), + descriptor.service_id.lower(), + descriptor.storage_key.lower(), + } + if raw_identity not in identity_values: + return raw_name + + return _product_name_from_identifier(descriptor.service_id) or _product_name_from_identifier(plugin_id) or raw_name + + +def _product_name_from_identifier(value: str) -> str: + candidate = value.strip() + if candidate.endswith("_api"): + candidate = candidate[:-4] + candidate = re.sub(r"_v\d+(?:_\d+)*(?:_[A-Za-z]\d+)?$", "", candidate) + return candidate or value + + +def _catalog_entry_root(entry: Any) -> Optional[Path]: + if entry.installPath: + path = Path(entry.installPath) + if path.exists(): + return path + return hub_catalog.system_plugin_root("device", entry.id) + + +def _entry_installed(entry: Any) -> bool: + return entry.state in {"installed", "updateAvailable", "localOnly"} or bool(entry.installPath) + + +def _normalize_state(state: str) -> str: + if state in {"available", "installed", "updateAvailable", "localOnly", "broken"}: + return state + return "broken" + + +def _source_from_entry(entry: Any) -> str: + if entry.installPath: + return _source_from_path(Path(entry.installPath)) + if entry.source == "system": + root = hub_catalog.system_plugin_root("device", entry.id) + return _source_from_path(root) if root else "project" + return "bundled" + + +def _source_from_path(path: Optional[Path]) -> str: + if path is None: + return "bundled" + try: + resolved = path.resolve() + except Exception: + resolved = path + try: + resolved.relative_to((Path.cwd() / ".flocks" / "plugins").resolve()) + return "project" + except Exception: + pass + try: + resolved.relative_to((Path.home() / ".flocks" / "plugins").resolve()) + return "global" + except Exception: + pass + return "bundled" + + +def _tool_count(storage_key: str, root: Path) -> int: + try: + ToolRegistry.init() + count = len([ + tool for tool in ToolRegistry.list_tools() + if tool.source == "device" and tool.provider == storage_key + ]) + if count: + return count + except Exception as exc: + log.debug("device.templates.tool_registry_unavailable", {"error": str(exc)}) + + return len([ + path for path in _tool_yaml_files(root) + if not path.name.startswith("_") + ]) + + +def _tool_yaml_files(root: Path) -> Iterable[Path]: + if not root.is_dir(): + return [] + return [ + path for path in root.iterdir() + if path.is_file() and path.suffix in {".yaml", ".yml"} + ] + + +def _read_provider_yaml(root: Path) -> Dict[str, Any]: + path = root / "_provider.yaml" + if not path.is_file(): + return {} + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except Exception as exc: + log.warning("device.templates.provider_yaml_read_failed", { + "path": str(path), + "error": str(exc), + }) + return {} + return data if isinstance(data, dict) else {} + + +def _write_yaml(path: Path, data: Dict[str, Any]) -> None: + path.write_text( + yaml.safe_dump(data, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + + +def _custom_provider_yaml(body: CustomDeviceTemplateCreate) -> Dict[str, Any]: + data: Dict[str, Any] = { + "name": body.name.strip(), + "service_id": body.service_id.strip(), + "integration_type": "device", + "credential_fields": body.credential_fields, + } + for key in ("vendor", "version", "description", "description_cn"): + value = getattr(body, key) + if value: + data[key] = value + return data + + +def _custom_tool_yaml(tool: Dict[str, Any], service_id: str) -> Dict[str, Any]: + tool["provider"] = service_id + tool.setdefault("enabled", True) + if not tool.get("category"): + tool["category"] = TOOL_TYPE_DEVICE + return tool + + +def _integration_type(provider: Dict[str, Any]) -> Optional[str]: + value = provider.get("integration_type") + return value.strip().lower() if isinstance(value, str) and value.strip() else None + + +def _optional_str(value: Any) -> Optional[str]: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _sort_key(template: DeviceTemplate) -> tuple[int, str, str, str, str]: + return ( + 0 if template.installed else 1, + (template.vendor or "").lower(), + template.name.lower(), + template.service_id.lower(), + template.version or "", + ) diff --git a/flocks/tool/device/secrets.py b/flocks/tool/device/secrets.py index d48a165fe..6f341a8d6 100644 --- a/flocks/tool/device/secrets.py +++ b/flocks/tool/device/secrets.py @@ -50,6 +50,21 @@ def _secret_keys_for(storage_key: str) -> FrozenSet[str]: return _FALLBACK_SECRET_KEYS +def _normalize_config_field(storage_key: str, key: str, value: str) -> str: + """Canonicalize non-secret device fields before storing them in SQL.""" + if key not in {"base_url", "baseUrl"}: + return value + if storage_key != "tdp_api" and not storage_key.startswith("tdp_api_v"): + return value + + normalized = value.strip().rstrip("/") + for suffix in ("/config/api", "/api/v1"): + if normalized.lower().endswith(suffix): + normalized = normalized[: -len(suffix)].rstrip("/") + break + return normalized + + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @@ -86,7 +101,7 @@ def persist_fields( continue result[key] = f"{_PLACEHOLDER_PREFIX}{sid}{_PLACEHOLDER_SUFFIX}" else: - result[key] = value + result[key] = _normalize_config_field(storage_key, key, value) return result @@ -135,8 +150,8 @@ def mask_for_display(db_fields: Dict[str, str]) -> Tuple[Dict[str, str], Dict[st def resolve_for_runtime(db_fields: Dict[str, str]) -> Dict[str, str]: """Resolve ``{secret:…}`` placeholders to plaintext. - Call ONLY at the moment of making an outbound API request. - Never store or return the result through a public interface. + Call ONLY at the moment of making an outbound API request or serving an + explicit authenticated reveal action. Never store or log the result. """ from flocks.security import get_secret_manager diff --git a/flocks/tool/registry.py b/flocks/tool/registry.py index 47ce998f1..229361ac8 100644 --- a/flocks/tool/registry.py +++ b/flocks/tool/registry.py @@ -1447,11 +1447,11 @@ def _register_builtin_tools(cls) -> None: # agent/ — agent delegation/coordination ("flocks.tool.agent", ["delegate_task", "task"]), # task/ — task/workflow - ("flocks.tool.task", ["schedule_task_center", "todo", "plan", "run_workflow", "run_workflow_node"]), + ("flocks.tool.task", ["schedule_task_center", "todo", "run_workflow", "run_workflow_node"]), # security/ — SSH forensics + threat intelligence (optional: asyncssh) ("flocks.tool.security", ["ssh_host_cmd", "ssh_run_script"]), - # system/ — background tasks, questions, model config, memory, MCP management, session management, slash commands - ("flocks.tool.system", ["background_output", "background_cancel", "question", "model_config", "memory", "flocks_mcp", "session_manage", "slash_command", "tool_search"]), + # system/ — questions, model config, memory, MCP management, session management, slash commands + ("flocks.tool.system", ["question", "model_config", "memory", "flocks_mcp", "session_manage", "slash_command", "tool_search"]), # skill/ — skill management (search, install, status, deps, remove, load) ("flocks.tool.skill", ["flocks_skills", "skill_load"]), # device/ — security device asset context @@ -1473,7 +1473,13 @@ def _register_builtin_tools(cls) -> None: # This is done in bulk here so individual @register_function call # sites don't need to pass native=True, and user plugin files using # the same decorator won't be misclassified. - builtin_native_exceptions = {"lsp", "task"} + builtin_native_exceptions = { + "lsp", + "task", + "list_providers", + "add_provider", + "add_model", + } for name in set(cls._tools.keys()) - before: if name in builtin_native_exceptions: cls._tools[name].info.native = False @@ -1760,7 +1766,7 @@ def _tool_event_should_reload(event: object) -> bool: class ToolFileWatcher: """Watch plugin tool directories and auto-reload plugin tools on change. - Monitors the ``api/`` and ``python/`` subdirectories under: + Monitors the ``api/``, ``device/``, and ``python/`` subdirectories under: - ``~/.flocks/plugins/tools/`` (user-level) - ``/.flocks/plugins/tools/`` (project-level) @@ -1770,7 +1776,7 @@ class ToolFileWatcher: """ _DEBOUNCE_SECONDS = 1.0 - _WATCH_SUBDIRS = ("api", "python") + _WATCH_SUBDIRS = ("api", "device", "python") def __init__(self) -> None: self._observer: Optional[object] = None diff --git a/flocks/tool/skill/flocks_skills.py b/flocks/tool/skill/flocks_skills.py index 0a804c183..acb8a7fe4 100644 --- a/flocks/tool/skill/flocks_skills.py +++ b/flocks/tool/skill/flocks_skills.py @@ -44,7 +44,8 @@ ## Subcommands **find ** - Search the **external public skill registry** by keyword. + Search external public skill registries by keyword (local, clawhub, skills.sh, + SafeSkill when available, and curated GitHub collections). This does NOT show installed skills — it discovers skills that can be installed. → Use BEFORE telling the user "I can't do X". A matching skill may exist. Example: flocks_skills(subcommand="find", args="malware phishing") @@ -54,7 +55,11 @@ Source formats: github:// e.g. github:octocat/skills/find-ioc clawhub: e.g. clawhub:ndr-alert-analysis + skills-sh:// e.g. skills-sh:owner/repo/code-review + safeskill: e.g. safeskill:safeskill://official/acme/code-review https://... direct SKILL.md URL + The tool auto-adds --yes so non-interactive agent calls do not hang on + downstream CLI confirmation prompts (e.g. `skills add`). → After install, always call status to check if deps are missing. Example: flocks_skills(subcommand="install", args="github:owner/repo/skill-name") @@ -70,7 +75,8 @@ Example: flocks_skills(subcommand="install-deps", args="find-ioc") **remove ** - Uninstall a user-managed skill from ~/.flocks. + Uninstall a user-managed skill from ~/.flocks. The tool adds --yes + automatically so non-interactive agent calls do not hang on confirmation. Example: flocks_skills(subcommand="remove", args="old-skill") """ @@ -150,6 +156,11 @@ async def flocks_skills( if args.strip(): # shlex.split preserves quoted tokens (e.g. paths with spaces). cmd += shlex.split(args.strip()) + # `skills add` (downstream of install for skills-sh sources) and remove + # both prompt interactively. Auto-add --yes so non-interactive agent + # calls don't hang. + if subcommand in ("install", "remove") and "--yes" not in cmd and "-y" not in cmd: + cmd.append("--yes") log.info("flocks_skills.run", {"cmd": cmd}) diff --git a/flocks/tool/system/background_cancel.py b/flocks/tool/system/background_cancel.py deleted file mode 100644 index a5c970484..000000000 --- a/flocks/tool/system/background_cancel.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -background_cancel tool - cancel background tasks. -""" - -from typing import Optional - -from flocks.tool.registry import ( - ToolRegistry, - ToolCategory, - ToolParameter, - ParameterType, - ToolResult, - ToolContext, -) -from flocks.task.background import get_background_manager - - -@ToolRegistry.register_function( - name="background_cancel", - description="Cancel a background task by task_id or cancel all.", - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="task_id", - type=ParameterType.STRING, - description="Background task ID to cancel", - required=False, - ), - ToolParameter( - name="all", - type=ParameterType.BOOLEAN, - description="Cancel all background tasks", - required=False, - ), - ], -) -async def background_cancel_tool( - ctx: ToolContext, - task_id: Optional[str] = None, - all: Optional[bool] = False, -) -> ToolResult: - await ctx.ask( - permission="background_cancel", - patterns=[task_id or "*"], - always=["*"], - metadata={"task_id": task_id, "all": all}, - ) - manager = get_background_manager() - cancelled = manager.cancel(task_id=task_id, all_tasks=bool(all)) - return ToolResult( - success=True, - output=f"Cancelled {cancelled} task(s).", - title="background_cancel", - metadata={"cancelled": cancelled}, - ) diff --git a/flocks/tool/system/background_output.py b/flocks/tool/system/background_output.py deleted file mode 100644 index f1628ff61..000000000 --- a/flocks/tool/system/background_output.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -background_output tool - query background task status/output. -""" - -from typing import Optional - -from flocks.tool.registry import ( - ToolRegistry, - ToolCategory, - ToolParameter, - ParameterType, - ToolResult, - ToolContext, -) -from flocks.task.background import get_background_manager, BackgroundTask -from flocks.session.session import Session -from flocks.session.message import Message - - -def _find_task(manager, task_id: str) -> Optional[BackgroundTask]: - """Find task by task_id, or fall back to session_id prefix match.""" - task = manager.get_task(task_id) - if task: - return task - # LLMs often pass session_id instead of task_id — search by session_id - for t in manager.list_tasks(): - if t.session_id and (t.session_id == task_id or t.session_id.startswith(task_id)): - return t - return None - - -@ToolRegistry.register_function( - name="background_output", - description="Check background task status/output by task_id or session_id. Optionally block until complete.", - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="task_id", - type=ParameterType.STRING, - description="Background task ID (bg_xxx) or session ID (ses_xxx)", - required=True, - ), - ToolParameter( - name="block", - type=ParameterType.BOOLEAN, - description="If true, wait for completion (optional)", - required=False, - ), - ToolParameter( - name="timeout_ms", - type=ParameterType.INTEGER, - description="Max wait time in milliseconds when block=true", - required=False, - ), - ], -) -async def background_output_tool( - ctx: ToolContext, - task_id: str, - block: Optional[bool] = False, - timeout_ms: Optional[int] = None, -) -> ToolResult: - await ctx.ask( - permission="background_output", - patterns=[task_id], - always=["*"], - metadata={"task_id": task_id}, - ) - manager = get_background_manager() - - # Try direct lookup first, then session_id fallback - task = _find_task(manager, task_id) - - if task and block: - waited = await manager.wait_for(task.id, timeout_ms) - task = waited or _find_task(manager, task.id) - - if task: - output = ( - f"Task ID: {task.id}\n" - f"Status: {task.status}\n" - f"Agent: {task.agent}\n" - f"Description: {task.description}\n" - f"Session ID: {task.session_id}\n" - ) - if task.error: - output += f"\nError: {task.error}\n" - if task.output: - output += f"\nOutput:\n{task.output}\n" - return ToolResult( - success=True, - output=output, - title=task.description, - metadata={"status": task.status, "sessionId": task.session_id}, - ) - - # No background task found — if it looks like a session_id, query the session directly. - # This handles the case where the task ran synchronously (not in background). - if task_id.startswith("ses_"): - session = await Session.get_by_id(task_id) - if not session: - # Try prefix match (LLMs sometimes truncate IDs) - return ToolResult( - success=False, - error=( - f'No background task or session found for "{task_id}". ' - "The task may have already completed synchronously. " - "Check the task tool output above for results." - ), - ) - # Fetch last assistant message from the session - messages = await Message.list(session.id) - last_assistant = None - for msg in reversed(messages): - if msg.role == "assistant": - last_assistant = msg - break - output_text = "" - if last_assistant: - output_text = await Message.get_text_content(last_assistant) - output = ( - f"Session ID: {session.id}\n" - f"Agent: {session.agent}\n" - f"Title: {session.title}\n" - f"Status: completed (ran synchronously, not in background)\n" - ) - if output_text: - output += f"\nOutput:\n{output_text}\n" - else: - output += "\nNo output (the subagent session may have encountered an error).\n" - return ToolResult( - success=True, - output=output, - title=session.title, - metadata={"status": "completed", "sessionId": session.id}, - ) - - return ToolResult( - success=False, - error=f'Task "{task_id}" not found. Use the task_id (bg_xxx) returned by the task tool when run_in_background=true.', - ) diff --git a/flocks/tool/system/model_config.py b/flocks/tool/system/model_config.py index 4936d98fe..5ad8cc0d0 100644 --- a/flocks/tool/system/model_config.py +++ b/flocks/tool/system/model_config.py @@ -42,6 +42,7 @@ name="list_providers", description=LIST_PROVIDERS_DESC, category=ToolCategory.SYSTEM, + native=False, parameters=[ ToolParameter( name="provider_id", @@ -130,6 +131,7 @@ async def list_providers_tool( description=ADD_PROVIDER_DESC, category=ToolCategory.SYSTEM, requires_confirmation=True, + native=False, parameters=[ ToolParameter( name="name", @@ -249,6 +251,7 @@ async def add_provider_tool( description=ADD_MODEL_DESC, category=ToolCategory.SYSTEM, requires_confirmation=True, + native=False, parameters=[ ToolParameter( name="provider_id", @@ -350,7 +353,7 @@ async def add_model_tool( "supports_vision": supports_vision, "supports_tools": supports_tools, "supports_streaming": True, - "supports_reasoning": False, + "supports_reasoning": True, "input_price": 0.0, "output_price": 0.0, "currency": "USD", diff --git a/flocks/tool/system/session_manage.py b/flocks/tool/system/session_manage.py index 352d285df..771a30280 100644 --- a/flocks/tool/system/session_manage.py +++ b/flocks/tool/system/session_manage.py @@ -1,13 +1,8 @@ """ Session management tools — 查询、创建、更新、删除 Flocks Session 的元数据。 -提供以下工具： -- session_list : 列出所有（或指定 project）的 session -- session_get : 获取单个 session 的完整元数据 -- session_create : 创建新 session -- session_update : 更新 session 元数据（标题、agent、状态等） -- session_delete : 删除 session（软删除，同时清理子 session） -- session_archive: 归档 / 取消归档 session +提供统一工具： +- session_manage(action=...) : list/get/create/update/delete/archive """ from __future__ import annotations @@ -27,6 +22,199 @@ log = Log.create(service="tool.session_manage") +SESSION_MANAGE_ACTIONS = ["list", "get", "create", "update", "delete", "archive"] + +SESSION_MANAGE_DESCRIPTION = """\ +管理 Flocks Session 元数据。 + +Use `action` to choose the operation: +- list: 列出 session；可用 project_id/status/category/limit/offset 过滤或分页 +- get: 获取单个 session；需要 session_id +- create: 创建 session；可传 title/project_id/directory/agent/parent_id +- update: 更新 session；需要 session_id，可传 title/agent/model/provider/memory_enabled +- delete: 软删除 session 及其子 session；需要 session_id，会请求确认 +- archive: 归档或取消归档 session；需要 session_id，archive=false 表示恢复 active +""" + + +SESSION_MANAGE_PARAMETERS = [ + ToolParameter( + name="action", + type=ParameterType.STRING, + required=True, + enum=SESSION_MANAGE_ACTIONS, + description="要执行的 session 操作：list/get/create/update/delete/archive", + ), + ToolParameter( + name="session_id", + type=ParameterType.STRING, + required=False, + description="Session ID；get/update/delete/archive 需要", + ), + ToolParameter( + name="project_id", + type=ParameterType.STRING, + required=False, + description="项目 ID；list 时用于过滤，create 时用于归属项目（默认 default）", + ), + ToolParameter( + name="status", + type=ParameterType.STRING, + required=False, + enum=["active", "archived"], + description="list 过滤状态：active 或 archived", + ), + ToolParameter( + name="category", + type=ParameterType.STRING, + required=False, + enum=["user", "task"], + description="list 过滤分类：user（人工会话）或 task（任务触发会话）", + ), + ToolParameter( + name="limit", + type=ParameterType.INTEGER, + required=False, + description="list 最多返回条数（默认 50）", + ), + ToolParameter( + name="offset", + type=ParameterType.INTEGER, + required=False, + description="list 跳过前 N 条（默认 0）", + ), + ToolParameter( + name="title", + type=ParameterType.STRING, + required=False, + description="create/update 的 session 标题", + ), + ToolParameter( + name="directory", + type=ParameterType.STRING, + required=False, + description="create 的工作目录路径（默认当前目录）", + ), + ToolParameter( + name="agent", + type=ParameterType.STRING, + required=False, + description="create/update 的 agent 类型", + ), + ToolParameter( + name="parent_id", + type=ParameterType.STRING, + required=False, + description="create 子 session 时使用的父 session ID", + ), + ToolParameter( + name="model", + type=ParameterType.STRING, + required=False, + description="update 的 model ID", + ), + ToolParameter( + name="provider", + type=ParameterType.STRING, + required=False, + description="update 的 provider ID", + ), + ToolParameter( + name="memory_enabled", + type=ParameterType.BOOLEAN, + required=False, + description="update 时是否启用 memory 系统", + ), + ToolParameter( + name="archive", + type=ParameterType.BOOLEAN, + required=False, + default=True, + description="archive action: true=归档（默认），false=取消归档", + ), +] + + +@ToolRegistry.register_function( + name="session_manage", + description=SESSION_MANAGE_DESCRIPTION, + category=ToolCategory.SYSTEM, + parameters=SESSION_MANAGE_PARAMETERS, +) +async def session_manage( + ctx: ToolContext, + action: str, + session_id: Optional[str] = None, + project_id: Optional[str] = None, + status: Optional[str] = None, + category: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + title: Optional[str] = None, + directory: Optional[str] = None, + agent: Optional[str] = None, + parent_id: Optional[str] = None, + model: Optional[str] = None, + provider: Optional[str] = None, + memory_enabled: Optional[bool] = None, + archive: Optional[bool] = True, +) -> ToolResult: + """Unified session management tool.""" + if action == "list": + return await _session_list_impl( + ctx, + project_id=project_id, + status=status, + category=category, + limit=limit, + offset=offset, + ) + if action == "get": + if not session_id: + return ToolResult(success=False, error="action=get 需要 session_id") + return await _session_get_impl(ctx, session_id=session_id) + if action == "create": + return await _session_create_impl( + ctx, + title=title, + project_id=project_id, + directory=directory, + agent=agent, + parent_id=parent_id, + ) + if action == "update": + if not session_id: + return ToolResult(success=False, error="action=update 需要 session_id") + return await _session_update_impl( + ctx, + session_id=session_id, + title=title, + agent=agent, + model=model, + provider=provider, + memory_enabled=memory_enabled, + ) + if action == "delete": + if not session_id: + return ToolResult(success=False, error="action=delete 需要 session_id") + await ctx.ask( + permission="session_manage", + patterns=[f"delete:{session_id}"], + always=[], + metadata={"action": "delete", "session_id": session_id}, + ) + return await _session_delete_impl(ctx, session_id=session_id) + if action == "archive": + if not session_id: + return ToolResult(success=False, error="action=archive 需要 session_id") + return await _session_archive_impl(ctx, session_id=session_id, archive=archive) + + return ToolResult( + success=False, + error=f"未知 action: {action!r}. 支持: {', '.join(SESSION_MANAGE_ACTIONS)}", + ) + + def _session_to_dict(session, bindings: list | None = None) -> dict[str, Any]: """Serialize SessionInfo to a readable dict. @@ -97,52 +285,10 @@ async def _enrich_with_channels(sessions_dict: list[dict]) -> list[dict]: # --------------------------------------------------------------------------- -# session_list +# list # --------------------------------------------------------------------------- -@ToolRegistry.register_function( - name="session_list", - description=( - "列出 Flocks 的所有 Session 元数据。" - "可按 project_id、status、category 过滤，支持分页。" - ), - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="project_id", - type=ParameterType.STRING, - required=False, - description="按 project_id 过滤；不填则列出所有项目的 session", - ), - ToolParameter( - name="status", - type=ParameterType.STRING, - required=False, - enum=["active", "archived"], - description="按状态过滤：active（默认）或 archived", - ), - ToolParameter( - name="category", - type=ParameterType.STRING, - required=False, - enum=["user", "task"], - description="按分类过滤：user（人工会话）或 task（任务触发会话）", - ), - ToolParameter( - name="limit", - type=ParameterType.INTEGER, - required=False, - description="最多返回条数（默认 50）", - ), - ToolParameter( - name="offset", - type=ParameterType.INTEGER, - required=False, - description="跳过前 N 条（用于翻页，默认 0）", - ), - ], -) -async def session_list( +async def _session_list_impl( ctx: ToolContext, project_id: Optional[str] = None, status: Optional[str] = None, @@ -194,23 +340,10 @@ async def session_list( # --------------------------------------------------------------------------- -# session_get +# get # --------------------------------------------------------------------------- -@ToolRegistry.register_function( - name="session_get", - description="获取指定 session 的完整元数据，包含时间戳、agent、状态、摘要等。", - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="session_id", - type=ParameterType.STRING, - required=True, - description="Session ID", - ), - ], -) -async def session_get(ctx: ToolContext, session_id: str) -> ToolResult: +async def _session_get_impl(ctx: ToolContext, session_id: str) -> ToolResult: from flocks.session.session import Session session = await Session.get_by_id(session_id) @@ -223,47 +356,10 @@ async def session_get(ctx: ToolContext, session_id: str) -> ToolResult: # --------------------------------------------------------------------------- -# session_create +# create # --------------------------------------------------------------------------- -@ToolRegistry.register_function( - name="session_create", - description="创建一个新的 Flocks Session。", - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="title", - type=ParameterType.STRING, - required=False, - description="Session 标题；不填则自动生成", - ), - ToolParameter( - name="project_id", - type=ParameterType.STRING, - required=False, - description="归属的 project ID（默认 'default'）", - ), - ToolParameter( - name="directory", - type=ParameterType.STRING, - required=False, - description="工作目录路径（默认使用当前目录）", - ), - ToolParameter( - name="agent", - type=ParameterType.STRING, - required=False, - description="指定 agent 类型，如 hephaestus、rex、build、plan 等", - ), - ToolParameter( - name="parent_id", - type=ParameterType.STRING, - required=False, - description="父 session ID（创建子 session 时使用）", - ), - ], -) -async def session_create( +async def _session_create_impl( ctx: ToolContext, title: Optional[str] = None, project_id: Optional[str] = None, @@ -295,56 +391,10 @@ async def session_create( # --------------------------------------------------------------------------- -# session_update +# update # --------------------------------------------------------------------------- -@ToolRegistry.register_function( - name="session_update", - description=( - "更新指定 session 的元数据字段。" - "支持修改标题、agent、model、provider、memory_enabled 等。" - ), - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="session_id", - type=ParameterType.STRING, - required=True, - description="要更新的 Session ID", - ), - ToolParameter( - name="title", - type=ParameterType.STRING, - required=False, - description="新标题", - ), - ToolParameter( - name="agent", - type=ParameterType.STRING, - required=False, - description="新 agent 类型", - ), - ToolParameter( - name="model", - type=ParameterType.STRING, - required=False, - description="新 model ID", - ), - ToolParameter( - name="provider", - type=ParameterType.STRING, - required=False, - description="新 provider ID", - ), - ToolParameter( - name="memory_enabled", - type=ParameterType.BOOLEAN, - required=False, - description="是否启用 memory 系统", - ), - ], -) -async def session_update( +async def _session_update_impl( ctx: ToolContext, session_id: str, title: Optional[str] = None, @@ -392,27 +442,10 @@ async def session_update( # --------------------------------------------------------------------------- -# session_delete +# delete # --------------------------------------------------------------------------- -@ToolRegistry.register_function( - name="session_delete", - description=( - "删除指定 session（软删除）。" - "同时会递归删除其所有子 session，并清空消息记录。" - ), - category=ToolCategory.SYSTEM, - requires_confirmation=True, - parameters=[ - ToolParameter( - name="session_id", - type=ParameterType.STRING, - required=True, - description="要删除的 Session ID", - ), - ], -) -async def session_delete(ctx: ToolContext, session_id: str) -> ToolResult: +async def _session_delete_impl(ctx: ToolContext, session_id: str) -> ToolResult: from flocks.session.session import Session session = await Session.get_by_id(session_id) @@ -434,29 +467,10 @@ async def session_delete(ctx: ToolContext, session_id: str) -> ToolResult: # --------------------------------------------------------------------------- -# session_archive +# archive # --------------------------------------------------------------------------- -@ToolRegistry.register_function( - name="session_archive", - description="归档或取消归档指定 session。归档后 session 仍可查询，但不再活跃。", - category=ToolCategory.SYSTEM, - parameters=[ - ToolParameter( - name="session_id", - type=ParameterType.STRING, - required=True, - description="Session ID", - ), - ToolParameter( - name="archive", - type=ParameterType.BOOLEAN, - required=False, - description="true=归档（默认），false=取消归档（恢复为 active）", - ), - ], -) -async def session_archive( +async def _session_archive_impl( ctx: ToolContext, session_id: str, archive: Optional[bool] = True, diff --git a/flocks/tool/task/plan.py b/flocks/tool/task/plan.py deleted file mode 100644 index a59bc7f1c..000000000 --- a/flocks/tool/task/plan.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -Plan Tools - Plan mode switching - -Provides tools for entering and exiting plan mode: -- plan_enter: Switch to plan agent for research and planning -- plan_exit: Switch back to build agent after planning is complete -""" - -from typing import Optional - -from flocks.tool.registry import ( - ToolRegistry, ToolCategory, ToolResult, ToolContext -) -from flocks.tool.system.question import QuestionRejectedError -from flocks.utils.log import Log - - -log = Log.create(service="tool.plan") - - -# Default plan file path -DEFAULT_PLAN_FILE = "PLAN.md" - - -PLAN_ENTER_DESCRIPTION = """Switch to plan mode to create a detailed implementation plan. - -Use this tool when: -- The task is complex and requires research first -- You need to explore the codebase before making changes -- The user asks for a plan or wants to discuss approach -- There are significant architectural decisions to make - -In plan mode: -- You will operate as the "plan" agent -- Focus on research, analysis, and creating a plan document -- Changes to non-plan files are restricted -- Use plan_exit when the plan is complete and you're ready to return to build mode""" - - -PLAN_EXIT_DESCRIPTION = """Exit plan mode and return to build mode. - -Use this tool when: -- The plan document is complete -- You're ready to start implementing - -This will: -- Switch to the "rex" agent -- Allow full file editing capabilities""" - - -# Callback for agent switching (to be set by application) -_agent_switch_callback: Optional[callable] = None - - -def set_agent_switch_callback(callback: callable) -> None: - """ - Set the callback for agent switching - - Args: - callback: Function(session_id, from_agent, to_agent, message) -> None - """ - global _agent_switch_callback - _agent_switch_callback = callback - - -async def _ask_user(ctx: ToolContext, question: str, header: str, options: list) -> str: - """ - Ask user a question using the question system - - Args: - ctx: Tool context - question: Question text - header: Header text - options: List of option dicts with label and description - - Returns: - Selected option label - """ - from flocks.tool.system.question import ( - _question_handler, - default_question_handler, - _current_call_id, - _current_message_id, - ) - - handler = _question_handler or default_question_handler - - questions = [{ - "question": question, - "header": header, - "options": options - }] - - # Set message_id in context for handler to use - _current_message_id.set(ctx.message_id) - _current_call_id.set(ctx.call_id) - - answers = await handler(ctx.session_id, questions) - - # Safe access to nested list to avoid index out of range - if answers and len(answers) > 0 and answers[0] and len(answers[0]) > 0: - return answers[0][0] - return "No" - - -@ToolRegistry.register_function( - name="plan_enter", - description=PLAN_ENTER_DESCRIPTION, - category=ToolCategory.SYSTEM, - parameters=[] -) -async def plan_enter_tool( - ctx: ToolContext, -) -> ToolResult: - """ - Enter plan mode - - Args: - ctx: Tool context - - Returns: - ToolResult with switch status - """ - plan_file = DEFAULT_PLAN_FILE - - # Ask user for confirmation - try: - answer = await _ask_user( - ctx, - question=f"Would you like to switch to the plan agent and create a plan saved to {plan_file}?", - header="Plan Mode", - options=[ - {"label": "Yes", "description": "Switch to plan agent for research and planning"}, - {"label": "No", "description": "Stay with build agent to continue making changes"} - ] - ) - - if answer == "No": - raise QuestionRejectedError() - - except QuestionRejectedError: - return ToolResult( - success=False, - error="User declined to enter plan mode" - ) - - # Notify agent switch - if _agent_switch_callback: - try: - await _agent_switch_callback( - ctx.session_id, - ctx.agent, - "plan", - "User has requested to enter plan mode. Switch to plan mode and begin planning." - ) - except Exception as e: - log.warn("plan_enter.callback_failed", {"error": str(e)}) - - return ToolResult( - success=True, - output=f"User confirmed to switch to plan mode. A new message has been created to switch you to plan mode. The plan file will be at {plan_file}. Begin planning.", - title="Switching to plan agent", - metadata={} - ) - - -@ToolRegistry.register_function( - name="plan_exit", - description=PLAN_EXIT_DESCRIPTION, - category=ToolCategory.SYSTEM, - parameters=[] -) -async def plan_exit_tool( - ctx: ToolContext, -) -> ToolResult: - """ - Exit plan mode - - Args: - ctx: Tool context - - Returns: - ToolResult with switch status - """ - plan_file = DEFAULT_PLAN_FILE - - # Notify agent switch - if _agent_switch_callback: - try: - await _agent_switch_callback( - ctx.session_id, - ctx.agent, - "rex", - f"The plan at {plan_file} is complete. Switch back to build mode and execute it." - ) - except Exception as e: - log.warn("plan_exit.callback_failed", {"error": str(e)}) - - return ToolResult( - success=True, - output="Exited plan mode and switched back to rex agent. Continue by executing the plan.", - title="Switching to rex agent", - metadata={} - ) diff --git a/flocks/tool/task/run_workflow.py b/flocks/tool/task/run_workflow.py index 2f266d1cf..c4588a89c 100644 --- a/flocks/tool/task/run_workflow.py +++ b/flocks/tool/task/run_workflow.py @@ -99,6 +99,32 @@ def _get_workflow_runtime(): _DESCRIPTION_CACHE_TTL: float = 60.0 # seconds +def _resolve_workflow_display_name_and_total_nodes( + workflow_source: Union[Dict[str, Any], Path], +) -> tuple[str, Optional[int]]: + """Return a friendly workflow name and node count when cheaply available.""" + if isinstance(workflow_source, dict): + workflow_name = str(workflow_source.get("name") or "").strip() or "unnamed workflow" + nodes = workflow_source.get("nodes") + total_nodes = len(nodes) if isinstance(nodes, list) else None + return workflow_name, total_nodes + + workflow_name = workflow_source.stem or workflow_source.name + try: + raw = json.loads(workflow_source.read_text(encoding="utf-8")) + except Exception: + return workflow_name, None + + if isinstance(raw, dict): + loaded_name = str(raw.get("name") or "").strip() + if loaded_name: + workflow_name = loaded_name + nodes = raw.get("nodes") + if isinstance(nodes, list): + return workflow_name, len(nodes) + return workflow_name, None + + def _create_nested_tool_context(ctx: ToolContext) -> ToolContext: """Create an isolated child ToolContext for workflow node tools. @@ -161,8 +187,13 @@ async def _build_description() -> str: return result -def _format_workflow_result(result: Any) -> str: - """Format RunWorkflowResult or dict as readable output""" +def _format_workflow_result(result: Any, *, include_history: bool = False) -> str: + """Format RunWorkflowResult or dict as readable output. + + By default the returned text omits step-by-step execution history so + session tool output stays concise. The structured history remains + available in metadata and persisted execution records. + """ if hasattr(result, '__dict__'): # RunWorkflowResult object data = result.__dict__ @@ -194,7 +225,7 @@ def _format_workflow_result(result: Any) -> str: except Exception: output_lines.append(str(data.get('outputs'))) - if data.get('history'): + if include_history and data.get('history'): history = data.get('history', []) if history: output_lines.append(f"\n{'='*80}") @@ -456,13 +487,12 @@ async def run_workflow_tool( ) # Request permission (workflow execution can run arbitrary code) + workflow_name, workflow_total_nodes = _resolve_workflow_display_name_and_total_nodes(workflow_source) + if isinstance(workflow_source, dict): - workflow_name = workflow_source.get("name", "unnamed workflow") # Use id if available, otherwise use name or generate a fallback workflow_id = workflow_source.get("id") or workflow_source.get("name") or "unknown" else: - # workflow_source is a Path object here; Path.name gives the filename. - workflow_name = workflow_source.name workflow_id = str(workflow_source) workflow_inputs = inputs or {} @@ -514,6 +544,8 @@ def _on_step_start( "title": f"Running workflow: {workflow_name}", "metadata": { "workflow_id": display_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"] if tracked_execution else None, "run_id": run_id, "status": "running", @@ -544,6 +576,8 @@ def _on_step_complete(step_result: Any) -> None: "title": f"Running workflow: {workflow_name}", "metadata": { "workflow_id": display_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"] if tracked_execution else None, "status": "running", "phase": "running", @@ -578,6 +612,8 @@ def _on_step_complete(step_result: Any) -> None: "title": f"Running workflow: {workflow_name}", "metadata": { "workflow_id": display_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"] if tracked_execution else None, "status": "running", "phase": "queued", @@ -610,6 +646,7 @@ def _on_step_complete(step_result: Any) -> None: # Backward-compatibility: older runtimes may not accept `use_llm`. supports_use_llm = False supports_step_start = False + supports_cancel = False try: sig = inspect.signature(_run_workflow_fn) supports_use_llm = ( @@ -620,16 +657,23 @@ def _on_step_complete(step_result: Any) -> None: "on_step_start" in sig.parameters or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) ) + supports_cancel = ( + "cancel" in sig.parameters + or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) + ) except Exception: # Best-effort: assume supported. supports_use_llm = True supports_step_start = True + supports_cancel = True if supports_use_llm: call_kwargs["use_llm"] = use_llm if supports_step_start: call_kwargs["on_step_start"] = _on_step_start call_kwargs["on_step_complete"] = _on_step_complete + if supports_cancel: + call_kwargs["cancel"] = ctx.abort.is_set try: result = await asyncio.to_thread(_run_workflow_fn, **call_kwargs) @@ -641,6 +685,9 @@ def _on_step_complete(step_result: Any) -> None: elif supports_step_start and "on_step_start" in str(te): call_kwargs.pop("on_step_start", None) result = await asyncio.to_thread(_run_workflow_fn, **call_kwargs) + elif supports_cancel and "cancel" in str(te): + call_kwargs.pop("cancel", None) + result = await asyncio.to_thread(_run_workflow_fn, **call_kwargs) else: raise @@ -699,6 +746,8 @@ def _on_step_complete(step_result: Any) -> None: "title": f"Workflow: {workflow_name}", "metadata": { "workflow_id": canonical_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"], "run_id": result_dict.get("run_id"), "status": status_value, @@ -720,6 +769,8 @@ def _on_step_complete(step_result: Any) -> None: title=f"Workflow: {workflow_name}", metadata={ "workflow_id": display_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"] if tracked_execution else None, "status": status_value, "steps": result_dict.get("steps", 0), @@ -736,6 +787,8 @@ def _on_step_complete(step_result: Any) -> None: title=f"Workflow: {workflow_name}", metadata={ "workflow_id": display_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"] if tracked_execution else None, "status": status_value, "steps": result_dict.get("steps", 0), @@ -771,6 +824,8 @@ def _on_step_complete(step_result: Any) -> None: "title": f"Workflow: {workflow_name}", "metadata": { "workflow_id": canonical_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"], "status": "error", "phase": "error", @@ -784,6 +839,8 @@ def _on_step_complete(step_result: Any) -> None: title=f"Workflow: {workflow_name}", metadata={ "workflow_id": display_workflow_id, + "workflow_name": workflow_name, + "total_nodes": workflow_total_nodes, "workflow_execution_id": tracked_execution["id"] if tracked_execution else None, "status": "FAILED", } diff --git a/flocks/tool/task/todo.py b/flocks/tool/task/todo.py index 52f0792f4..a9e5f04b0 100644 --- a/flocks/tool/task/todo.py +++ b/flocks/tool/task/todo.py @@ -54,7 +54,7 @@ } -TODOWRITE_DESCRIPTION = """Use this tool to create and manage a structured task list for your current SecOps session. This helps track progress, organize complex tasks, and demonstrate thoroughness. +TODO_DESCRIPTION = """Use this tool to read or manage a structured task list for your current SecOps session. This helps track progress, organize complex tasks, and demonstrate thoroughness. When to Use This Tool: 1. Complex multi-step tasks (3+ distinct steps) @@ -79,8 +79,14 @@ - Mark complete IMMEDIATELY after finishing - Only ONE task in_progress at a time -Valid input example: +Read example: { + "action": "read" +} + +Write example: +{ + "action": "write", "todos": [ {"id": "investigate", "content": "Investigate the alert", "activeForm": "Investigating the alert", "status": "in_progress"}, {"id": "verify", "content": "Verify the fix", "status": "pending"} @@ -89,15 +95,11 @@ Invalid input example: { + "action": "write", "todos": ["1. Investigate the alert", "2. Verify the fix"] }""" -TODOREAD_DESCRIPTION = """Use this tool to read your current todo list. - -Returns the current state of all todo items for this session.""" - - def _validation_error_message(index: int, error: ValidationError) -> str: """Return a concise validation message for a todo item.""" issues: List[str] = [] @@ -156,15 +158,22 @@ def _verification_nudge_needed(todos: List[TodoInfo]) -> bool: @ToolRegistry.register_function( - name="todowrite", - description=TODOWRITE_DESCRIPTION, + name="todo", + description=TODO_DESCRIPTION, category=ToolCategory.SYSTEM, parameters=[ + ToolParameter( + name="action", + type=ParameterType.STRING, + description="Action to perform: read current todos or write the full todo list", + required=True, + enum=["read", "write"], + ), ToolParameter( name="todos", type=ParameterType.ARRAY, - description="Array of todo items with id, content, and status fields", - required=True, + description="For action=write: array of todo items with id, content, and status fields", + required=False, json_schema={ "type": "array", "items": TODO_ITEM_JSON_SCHEMA, @@ -173,28 +182,58 @@ def _verification_nudge_needed(todos: List[TodoInfo]) -> bool: ), ] ) -async def todowrite_tool( +async def todo_tool( ctx: ToolContext, - todos: List[Dict[str, Any]], + action: str, + todos: List[Dict[str, Any]] | None = None, ) -> ToolResult: """ - Update the todo list + Read or update the todo list. Args: ctx: Tool context - todos: List of todo items + action: read or write + todos: List of todo items for write Returns: - ToolResult with updated todos + ToolResult with current or updated todos """ - # Request permission await ctx.ask( - permission="todowrite", + permission="todo", patterns=["*"], always=["*"], metadata={} ) - + + if action == "read": + current_todos = await Todo.get(ctx.session_id) + serialized_todos = _serialize_todos(current_todos) + pending_count = sum( + 1 for todo in current_todos if todo.status in ACTIVE_TODO_STATUSES + ) + + return ToolResult( + success=True, + output=json.dumps(serialized_todos, ensure_ascii=False, indent=2), + title=f"{pending_count} todos", + metadata={ + "action": "read", + "todos": serialized_todos + } + ) + + if action != "write": + return ToolResult( + success=False, + error=f"Unsupported todo action: {action!r}. Expected 'read' or 'write'.", + ) + + if todos is None: + return ToolResult( + success=False, + error="todos is required when action='write'", + ) + old_todos = await Todo.get(ctx.session_id) normalized_todos = _normalize_todos(todos) if _all_terminal(normalized_todos): @@ -219,49 +258,10 @@ async def todowrite_tool( output=json.dumps(output_payload, ensure_ascii=False, indent=2), title=f"{pending_count} todos", metadata={ + "action": "write", "todos": new_serialized, "oldTodos": old_serialized, "newTodos": new_serialized, "verificationNudgeNeeded": verification_nudge_needed, } ) - - -@ToolRegistry.register_function( - name="todoread", - description=TODOREAD_DESCRIPTION, - category=ToolCategory.SYSTEM, - parameters=[] -) -async def todoread_tool( - ctx: ToolContext, -) -> ToolResult: - """ - Read the current todo list - - Args: - ctx: Tool context - - Returns: - ToolResult with current todos - """ - # Request permission - await ctx.ask( - permission="todoread", - patterns=["*"], - always=["*"], - metadata={} - ) - - todos = await Todo.get(ctx.session_id) - serialized_todos = _serialize_todos(todos) - pending_count = sum(1 for todo in todos if todo.status in ACTIVE_TODO_STATUSES) - - return ToolResult( - success=True, - output=json.dumps(serialized_todos, ensure_ascii=False, indent=2), - title=f"{pending_count} todos", - metadata={ - "todos": serialized_todos - } - ) diff --git a/flocks/updater/updater.py b/flocks/updater/updater.py index a30b54ce9..2196dc974 100644 --- a/flocks/updater/updater.py +++ b/flocks/updater/updater.py @@ -81,7 +81,7 @@ class ConsoleManifestRelease: def _record_update_journal(message: str) -> None: - """Append a human-readable line to ``update.log`` (see ``append_upgrade_text_log``).""" + """Append a human-readable upgrade line to today's errors log.""" from flocks.utils.log import append_upgrade_text_log append_upgrade_text_log(message) @@ -570,6 +570,16 @@ def _dependency_sync_timeout_seconds() -> int: return _DEPENDENCY_SYNC_TIMEOUT_SECONDS +def _build_dependency_sync_command(uv_path: str, *, uv_default_index: str | None = None) -> list[str]: + """Build the ``uv sync`` command used by the self-updater.""" + cmd = [uv_path, "sync"] + if sys.platform == "win32": + cmd.append("--no-install-project") + if uv_default_index: + cmd.extend(["--default-index", uv_default_index]) + return cmd + + # ------------------------------------------------------------------ # # Async subprocess helpers # ------------------------------------------------------------------ # @@ -2892,9 +2902,7 @@ async def _restore_after_apply_failure() -> None: return log.info("updater.dependencies.sync", {"tool": "uv sync", "path": uv_path}) - uv_cmd = [uv_path, "sync"] - if profile.uv_default_index: - uv_cmd.extend(["--default-index", profile.uv_default_index]) + uv_cmd = _build_dependency_sync_command(uv_path, uv_default_index=profile.uv_default_index) sync_env = _build_uv_sync_env() sync_timeout = _dependency_sync_timeout_seconds() @@ -2957,7 +2965,7 @@ def _dependency_sync_timeout_message() -> str: }, ) await asyncio.sleep(3) - uv_cmd = [uv_path, "sync"] + uv_cmd = _build_dependency_sync_command(uv_path) try: code, _, err = await _run_uv_sync(uv_cmd) except subprocess.TimeoutExpired: diff --git a/flocks/user_defined_pages/__init__.py b/flocks/user_defined_pages/__init__.py new file mode 100644 index 000000000..6303f1d05 --- /dev/null +++ b/flocks/user_defined_pages/__init__.py @@ -0,0 +1,6 @@ +"""User-space user-defined custom pages under ~/.flocks/plugins/user_defined_pages.""" + +from flocks.user_defined_pages.store import UserDefinedPagesStore +from flocks.user_defined_pages.watcher import UserDefinedPagesWatcher + +__all__ = ["UserDefinedPagesStore", "UserDefinedPagesWatcher"] diff --git a/flocks/user_defined_pages/api_runtime.py b/flocks/user_defined_pages/api_runtime.py new file mode 100644 index 000000000..e009a1359 --- /dev/null +++ b/flocks/user_defined_pages/api_runtime.py @@ -0,0 +1,375 @@ +"""Page-scoped API runtime for user-defined pages.""" + +from __future__ import annotations + +import asyncio +import builtins +import importlib.util +import inspect +import json +import sys +import sysconfig +import time +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any, Optional + +import yaml +from fastapi import HTTPException, Request, status +from fastapi.responses import JSONResponse, Response + +from flocks.user_defined_pages.models import UserDefinedPageApiMeta +from flocks.user_defined_pages.store import UserDefinedPagesStore +from flocks.utils.log import Log + +log = Log.create(service="user-defined-pages-api-runtime") + +_ALLOWED_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE"} +_DEFAULT_TIMEOUT_MS = 5000 +_MAX_TIMEOUT_MS = 30000 +_MAX_RESPONSE_BYTES = 2_000_000 +_MAX_REQUEST_BODY_BYTES = 1_000_000 +_STDLIB_DIR = Path(sysconfig.get_paths()["stdlib"]).resolve() + + +@dataclass(frozen=True) +class _RouteSpec: + method: str + path: str + handler_name: str + timeout_ms: int + description: str + + +@dataclass(frozen=True) +class _RouteEntry: + spec: _RouteSpec + handler: Any + + +@dataclass +class _PageRuntime: + page_id: str + routes: dict[tuple[str, str], _RouteEntry] + module: ModuleType + routes_mtime_ns: int + handlers_mtime_ns: int + loaded_at: int + + +class UserDefinedPageApiRuntime: + """Load and dispatch api/routes.yaml + api/handlers.py for a page.""" + + def __init__(self, store: Optional[UserDefinedPagesStore] = None) -> None: + self._store = store or UserDefinedPagesStore() + self._cache: dict[str, _PageRuntime] = {} + self._lock = asyncio.Lock() + + def clear_page(self, page_id: str) -> None: + page_id = self._store.validate_page_id(page_id) + self._cache.pop(page_id, None) + + async def list_routes(self, page_id: str) -> list[dict[str, str]]: + runtime = await self._load_page_runtime(page_id, force_reload=False) + return [ + { + "method": entry.spec.method, + "path": entry.spec.path, + "handler": entry.spec.handler_name, + "description": entry.spec.description, + } + for entry in runtime.routes.values() + ] + + async def reload_page(self, page_id: str) -> list[dict[str, str]]: + runtime = await self._load_page_runtime(page_id, force_reload=True) + return [ + { + "method": entry.spec.method, + "path": entry.spec.path, + "handler": entry.spec.handler_name, + "description": entry.spec.description, + } + for entry in runtime.routes.values() + ] + + async def dispatch(self, page_id: str, api_path: str, request: Request, user: Any) -> Response: + page_id = self._store.validate_page_id(page_id) + if not self._store.page_dir(page_id).is_dir(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"page not found: {page_id}") + await self._guard_request_size(request) + + runtime = await self._load_page_runtime(page_id, force_reload=False) + normalized_path = "/" + api_path.strip("/") + key = (request.method.upper(), normalized_path) + entry = runtime.routes.get(key) + if entry is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="page api route not found") + + ctx = self._create_context(page_id, user) + try: + result = await asyncio.wait_for( + self._invoke_handler(entry.handler, ctx, request), + timeout=entry.spec.timeout_ms / 1000, + ) + except asyncio.TimeoutError as exc: + await self._mark_failed(page_id, "page api handler timed out") + raise HTTPException(status_code=status.HTTP_504_GATEWAY_TIMEOUT, detail="page api handler timed out") from exc + except HTTPException: + raise + except Exception as exc: + await self._mark_failed(page_id, f"handler execution failed: {exc}") + log.warning("user_defined_pages.api.handler_failed", {"pageId": page_id, "error": str(exc)}) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="page api execution failed") from exc + + try: + response = self._normalize_response(result) + except ValueError as exc: + await self._mark_failed(page_id, str(exc)) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + + return response + + async def _guard_request_size(self, request: Request) -> None: + header_value = request.headers.get("content-length") + if header_value: + try: + content_length = int(header_value) + except ValueError: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="invalid content-length header") + if content_length > _MAX_REQUEST_BODY_BYTES: + raise HTTPException( + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + detail="request body is too large", + ) + + # Enforce upper bound even when content-length is missing/spoofed. + body = bytearray() + async for chunk in request.stream(): + body.extend(chunk) + if len(body) > _MAX_REQUEST_BODY_BYTES: + raise HTTPException( + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + detail="request body is too large", + ) + request._body = bytes(body) # type: ignore[attr-defined] + + async def _load_page_runtime(self, page_id: str, *, force_reload: bool) -> _PageRuntime: + page_id = self._store.validate_page_id(page_id) + async with self._lock: + routes_path = self._store.routes_path(page_id) + handlers_path = self._store.api_handlers_path(page_id) + if not routes_path.is_file() or not handlers_path.is_file(): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="page api is not configured") + + routes_mtime_ns = routes_path.stat().st_mtime_ns + handlers_mtime_ns = handlers_path.stat().st_mtime_ns + cached = self._cache.get(page_id) + if ( + not force_reload + and cached is not None + and cached.routes_mtime_ns == routes_mtime_ns + and cached.handlers_mtime_ns == handlers_mtime_ns + ): + return cached + + runtime = self._compile_runtime(page_id, routes_path, handlers_path, routes_mtime_ns, handlers_mtime_ns) + self._cache[page_id] = runtime + self._store.write_api_meta( + page_id, + UserDefinedPageApiMeta( + status="ready", + loadedAt=runtime.loaded_at, + error=None, + routes=[ + { + "method": entry.spec.method, + "path": entry.spec.path, + "handler": entry.spec.handler_name, + } + for entry in runtime.routes.values() + ], + ), + ) + return runtime + + def _compile_runtime( + self, + page_id: str, + routes_path: Path, + handlers_path: Path, + routes_mtime_ns: int, + handlers_mtime_ns: int, + ) -> _PageRuntime: + try: + routes_text = routes_path.read_text(encoding="utf-8") + raw = yaml.safe_load(routes_text) or {} + except Exception as exc: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"invalid routes.yaml: {exc}") from exc + + route_items = raw.get("routes") + if not isinstance(route_items, list): + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="routes.yaml must contain a routes list") + + module_name = f"flocks_user_defined_page_{page_id}_{int(time.time() * 1000)}" + spec = importlib.util.spec_from_file_location(module_name, handlers_path) + if spec is None or spec.loader is None: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="failed to load handlers.py") + module = importlib.util.module_from_spec(spec) + guarded_import = self._create_guarded_import(api_root=handlers_path.parent) + original_import = builtins.__import__ + try: + builtins.__import__ = guarded_import # type: ignore[assignment] + spec.loader.exec_module(module) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"failed to load handlers.py: {exc}", + ) from exc + finally: + builtins.__import__ = original_import # type: ignore[assignment] + + routes: dict[tuple[str, str], _RouteEntry] = {} + for item in route_items: + if not isinstance(item, dict): + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="invalid route item") + method = str(item.get("method", "")).upper().strip() + path = str(item.get("path", "")).strip() + handler_name = str(item.get("handler", "")).strip() + description = str(item.get("description", "")).strip() + timeout_ms_raw = item.get("timeoutMs", _DEFAULT_TIMEOUT_MS) + + if method not in _ALLOWED_METHODS: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"unsupported method: {method}") + if not path.startswith("/") or ".." in path or "//" in path: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"invalid route path: {path}") + normalized_path = "/" + path.strip("/") + if not handler_name: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="route handler is required") + + timeout_ms = _DEFAULT_TIMEOUT_MS + try: + timeout_ms = min(max(int(timeout_ms_raw), 1), _MAX_TIMEOUT_MS) + except Exception: + pass + + callable_name = handler_name.split(".", 1)[1] if handler_name.startswith("handlers.") else handler_name + handler = getattr(module, callable_name, None) + if handler is None or not callable(handler): + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"handler not found: {handler_name}", + ) + + key = (method, normalized_path) + routes[key] = _RouteEntry( + spec=_RouteSpec( + method=method, + path=normalized_path, + handler_name=handler_name, + timeout_ms=timeout_ms, + description=description, + ), + handler=handler, + ) + + return _PageRuntime( + page_id=page_id, + routes=routes, + module=module, + routes_mtime_ns=routes_mtime_ns, + handlers_mtime_ns=handlers_mtime_ns, + loaded_at=int(time.time() * 1000), + ) + + def _create_guarded_import(self, *, api_root: Path): + api_root_resolved = api_root.resolve() + original_import = builtins.__import__ + + def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + module = original_import(name, globals, locals, fromlist, level) + modules_to_check = [module] + if fromlist: + for item in fromlist: + if item == "*": + continue + sub_name = f"{module.__name__}.{item}" + maybe_sub = sys.modules.get(sub_name) + if maybe_sub is not None: + modules_to_check.append(maybe_sub) + for mod in modules_to_check: + if not self._is_allowed_import_module(mod, api_root=api_root_resolved): + origin = getattr(mod, "__file__", None) or getattr(getattr(mod, "__spec__", None), "origin", None) + raise ImportError(f"disallowed import outside page api directory: {origin or mod.__name__}") + return module + + return _guarded_import + + def _is_allowed_import_module(self, module: ModuleType, *, api_root: Path) -> bool: + spec = getattr(module, "__spec__", None) + origin = getattr(spec, "origin", None) + if origin in {None, "built-in", "frozen"}: + return True + try: + origin_path = Path(str(origin)).resolve() + except Exception: + return False + if str(origin_path).startswith(str(api_root)): + return True + if str(origin_path).startswith(str(_STDLIB_DIR)): + return True + return False + + async def _invoke_handler(self, handler: Any, ctx: Any, request: Request) -> Any: + result = handler(ctx, request) + if inspect.isawaitable(result): + return await result + return result + + def _normalize_response(self, result: Any) -> Response: + if isinstance(result, Response): + return result + payload = json.dumps(result, ensure_ascii=False, separators=(",", ":")) + if len(payload.encode("utf-8")) > _MAX_RESPONSE_BYTES: + raise ValueError("response body is too large") + return JSONResponse(content=result) + + def _create_context(self, page_id: str, user: Any) -> Any: + logger = Log.create(service=f"user-defined-page-api:{page_id}") + return SimpleNamespace( + page_id=page_id, + user=user, + secrets=_SecretAccessor(), + logger=logger, + cache=None, + ) + + async def _mark_failed(self, page_id: str, error: str) -> None: + self._store.write_api_meta( + page_id, + UserDefinedPageApiMeta( + status="failed", + loadedAt=int(time.time() * 1000), + error=(error or "page api runtime failed")[:2000], + routes=[], + ), + ) + + +class _SecretAccessor: + def get(self, key: str, default: Any = None) -> Any: + from flocks.security import get_secret_manager + + value = get_secret_manager().get(key) + return default if value is None else value + + +_runtime: Optional[UserDefinedPageApiRuntime] = None + + +def get_api_runtime() -> UserDefinedPageApiRuntime: + global _runtime + if _runtime is None: + _runtime = UserDefinedPageApiRuntime() + return _runtime diff --git a/flocks/user_defined_pages/bootstrap.py b/flocks/user_defined_pages/bootstrap.py new file mode 100644 index 000000000..76fc3100c --- /dev/null +++ b/flocks/user_defined_pages/bootstrap.py @@ -0,0 +1,81 @@ +"""Startup reconciliation for user-defined pages.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +from flocks.user_defined_pages.api_runtime import UserDefinedPageApiRuntime +from flocks.user_defined_pages.builder import RUNTIME_NAME, RUNTIME_VERSION, UserDefinedPagesBuilder +from flocks.user_defined_pages.store import UserDefinedPagesStore +from flocks.utils.log import Log + +log = Log.create(service="user-defined-pages-bootstrap") + +_SOURCE_SUFFIXES = {".ts", ".tsx", ".js", ".jsx", ".css", ".json"} + + +async def reconcile_user_defined_pages( + *, + store: Optional[UserDefinedPagesStore] = None, + builder: Optional[UserDefinedPagesBuilder] = None, + runtime: Optional[UserDefinedPageApiRuntime] = None, +) -> None: + store = store or UserDefinedPagesStore() + builder = builder or UserDefinedPagesBuilder(store) + runtime = runtime or UserDefinedPageApiRuntime(store) + store.ensure_root() + + for page in store.list_pages(enabled_only=False): + page_id = page.id + page_dir = store.page_dir(page_id) + if not page_dir.is_dir(): + continue + + try: + manifest = store.get_page(page_id).manifest + except Exception as exc: + log.warning("user_defined_pages.bootstrap.skip_invalid_manifest", {"pageId": page_id, "error": str(exc)}) + continue + if not manifest.enabled: + continue + + try: + if _should_rebuild_page(store, page_id): + meta = builder.build(page_id) + if meta.status != "ready": + log.warning( + "user_defined_pages.bootstrap.rebuild_failed", + {"pageId": page_id, "error": meta.error or "build failed"}, + ) + except Exception as exc: + log.warning("user_defined_pages.bootstrap.rebuild_error", {"pageId": page_id, "error": str(exc)}) + + try: + if store.routes_path(page_id).is_file(): + # Warm up page API runtime so restart/upgrade immediately serves APIs. + await runtime.reload_page(page_id) + except Exception as exc: + log.warning("user_defined_pages.bootstrap.api_preload_failed", {"pageId": page_id, "error": str(exc)}) + + +def _should_rebuild_page(store: UserDefinedPagesStore, page_id: str) -> bool: + bundle_path = store.bundle_path(page_id) + build_meta = store.read_build_meta(page_id) + if not bundle_path.is_file(): + return True + if build_meta.status == "failed": + return True + if build_meta.runtime != RUNTIME_NAME or build_meta.runtimeVersion != RUNTIME_VERSION: + return True + return _sources_newer_than_bundle(store.page_dir(page_id), bundle_path) + + +def _sources_newer_than_bundle(page_dir: Path, bundle_path: Path) -> bool: + bundle_mtime = bundle_path.stat().st_mtime_ns + for path in (page_dir / "src").rglob("*"): + if not path.is_file() or path.suffix not in _SOURCE_SUFFIXES: + continue + if path.stat().st_mtime_ns > bundle_mtime: + return True + return False diff --git a/flocks/user_defined_pages/builder.py b/flocks/user_defined_pages/builder.py new file mode 100644 index 000000000..895d7486c --- /dev/null +++ b/flocks/user_defined_pages/builder.py @@ -0,0 +1,168 @@ +"""Build user-defined page TSX sources into browser-loadable ESM bundles.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Optional + +from flocks.user_defined_pages.models import UserDefinedPageBuildMeta +from flocks.user_defined_pages.store import UserDefinedPagesStore +from flocks.utils.log import Log + +log = Log.create(service="user-defined-pages-builder") + +MAX_OUTPUT_BYTES = 2_000_000 +BUILD_TIMEOUT_SECONDS = 30 +_SHIMS_DIR = Path(__file__).resolve().parent / "shims" +RUNTIME_NAME = "user_defined_page" +RUNTIME_VERSION = 1 +SDK_IMPORT_NAME = "@flocks/user-defined-page-sdk" + + +def _repo_root() -> Path: + # flocks/user_defined_pages/builder.py -> repo root is parents[2] + return Path(__file__).resolve().parents[2] + + +def resolve_esbuild_bin() -> Optional[Path]: + """Locate esbuild from the bundled webui toolchain.""" + webui_bin = _repo_root() / "webui" / "node_modules" / ".bin" + if sys.platform == "win32": + candidate = webui_bin / "esbuild.cmd" + if candidate.is_file(): + return candidate + candidate = webui_bin / "esbuild" + if candidate.is_file(): + return candidate + return None + + +class UserDefinedPagesBuilder: + """Compile a page entry file into dist/page.js.""" + + def __init__(self, store: Optional[UserDefinedPagesStore] = None) -> None: + self._store = store or UserDefinedPagesStore() + + def build(self, page_id: str) -> UserDefinedPageBuildMeta: + page_id = self._store.validate_page_id(page_id) + detail = self._store.get_page(page_id) + page_dir = self._store.page_dir(page_id) + entry = detail.manifest.entry.replace("\\", "/") + entry_path = (page_dir / entry).resolve() + try: + entry_path.relative_to(page_dir.resolve()) + except ValueError: + raise ValueError("invalid entry path") + if not entry_path.is_file(): + raise FileNotFoundError(f"entry file not found: {entry}") + + esbuild = resolve_esbuild_bin() + if esbuild is None: + raise RuntimeError("esbuild is not available; install webui dependencies first") + + dist_dir = page_dir / "dist" + dist_dir.mkdir(parents=True, exist_ok=True) + outfile = dist_dir / "page.js" + + building = UserDefinedPageBuildMeta(status="building", hash="", builtAt=0, error=None) + self._store.write_build_meta(page_id, building) + + cmd = [ + str(esbuild), + str(entry_path), + "--bundle", + "--format=esm", + f"--outfile={outfile}", + "--platform=browser", + "--target=es2020", + "--jsx=automatic", + f"--alias:react={_SHIMS_DIR / 'react.js'}", + f"--alias:react/jsx-runtime={_SHIMS_DIR / 'jsx-runtime.js'}", + f"--alias:@flocks/user-defined-page-sdk={_SHIMS_DIR / 'sdk.js'}", + ] + + env = os.environ.copy() + try: + result = subprocess.run( + cmd, + cwd=str(page_dir), + capture_output=True, + text=True, + timeout=BUILD_TIMEOUT_SECONDS, + env=env, + check=False, + ) + except subprocess.TimeoutExpired as exc: + meta = UserDefinedPageBuildMeta( + status="failed", + hash="", + builtAt=int(time.time() * 1000), + error=f"build timed out after {BUILD_TIMEOUT_SECONDS}s", + runtime=RUNTIME_NAME, + runtimeVersion=RUNTIME_VERSION, + sdkImport=SDK_IMPORT_NAME, + ) + self._store.write_build_meta(page_id, meta) + raise RuntimeError(meta.error) from exc + + if result.returncode != 0: + stderr = (result.stderr or result.stdout or "esbuild failed").strip() + meta = UserDefinedPageBuildMeta( + status="failed", + hash="", + builtAt=int(time.time() * 1000), + error=stderr[:4000], + runtime=RUNTIME_NAME, + runtimeVersion=RUNTIME_VERSION, + sdkImport=SDK_IMPORT_NAME, + ) + self._store.write_build_meta(page_id, meta) + log.warning("user_defined_pages.build.failed", {"pageId": page_id, "error": stderr[:500]}) + return meta + + if not outfile.is_file(): + meta = UserDefinedPageBuildMeta( + status="failed", + hash="", + builtAt=int(time.time() * 1000), + error="build produced no output", + runtime=RUNTIME_NAME, + runtimeVersion=RUNTIME_VERSION, + sdkImport=SDK_IMPORT_NAME, + ) + self._store.write_build_meta(page_id, meta) + return meta + + content = outfile.read_bytes() + if len(content) > MAX_OUTPUT_BYTES: + outfile.unlink(missing_ok=True) + meta = UserDefinedPageBuildMeta( + status="failed", + hash="", + builtAt=int(time.time() * 1000), + error="build output is too large", + runtime=RUNTIME_NAME, + runtimeVersion=RUNTIME_VERSION, + sdkImport=SDK_IMPORT_NAME, + ) + self._store.write_build_meta(page_id, meta) + return meta + + digest = hashlib.sha256(content).hexdigest()[:16] + meta = UserDefinedPageBuildMeta( + status="ready", + hash=digest, + builtAt=int(time.time() * 1000), + error=None, + runtime=RUNTIME_NAME, + runtimeVersion=RUNTIME_VERSION, + sdkImport=SDK_IMPORT_NAME, + ) + self._store.write_build_meta(page_id, meta) + log.info("user_defined_pages.build.ready", {"pageId": page_id, "hash": digest}) + return meta diff --git a/flocks/user_defined_pages/models.py b/flocks/user_defined_pages/models.py new file mode 100644 index 000000000..02189fca4 --- /dev/null +++ b/flocks/user_defined_pages/models.py @@ -0,0 +1,67 @@ +"""Pydantic models for user-defined custom pages.""" + +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class UserDefinedPageManifest(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + id: str = Field(..., description="Stable page identifier") + title: str = Field(..., description="Navigation label") + route: str = Field(..., description="WebUI route path") + icon: str = Field("LayoutDashboard", description="Lucide icon name") + order: int = Field(100, description="Sort order in navigation") + enabled: bool = Field(True, description="Whether page appears in navigation") + placement: Literal["home.after"] = Field( + "home.after", + description="Where to insert the nav item", + ) + entry: str = Field("src/index.tsx", description="Source entry relative to page dir") + updatedAt: int = Field(0, description="Last manifest update timestamp (ms)") + + +class UserDefinedPageBuildMeta(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + hash: str = Field("", description="Content hash for cache busting") + builtAt: int = Field(0, description="Build timestamp (ms)") + status: Literal["idle", "building", "ready", "failed"] = Field("idle") + error: Optional[str] = Field(None, description="Last build error message") + runtime: str = Field("user_defined_page", description="Builder runtime marker") + runtimeVersion: int = Field(1, description="Builder runtime version") + sdkImport: str = Field("@flocks/user-defined-page-sdk", description="SDK import marker") + + +class UserDefinedPageApiMeta(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + status: Literal["idle", "ready", "failed"] = Field("idle") + loadedAt: int = Field(0, description="Runtime load timestamp (ms)") + error: Optional[str] = Field(None, description="Last API runtime error") + routes: list[dict[str, str]] = Field(default_factory=list, description="Loaded route descriptors") + + +class UserDefinedPageListItem(BaseModel): + model_config = ConfigDict(populate_by_name=True, by_alias=True) + + id: str + title: str + route: str + icon: str + order: int + enabled: bool + placement: str + buildHash: str = Field("", alias="buildHash") + buildStatus: str = Field("idle", alias="buildStatus") + + +class UserDefinedPageDetail(BaseModel): + model_config = ConfigDict(populate_by_name=True, by_alias=True) + + manifest: UserDefinedPageManifest + build: UserDefinedPageBuildMeta + sourceFiles: list[str] = Field(default_factory=list, alias="sourceFiles") diff --git a/flocks/user_defined_pages/shims/jsx-runtime.js b/flocks/user_defined_pages/shims/jsx-runtime.js new file mode 100644 index 000000000..dc2740653 --- /dev/null +++ b/flocks/user_defined_pages/shims/jsx-runtime.js @@ -0,0 +1,7 @@ +const runtime = globalThis.__FLOCKS_USER_DEFINED_PAGE_SDK__; +if (!runtime?.jsx || !runtime?.jsxs) { + throw new Error('Flocks user-defined page runtime is not initialized (missing jsx runtime).'); +} +export const jsx = runtime.jsx; +export const jsxs = runtime.jsxs; +export const Fragment = runtime.React.Fragment; diff --git a/flocks/user_defined_pages/shims/react.js b/flocks/user_defined_pages/shims/react.js new file mode 100644 index 000000000..94ddbe5eb --- /dev/null +++ b/flocks/user_defined_pages/shims/react.js @@ -0,0 +1,39 @@ +const React = globalThis.__FLOCKS_USER_DEFINED_PAGE_SDK__?.React; +if (!React) { + throw new Error('Flocks user-defined page runtime is not initialized (missing React).'); +} +export default React; +export const { + Children, + Component, + Fragment, + Profiler, + PureComponent, + StrictMode, + Suspense, + cloneElement, + createContext, + createElement, + createRef, + forwardRef, + isValidElement, + lazy, + memo, + startTransition, + useCallback, + useContext, + useDebugValue, + useDeferredValue, + useEffect, + useId, + useImperativeHandle, + useInsertionEffect, + useLayoutEffect, + useMemo, + useReducer, + useRef, + useState, + useSyncExternalStore, + useTransition, + version, +} = React; diff --git a/flocks/user_defined_pages/shims/sdk.js b/flocks/user_defined_pages/shims/sdk.js new file mode 100644 index 000000000..81efdd874 --- /dev/null +++ b/flocks/user_defined_pages/shims/sdk.js @@ -0,0 +1,7 @@ +const sdk = globalThis.__FLOCKS_USER_DEFINED_PAGE_SDK__; +if (!sdk) { + throw new Error('Flocks user-defined page runtime is not initialized (missing SDK).'); +} +export const api = sdk.api; +export const Card = sdk.Card; +export const useCurrentUser = sdk.useCurrentUser; diff --git a/flocks/user_defined_pages/store.py b/flocks/user_defined_pages/store.py new file mode 100644 index 000000000..0160f0395 --- /dev/null +++ b/flocks/user_defined_pages/store.py @@ -0,0 +1,331 @@ +"""Filesystem store for user-defined pages.""" + +from __future__ import annotations + +import json +import os +import re +import time +from pathlib import Path +from typing import Any, Optional + +from flocks.user_defined_pages.models import ( + UserDefinedPageApiMeta, + UserDefinedPageBuildMeta, + UserDefinedPageDetail, + UserDefinedPageListItem, + UserDefinedPageManifest, +) +from flocks.utils.log import Log + +log = Log.create(service="user-defined-pages-store") + +PAGE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +MAX_SOURCE_FILE_BYTES = 512_000 +ALLOWED_WRITE_PREFIXES = ("src/", "assets/", "api/") +ALLOWED_WRITE_FILES = frozenset({"manifest.json"}) +_SOURCE_SUFFIXES = {".tsx", ".ts", ".jsx", ".js", ".css", ".json"} +_API_SUFFIXES = {".py", ".yaml", ".yml"} + +def _default_page_tsx(title: str) -> str: + safe_title = title.replace("\\", "\\\\").replace('"', '\\"') + return f"""import {{ useEffect, useState }} from 'react'; +import {{ Card }} from '@flocks/user-defined-page-sdk'; + +export default function Page() {{ + const [ready, setReady] = useState(false); + + useEffect(() => {{ + setReady(true); + }}, []); + + return ( + + {{ready ? 'Ready' : 'Loading...'}} + + ); +}} +""" + +_DEFAULT_INDEX_TSX = """import Page from './Page'; + +export default Page; +""" + + +def get_user_defined_pages_root() -> Path: + """Return canonical user-space root for user-defined pages.""" + override = os.environ.get("FLOCKS_USER_DEFINED_PAGES_ROOT") + if override: + return Path(override).expanduser().resolve() + return (Path.home() / ".flocks" / "plugins" / "user_defined_pages").resolve() + + +class UserDefinedPagesStore: + """CRUD and scan helpers for ~/.flocks/plugins/user_defined_pages.""" + + def __init__(self, root: Optional[Path] = None) -> None: + self._root = (root or get_user_defined_pages_root()).resolve() + + @property + def root(self) -> Path: + return self._root + + def ensure_root(self) -> Path: + self._root.mkdir(parents=True, exist_ok=True) + return self._root + + @staticmethod + def validate_page_id(page_id: str) -> str: + normalized = (page_id or "").strip().lower() + if not PAGE_ID_RE.fullmatch(normalized): + raise ValueError("invalid page id: use lowercase letters, numbers, and hyphens") + return normalized + + def page_dir(self, page_id: str) -> Path: + page_id = self.validate_page_id(page_id) + page_path = (self._root / page_id).resolve() + try: + page_path.relative_to(self._root) + except ValueError: + raise ValueError("invalid page path") + return page_path + + def _assert_writable_relative(self, relative_path: str) -> Path: + if not relative_path or Path(relative_path).is_absolute(): + raise ValueError("absolute path is not allowed") + rel = relative_path.replace("\\", "/").lstrip("/") + if rel in ALLOWED_WRITE_FILES: + return Path(rel) + if any(rel.startswith(prefix) for prefix in ALLOWED_WRITE_PREFIXES): + parts = rel.split("/") + if ".." in parts: + raise ValueError("path traversal is not allowed") + if any(part.startswith(".") for part in parts if part): + raise ValueError("hidden path is not allowed") + return Path(rel) + raise ValueError(f"writes are not allowed for path: {relative_path}") + + def list_pages(self, *, enabled_only: bool = False) -> list[UserDefinedPageListItem]: + self.ensure_root() + items: list[UserDefinedPageListItem] = [] + for child in sorted(self._root.iterdir()): + if not child.is_dir(): + continue + manifest = self._read_manifest(child.name) + if manifest is None: + continue + if enabled_only and not manifest.enabled: + continue + build = self._read_build_meta(child.name) + items.append( + UserDefinedPageListItem( + id=manifest.id, + title=manifest.title, + route=manifest.route, + icon=manifest.icon, + order=manifest.order, + enabled=manifest.enabled, + placement=manifest.placement, + buildHash=build.hash, + buildStatus=build.status, + ) + ) + items.sort(key=lambda item: (item.order, item.title)) + return items + + def get_page(self, page_id: str) -> UserDefinedPageDetail: + page_dir = self.page_dir(page_id) + if not page_dir.is_dir(): + raise FileNotFoundError(f"page not found: {page_id}") + manifest = self._read_manifest(page_id) + if manifest is None: + raise FileNotFoundError(f"manifest missing for page: {page_id}") + build = self._read_build_meta(page_id) + source_files = sorted( + str(path.relative_to(page_dir)).replace("\\", "/") + for path in page_dir.rglob("*") + if path.is_file() and "dist/" not in str(path.relative_to(page_dir)).replace("\\", "/") + ) + return UserDefinedPageDetail(manifest=manifest, build=build, sourceFiles=source_files) + + def create_page( + self, + *, + page_id: str, + title: str, + icon: str = "LayoutDashboard", + order: int = 100, + ) -> UserDefinedPageDetail: + page_id = self.validate_page_id(page_id) + page_dir = self.page_dir(page_id) + if page_dir.exists(): + raise FileExistsError(f"page already exists: {page_id}") + + now_ms = int(time.time() * 1000) + manifest = UserDefinedPageManifest( + id=page_id, + title=title.strip() or page_id, + route=f"/user-defined-pages/{page_id}", + icon=icon, + order=order, + enabled=True, + placement="home.after", + entry="src/index.tsx", + updatedAt=now_ms, + ) + + page_dir.mkdir(parents=True, exist_ok=False) + (page_dir / "src").mkdir(parents=True, exist_ok=True) + (page_dir / "api").mkdir(parents=True, exist_ok=True) + (page_dir / "assets").mkdir(parents=True, exist_ok=True) + (page_dir / "dist").mkdir(parents=True, exist_ok=True) + + self._write_manifest(page_id, manifest) + self._write_source_file(page_id, "src/Page.tsx", _default_page_tsx(manifest.title)) + self._write_source_file(page_id, "src/index.tsx", _DEFAULT_INDEX_TSX) + self._write_build_meta( + page_id, + UserDefinedPageBuildMeta(status="idle", hash="", builtAt=0, error=None), + ) + log.info("user_defined_pages.created", {"pageId": page_id}) + return self.get_page(page_id) + + def save_manifest(self, page_id: str, manifest_data: dict[str, Any]) -> UserDefinedPageManifest: + page_id = self.validate_page_id(page_id) + existing = self._read_manifest(page_id) + if existing is None: + raise FileNotFoundError(f"page not found: {page_id}") + + merged = existing.model_dump() + merged.update(manifest_data) + merged["id"] = page_id + merged["route"] = f"/user-defined-pages/{page_id}" + merged["updatedAt"] = int(time.time() * 1000) + manifest = UserDefinedPageManifest.model_validate(merged) + self._write_manifest(page_id, manifest) + return manifest + + def save_source_file(self, page_id: str, relative_path: str, content: str) -> None: + rel = self._assert_writable_relative(relative_path) + rel_str = str(rel).replace("\\", "/") + if rel_str.startswith("api/"): + allowed_suffixes = _API_SUFFIXES + else: + allowed_suffixes = _SOURCE_SUFFIXES + if rel.suffix not in allowed_suffixes: + raise ValueError("unsupported source file type") + encoded = content.encode("utf-8") + if len(encoded) > MAX_SOURCE_FILE_BYTES: + raise ValueError("source file is too large") + self._write_source_file(page_id, rel_str, content) + + def read_source_file(self, page_id: str, relative_path: str) -> str: + rel = self._assert_writable_relative(relative_path) + path = self.page_dir(page_id) / rel + if not path.is_file(): + raise FileNotFoundError(relative_path) + return path.read_text(encoding="utf-8") + + def bundle_path(self, page_id: str) -> Path: + return self.page_dir(page_id) / "dist" / "page.js" + + def asset_path(self, page_id: str, relative_path: str) -> Path: + rel = relative_path.replace("\\", "/").lstrip("/") + if ".." in rel.split("/"): + raise ValueError("path traversal is not allowed") + path = (self.page_dir(page_id) / "assets" / rel).resolve() + assets_root = (self.page_dir(page_id) / "assets").resolve() + try: + path.relative_to(assets_root) + except ValueError: + raise ValueError("invalid asset path") + return path + + def write_build_meta(self, page_id: str, meta: UserDefinedPageBuildMeta) -> None: + self._write_build_meta(page_id, meta) + + def read_build_meta(self, page_id: str) -> UserDefinedPageBuildMeta: + return self._read_build_meta(page_id) + + def routes_path(self, page_id: str) -> Path: + return self.page_dir(page_id) / "api" / "routes.yaml" + + def api_handlers_path(self, page_id: str) -> Path: + return self.page_dir(page_id) / "api" / "handlers.py" + + def read_api_routes(self, page_id: str) -> Optional[str]: + path = self.routes_path(page_id) + if not path.is_file(): + return None + return path.read_text(encoding="utf-8") + + def write_api_meta(self, page_id: str, meta: UserDefinedPageApiMeta) -> None: + path = self._api_meta_path(page_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(meta.model_dump(), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + def read_api_meta(self, page_id: str) -> UserDefinedPageApiMeta: + path = self._api_meta_path(page_id) + if not path.is_file(): + return UserDefinedPageApiMeta() + try: + raw = json.loads(path.read_text(encoding="utf-8")) + return UserDefinedPageApiMeta.model_validate(raw) + except Exception: + return UserDefinedPageApiMeta() + + def _manifest_path(self, page_id: str) -> Path: + return self.page_dir(page_id) / "manifest.json" + + def _build_meta_path(self, page_id: str) -> Path: + return self.page_dir(page_id) / "dist" / "meta.json" + + def _api_meta_path(self, page_id: str) -> Path: + return self.page_dir(page_id) / "dist" / "api-meta.json" + + def _read_manifest(self, page_id: str) -> Optional[UserDefinedPageManifest]: + path = self._manifest_path(page_id) + if not path.is_file(): + return None + try: + raw = json.loads(path.read_text(encoding="utf-8")) + return UserDefinedPageManifest.model_validate(raw) + except Exception as exc: + log.warning("user_defined_pages.manifest.invalid", {"pageId": page_id, "error": str(exc)}) + return None + + def _write_manifest(self, page_id: str, manifest: UserDefinedPageManifest) -> None: + path = self._manifest_path(page_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(manifest.model_dump(), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + def _read_build_meta(self, page_id: str) -> UserDefinedPageBuildMeta: + path = self._build_meta_path(page_id) + if not path.is_file(): + return UserDefinedPageBuildMeta() + try: + raw = json.loads(path.read_text(encoding="utf-8")) + return UserDefinedPageBuildMeta.model_validate(raw) + except Exception: + return UserDefinedPageBuildMeta() + + def _write_build_meta(self, page_id: str, meta: UserDefinedPageBuildMeta) -> None: + path = self._build_meta_path(page_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(meta.model_dump(), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + def _write_source_file(self, page_id: str, relative_path: str, content: str) -> None: + rel = self._assert_writable_relative(relative_path) + target = self.page_dir(page_id) / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") diff --git a/flocks/user_defined_pages/watcher.py b/flocks/user_defined_pages/watcher.py new file mode 100644 index 000000000..49769cafc --- /dev/null +++ b/flocks/user_defined_pages/watcher.py @@ -0,0 +1,235 @@ +"""Watch ~/.flocks/plugins/user_defined_pages for changes and trigger rebuilds.""" + +from __future__ import annotations + +import asyncio +import threading +from dataclasses import dataclass +from concurrent.futures import TimeoutError as FutureTimeoutError +from pathlib import Path +from typing import Any, Callable, Coroutine, Optional + +from flocks.user_defined_pages.api_runtime import UserDefinedPageApiRuntime +from flocks.user_defined_pages.builder import UserDefinedPagesBuilder +from flocks.user_defined_pages.store import UserDefinedPagesStore +from flocks.server.routes.event import publish_event +from flocks.utils.log import Log + +log = Log.create(service="user-defined-pages-watcher") + +_DEBOUNCE_SECONDS = 0.8 +_RELOAD_EVENT_TYPES = frozenset({"modified", "created", "deleted", "moved"}) + +_main_loop: Optional[asyncio.AbstractEventLoop] = None + + +def set_event_loop(loop: asyncio.AbstractEventLoop) -> None: + """Register the FastAPI event loop for cross-thread SSE publishing.""" + global _main_loop + _main_loop = loop + + +def _publish_event_sync(event_type: str, properties: dict) -> None: + if _main_loop is None: + return + try: + asyncio.run_coroutine_threadsafe( + publish_event(event_type, properties), + _main_loop, + ) + except Exception as exc: + log.warning("user_defined_pages.event.publish_failed", {"type": event_type, "error": str(exc)}) + + +def _run_on_main_loop_sync(coro: Coroutine[Any, Any, Any], *, timeout_seconds: float = 5.0) -> Any: + if _main_loop is None: + raise RuntimeError("main event loop is not ready") + future = asyncio.run_coroutine_threadsafe(coro, _main_loop) + try: + return future.result(timeout=timeout_seconds) + except FutureTimeoutError as exc: + future.cancel() + raise TimeoutError("main loop task timed out") from exc + + +@dataclass +class _PendingAction: + manifest_changed: bool = False + source_changed: bool = False + api_changed: bool = False + page_removed: bool = False + + +class UserDefinedPagesWatcher: + """Debounced filesystem watcher for user-defined pages.""" + + def __init__( + self, + *, + store: Optional[UserDefinedPagesStore] = None, + builder: Optional[UserDefinedPagesBuilder] = None, + api_runtime: Optional[UserDefinedPageApiRuntime] = None, + on_build_complete: Optional[Callable[[str, bool, Optional[str]], None]] = None, + ) -> None: + self._store = store or UserDefinedPagesStore() + self._builder = builder or UserDefinedPagesBuilder(self._store) + self._api_runtime = api_runtime or UserDefinedPageApiRuntime(self._store) + self._on_build_complete = on_build_complete + self._observer: Optional[object] = None + self._debounce_timer: Optional[threading.Timer] = None + self._lock = threading.Lock() + self._pending_pages: dict[str, _PendingAction] = {} + + def start(self) -> None: + try: + from watchdog.events import FileSystemEvent, FileSystemEventHandler + from watchdog.observers import Observer + except ImportError: + log.warning( + "user_defined_pages.watcher.watchdog_missing", + {"msg": "watchdog not installed, user defined pages watcher disabled"}, + ) + return + + root = self._store.ensure_root() + watcher = self + + class _Handler(FileSystemEventHandler): + def on_any_event(self, event: FileSystemEvent) -> None: + if getattr(event, "event_type", "") not in _RELOAD_EVENT_TYPES: + return + src = Path(getattr(event, "src_path", "")) + event_type = getattr(event, "event_type", "") + action = watcher._classify_event(src, root, event_type=event_type, is_directory=event.is_directory) + if action is None: + return + page_id, pending = action + watcher._schedule(page_id, pending) + + handler = _Handler() + observer = Observer() + observer.schedule(handler, str(root), recursive=True) + observer.daemon = True + observer.start() + self._observer = observer + log.info("user_defined_pages.watcher.started", {"directory": str(root)}) + + def stop(self) -> None: + with self._lock: + if self._debounce_timer is not None: + self._debounce_timer.cancel() + self._debounce_timer = None + self._pending_pages.clear() + if self._observer is not None: + try: + self._observer.stop() # type: ignore[union-attr] + self._observer.join(timeout=2) # type: ignore[union-attr] + except Exception: + pass + self._observer = None + log.info("user_defined_pages.watcher.stopped") + + def _classify_event( + self, + src: Path, + root: Path, + *, + event_type: str, + is_directory: bool, + ) -> Optional[tuple[str, _PendingAction]]: + try: + rel = src.resolve().relative_to(root.resolve()) + except Exception: + return None + if not rel.parts: + return None + page_id = rel.parts[0] + if is_directory and len(rel.parts) == 1 and event_type == "deleted": + return page_id, _PendingAction(page_removed=True) + if len(rel.parts) < 2: + return None + rel_str = str(Path(*rel.parts[1:])).replace("\\", "/") + if rel_str == "manifest.json": + return page_id, _PendingAction(manifest_changed=True) + if rel_str.startswith("src/") and rel.suffix in {".ts", ".tsx", ".js", ".jsx", ".css"}: + return page_id, _PendingAction(source_changed=True) + if rel_str == "api/routes.yaml" or (rel_str.startswith("api/") and rel.suffix == ".py"): + return page_id, _PendingAction(api_changed=True) + return None + + def _schedule(self, page_id: str, update: _PendingAction) -> None: + with self._lock: + pending = self._pending_pages.get(page_id, _PendingAction()) + pending.manifest_changed = pending.manifest_changed or update.manifest_changed + pending.source_changed = pending.source_changed or update.source_changed + pending.api_changed = pending.api_changed or update.api_changed + pending.page_removed = pending.page_removed or update.page_removed + self._pending_pages[page_id] = pending + if self._debounce_timer is not None: + self._debounce_timer.cancel() + self._debounce_timer = threading.Timer(_DEBOUNCE_SECONDS, self._run_pending_builds) + self._debounce_timer.daemon = True + self._debounce_timer.start() + + def _run_pending_builds(self) -> None: + with self._lock: + pages = dict(self._pending_pages) + self._pending_pages.clear() + + for page_id, pending in pages.items(): + if pending.page_removed: + self._api_runtime.clear_page(page_id) + _publish_event_sync("user_defined_pages.nav_changed", {"id": page_id}) + continue + + if pending.source_changed: + try: + meta = self._builder.build(page_id) + if meta.status == "ready": + _publish_event_sync("user_defined_pages.updated", {"id": page_id, "hash": meta.hash}) + _publish_event_sync("user_defined_pages.nav_changed", {"id": page_id}) + else: + _publish_event_sync( + "user_defined_pages.build_failed", + {"id": page_id, "error": meta.error or "build failed"}, + ) + if self._on_build_complete: + self._on_build_complete(page_id, meta.status == "ready", meta.error) + except Exception as exc: + _publish_event_sync( + "user_defined_pages.build_failed", + {"id": page_id, "error": str(exc)}, + ) + log.warning("user_defined_pages.watcher.build_failed", {"pageId": page_id, "error": str(exc)}) + + if pending.api_changed: + try: + routes = _run_on_main_loop_sync(self._api_runtime.reload_page(page_id)) + _publish_event_sync("user_defined_pages.api_changed", {"id": page_id, "routes": routes}) + except Exception as exc: + _publish_event_sync("user_defined_pages.api_failed", {"id": page_id, "error": str(exc)}) + log.warning("user_defined_pages.watcher.api_reload_failed", {"pageId": page_id, "error": str(exc)}) + + if pending.manifest_changed and not pending.source_changed: + _publish_event_sync("user_defined_pages.nav_changed", {"id": page_id}) + + +_watcher: Optional[UserDefinedPagesWatcher] = None + + +def get_watcher() -> UserDefinedPagesWatcher: + global _watcher + if _watcher is None: + _watcher = UserDefinedPagesWatcher() + return _watcher + + +def start_watcher() -> None: + get_watcher().start() + + +def stop_watcher() -> None: + global _watcher + if _watcher is not None: + _watcher.stop() + _watcher = None diff --git a/flocks/utils/log.py b/flocks/utils/log.py index f22d7a4fb..9ea3850cd 100644 --- a/flocks/utils/log.py +++ b/flocks/utils/log.py @@ -11,15 +11,15 @@ import sys import time import threading +import shutil from pathlib import Path from typing import Any, Dict, Optional, TextIO -from datetime import datetime +from datetime import date, datetime, timedelta import json import glob as file_glob -_DEFAULT_LOG_MAX_BYTES = 5 * 1024 * 1024 -_DEFAULT_LOG_BACKUP_COUNT = 3 +_DEFAULT_LOG_RETENTION_DAYS = 30 _DEFAULT_LOG_VALUE_MAX_CHARS = 8 * 1024 _MAX_STRUCTURED_ITEMS = 50 _MAX_STRUCTURED_DEPTH = 4 @@ -37,7 +37,7 @@ def _log_dir() -> Path: def get_log_dir() -> Path: - """Return the log directory for file handlers (e.g. workflow). Same as Log.init() uses.""" + """Return the root log directory used by Flocks.""" return _log_dir() @@ -51,99 +51,29 @@ def _env_int(name: str, default: int) -> int: return default -def get_log_max_bytes(default: int = _DEFAULT_LOG_MAX_BYTES) -> int: - """Return the per-file log size limit in bytes. - - ``FLOCKS_LOG_MAX_BYTES`` is exact; ``FLOCKS_LOG_MAX_MB`` is a convenient - human-facing override. When both are set, bytes wins. Values <= 0 disable - rotation. - """ - if os.getenv("FLOCKS_LOG_MAX_BYTES") is not None: - return _env_int("FLOCKS_LOG_MAX_BYTES", default) - max_mb = os.getenv("FLOCKS_LOG_MAX_MB") - if max_mb is not None: - try: - return int(float(max_mb) * 1024 * 1024) - except ValueError: - return default - return default - - -def get_log_backup_count(default: int = _DEFAULT_LOG_BACKUP_COUNT) -> int: - """Return how many rotated backups to keep for long-lived log files.""" - return max(0, _env_int("FLOCKS_LOG_BACKUP_COUNT", default)) - - -def rotate_log_file( - path: Path, - *, - max_bytes: Optional[int] = None, - backup_count: Optional[int] = None, - force: bool = False, -) -> None: - """Rotate ``path`` if it is already over the configured size limit.""" - limit = get_log_max_bytes() if max_bytes is None else max_bytes - backups = get_log_backup_count() if backup_count is None else backup_count - if limit <= 0 or not path.exists(): - return - try: - if not force and path.stat().st_size < limit: - return - if backups <= 0: - path.unlink(missing_ok=True) - return - for index in range(backups - 1, 0, -1): - src = path.with_name(f"{path.name}.{index}") - dst = path.with_name(f"{path.name}.{index + 1}") - if src.exists(): - src.replace(dst) - path.replace(path.with_name(f"{path.name}.1")) - except OSError: - return +def get_log_retention_days(default: int = _DEFAULT_LOG_RETENTION_DAYS) -> int: + """Return how long daily log directories and legacy timestamp logs are retained.""" + return _env_int("FLOCKS_LOG_RETENTION_DAYS", default) -class _RotatingTextWriter: - """Small line-buffered writer with size-based rotation for Flocks logs.""" +class _AppendTextWriter: + """Small line-buffered writer for Flocks daily logs.""" - def __init__(self, path: Path, *, max_bytes: int, backup_count: int): + def __init__(self, path: Path): self.path = path - self.max_bytes = max_bytes - self.backup_count = backup_count self._handle: Optional[TextIO] = None - self._bytes_written = 0 self._lock = threading.RLock() self._open() def _open(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) self._handle = open(self.path, "a", buffering=1, encoding="utf-8") - try: - self._bytes_written = self.path.stat().st_size - except OSError: - self._bytes_written = 0 - - def _should_rotate(self, message: str) -> bool: - if self.max_bytes <= 0: - return False - return self._bytes_written + len(message.encode("utf-8")) > self.max_bytes def write(self, message: str) -> int: - encoded_len = len(message.encode("utf-8")) with self._lock: - if self._should_rotate(message): - self.close() - rotate_log_file( - self.path, - max_bytes=self.max_bytes, - backup_count=self.backup_count, - force=True, - ) - self._open() if self._handle is None: self._open() - written = self._handle.write(message) - self._bytes_written += encoded_len - return written + return self._handle.write(message) def flush(self) -> None: with self._lock: @@ -213,15 +143,16 @@ def _format_log_value(value: Any) -> str: def append_upgrade_text_log(message: str) -> None: - """Append timestamped lines to ``update.log`` under the configured log directory. + """Append timestamped upgrade lines to today's ``errors.log``. Used for upgrade flows so errors remain on disk when the process had no TTY - or when structured ``Log`` output went to a different file than ``backend.log``. + or when structured ``Log`` output was not initialized. """ try: log_dir = _log_dir() - log_dir.mkdir(parents=True, exist_ok=True) - path = log_dir / "update.log" + day_dir = log_dir / date.today().isoformat() + day_dir.mkdir(parents=True, exist_ok=True) + path = day_dir / "errors.log" stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") normalized = message.replace("\r\n", "\n").replace("\r", "\n") with path.open("a", encoding="utf-8") as handle: @@ -310,12 +241,12 @@ def info(self, message: Any = None, extra: Optional[Dict[str, Any]] = None) -> N def warn(self, message: Any = None, extra: Optional[Dict[str, Any]] = None) -> None: """Log warning message""" if Log._should_log(LogLevel.WARN): - Log._write("WARN " + self._build_message(message, extra)) + Log._write("WARN " + self._build_message(message, extra), error=True) def error(self, message: Any = None, extra: Optional[Dict[str, Any]] = None) -> None: """Log error message""" if Log._should_log(LogLevel.ERROR): - Log._write("ERROR " + self._build_message(message, extra)) + Log._write("ERROR " + self._build_message(message, extra), error=True) # Alias for compatibility with standard logging library warning = warn @@ -403,6 +334,10 @@ class Log: _last_time: int = int(time.time() * 1000) _log_file: Optional[Path] = None _writer: Optional[TextIO] = None + _error_writer: Optional[TextIO] = None + _log_dir_path: Optional[Path] = None + _log_date: Optional[str] = None + _state_lock = threading.RLock() # Default logger instance Default: Logger = None # Will be initialized @@ -413,16 +348,21 @@ def _should_log(cls, level: str) -> bool: return _LEVEL_PRIORITY.get(level, 0) >= _LEVEL_PRIORITY.get(cls._level, 1) @classmethod - def _write(cls, message: str) -> int: + def _write(cls, message: str, *, error: bool = False) -> int: """Write log message to file and/or stderr""" try: - if cls._writer: - cls._writer.write(message) - cls._writer.flush() - else: - # Fallback to stderr - sys.stderr.write(message) - sys.stderr.flush() + with cls._state_lock: + cls._ensure_current_day() + if cls._writer: + cls._writer.write(message) + cls._writer.flush() + else: + # Fallback to stderr + sys.stderr.write(message) + sys.stderr.flush() + if error and cls._error_writer: + cls._error_writer.write(message) + cls._error_writer.flush() return len(message) except Exception: # Silently fail - logging should never break the app @@ -457,7 +397,7 @@ async def init( Args: print: Whether to print logs to stderr (if False, logs to file) - dev: Whether in development mode (affects filename) + dev: Kept for compatibility; file output always uses daily logs level: Log level (DEBUG, INFO, WARN, ERROR) """ cls._level = level @@ -469,53 +409,112 @@ async def init( # Cleanup old logs await cls._cleanup(log_dir) - if print: - # Print to stderr + with cls._state_lock: + if print: + # Print to stderr + if cls._writer: + cls._writer.close() + if cls._error_writer: + cls._error_writer.close() + cls._writer = None + cls._error_writer = None + cls._log_file = None + cls._log_dir_path = None + cls._log_date = None + return + + if cls._writer: + cls._writer.close() + if cls._error_writer: + cls._error_writer.close() + + cls._log_dir_path = log_dir cls._writer = None - return - - # Setup log file - if dev: - filename = "dev.log" - else: - # Format: YYYY-MM-DDTHHMMSS.log - filename = datetime.now().strftime("%Y-%m-%dT%H%M%S") + ".log" - - cls._log_file = log_dir / filename - - # Truncate if exists - if cls._log_file.exists(): - cls._log_file.write_text("") - - # Open for writing with size-based rotation for long-running sessions. - cls._writer = _RotatingTextWriter( - cls._log_file, - max_bytes=get_log_max_bytes(), - backup_count=get_log_backup_count(), - ) + cls._error_writer = None + cls._log_date = None + cls._open_daily_writers() # Create default logger cls.Default = cls.create(service="default") + + @classmethod + def _open_daily_writers(cls) -> None: + if cls._log_dir_path is None: + return + today = date.today().isoformat() + day_dir = cls._log_dir_path / today + day_dir.mkdir(parents=True, exist_ok=True) + cls._log_date = today + cls._log_file = day_dir / "flocks.log" + cls._writer = _AppendTextWriter(cls._log_file) + cls._error_writer = _AppendTextWriter(day_dir / "errors.log") + + @classmethod + def _ensure_current_day(cls) -> None: + if cls._writer is None or cls._log_dir_path is None: + return + today = date.today().isoformat() + if cls._log_date == today: + return + if cls._writer: + cls._writer.close() + if cls._error_writer: + cls._error_writer.close() + cls._writer = None + cls._error_writer = None + cls._open_daily_writers() + cls._cleanup_sync(cls._log_dir_path) @classmethod - async def _cleanup(cls, log_dir: Path) -> None: - """ - Clean up old log files, keeping only the 10 most recent + async def _cleanup(cls, log_dir: Path, retention_days: Optional[int] = None) -> None: + """Clean up date directories and legacy timestamp logs by age. Args: log_dir: Directory containing log files """ + cls._cleanup_sync(log_dir, retention_days=retention_days) + + @classmethod + def _cleanup_sync(cls, log_dir: Path, retention_days: Optional[int] = None) -> None: + """Clean up date directories and legacy timestamp logs by age.""" + days = get_log_retention_days() if retention_days is None else retention_days + if days <= 0: + return + cutoff = datetime.now() - timedelta(days=days) + + def _timestamp_from_name(path: Path) -> Optional[datetime]: + stem = path.name.split(".log", 1)[0] + try: + return datetime.strptime(stem, "%Y-%m-%dT%H%M%S") + except ValueError: + return None + + def _date_from_dir(path: Path) -> Optional[date]: + try: + return datetime.strptime(path.name, "%Y-%m-%d").date() + except ValueError: + return None + try: + for path in log_dir.iterdir(): + if not path.is_dir(): + continue + day = _date_from_dir(path) + if day is not None and day < cutoff.date(): + try: + shutil.rmtree(path) + except Exception: + pass + # Find base log files matching pattern YYYY-MM-DDTHHMMSS.log. # Rotated siblings are deleted together with their base file so # old ``.log.1``/``.log.2`` files do not leak forever. pattern = str(log_dir / "????-??-??T??????.log") files = [Path(path) for path in sorted(file_glob.glob(pattern))] - - # Keep only the 10 most recent - if len(files) > 10: - files_to_delete = files[:-10] - for path in files_to_delete: + + for path in files: + timestamp = _timestamp_from_name(path) + if timestamp is not None and timestamp < cutoff: try: path.unlink(missing_ok=True) for rotated in path.parent.glob(f"{path.name}.*"): @@ -523,12 +522,12 @@ async def _cleanup(cls, log_dir: Path) -> None: except Exception: pass # Silently ignore deletion errors - kept_files = set(files[-10:]) rotated_pattern = str(log_dir / "????-??-??T??????.log.*") for rotated_path in (Path(path) for path in file_glob.glob(rotated_pattern)): base_name = rotated_path.name.split(".log.", 1)[0] + ".log" base_path = rotated_path.with_name(base_name) - if base_path not in kept_files and not base_path.exists(): + timestamp = _timestamp_from_name(base_path) + if not base_path.exists() and timestamp is not None and timestamp < cutoff: try: rotated_path.unlink(missing_ok=True) except Exception: @@ -571,7 +570,7 @@ def file(cls) -> str: """Get the current log file path""" if cls._log_file: return str(cls._log_file) - return str(_log_dir() / "flocks.log") + return str(_log_dir() / date.today().isoformat() / "flocks.log") # Initialize Default logger on module import diff --git a/flocks/workflow/engine.py b/flocks/workflow/engine.py index e58281f3d..b47fb2303 100644 --- a/flocks/workflow/engine.py +++ b/flocks/workflow/engine.py @@ -713,7 +713,7 @@ def _execute_node( if node.type == "tool": return self._execute_tool_node(node, inputs, _runtime=_runtime) if node.type == "llm": - return self._execute_llm_node(node, inputs) + return self._execute_llm_node(node, inputs, _runtime=_runtime) if node.type == "http_request": return self._execute_http_request_node(node, inputs) if node.type == "subworkflow": @@ -759,7 +759,13 @@ def _execute_tool_node( output_k = node.output_key or "result" return {output_k: result}, "" - def _execute_llm_node(self, node: Node, inputs: Dict[str, Any]) -> Tuple[Dict[str, Any], str]: + def _execute_llm_node( + self, + node: Node, + inputs: Dict[str, Any], + *, + _runtime: Optional["Runtime"] = None, + ) -> Tuple[Dict[str, Any], str]: """Execute an LLM node: render Jinja2 prompt template, call LLM.""" assert node.prompt, "llm node requires prompt" try: @@ -771,9 +777,13 @@ def _execute_llm_node(self, node: Node, inputs: Dict[str, Any]) -> Tuple[Dict[st message=f"Prompt template render failed: {type(e).__name__}: {e}", ) from e from .llm import get_llm_client + _rt = _runtime or self.runtime + cancel_checker = getattr(_rt, "cancel_checker", None) try: - client = get_llm_client(model=node.model) + client = get_llm_client(model=node.model, cancel_checker=cancel_checker) text = client.ask(rendered) + except RunCancelledError: + raise except Exception as e: raise NodeExecutionError( node_id=node.id, diff --git a/flocks/workflow/llm.py b/flocks/workflow/llm.py index 66fe21400..850130a4b 100644 --- a/flocks/workflow/llm.py +++ b/flocks/workflow/llm.py @@ -2,14 +2,21 @@ from copy import copy from dataclasses import dataclass import time -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional from flocks.config.config import Config from flocks.provider.provider import ChatMessage, Provider, ProviderConfig -from flocks.workflow._async_runtime import run_sync as _run_sync_on_shared_loop +from flocks.workflow._async_runtime import ( + run_sync as _run_sync_on_shared_loop, + run_sync_cancellable as _run_sync_cancellable_on_shared_loop, +) +from flocks.workflow.errors import RunCancelledError -def _run_coro_sync(coro): +def _run_coro_sync( + coro, + cancel_checker: Optional[Callable[[], bool]] = None, +): """Run async provider calls from sync workflow code on the shared workflow loop. All workflow-triggered async work (config loading, provider.apply_config, @@ -17,6 +24,8 @@ def _run_coro_sync(coro): that loop-bound resources (httpx.AsyncClient, OpenAI streams) never outlive their owning loop. See ``flocks.workflow._async_runtime``. """ + if cancel_checker is not None: + return _run_sync_cancellable_on_shared_loop(coro, cancel_checker) return _run_sync_on_shared_loop(coro) @@ -47,6 +56,7 @@ def __init__( model: Optional[str] = None, *, provider_id: Optional[str] = None, + cancel_checker: Optional[Callable[[], bool]] = None, ): # NOTE: This module is intentionally synchronous at the edges (workflow runtime), # but uses flocks Provider (async) internally for consistency with the rest of flocks. @@ -57,6 +67,7 @@ def __init__( self.model = ((model or "") or "").strip() self.api_key = (api_key or "").strip() or None self.base_url = (base_url or "").strip() or None + self.cancel_checker = cancel_checker workflow_llm_cfg = self._load_workflow_llm_config() # Only treat ``trust_env`` as user-specified when it actually appears # in workflow.llm config. Otherwise we leave the provider's existing @@ -68,6 +79,30 @@ def __init__( Provider._ensure_initialized() + def _cancel_requested(self) -> bool: + try: + return bool(self.cancel_checker and self.cancel_checker()) + except Exception: + return False + + def _raise_if_cancelled(self) -> None: + if self._cancel_requested(): + raise RunCancelledError("") + + def _sleep_with_cancel(self, delay_s: float) -> None: + if delay_s <= 0: + return + if self.cancel_checker is None: + time.sleep(delay_s) + return + deadline = time.monotonic() + delay_s + while True: + self._raise_if_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + return + time.sleep(min(0.1, remaining)) + def _get_provider(self, provider_id: str) -> Any: provider = Provider.get(provider_id) if provider is None: @@ -79,7 +114,7 @@ def _get_provider(self, provider_id: str) -> Any: def _load_workflow_llm_config(self) -> Dict[str, Any]: try: - cfg = _run_coro_sync(Config.get()) + cfg = _run_coro_sync(Config.get(), self.cancel_checker) except Exception: return {} if not hasattr(cfg, "model_dump"): @@ -93,7 +128,10 @@ def _load_workflow_llm_config(self) -> Dict[str, Any]: def _resolve_default_target(self) -> Optional[_ResolvedTarget]: try: - default_llm = _run_coro_sync(Config.resolve_default_llm()) + default_llm = _run_coro_sync( + Config.resolve_default_llm(), + self.cancel_checker, + ) except Exception: return None if not default_llm: @@ -238,7 +276,10 @@ def _clone_provider_for_workflow(self, shared_provider: Any) -> Any: def _prepare_provider(self, provider_id: str) -> Any: """Build a workflow-local provider without mutating the shared singleton.""" try: - _run_coro_sync(Provider.apply_config(provider_id=provider_id)) + _run_coro_sync( + Provider.apply_config(provider_id=provider_id), + self.cancel_checker, + ) except Exception: # Keep workflow runtime resilient: provider apply_config failure # should not block ask() for environments driven by env vars. @@ -332,12 +373,14 @@ def ask( max_retries: int = 0, retry_delay_s: float = 1.0, ) -> str: + self._raise_if_cancelled() if model is not None or provider_id is not None: return LLMClient( api_key=self.api_key, base_url=self.base_url, model=model if model is not None else self.model, provider_id=provider_id if provider_id is not None else self.provider_id, + cancel_checker=self.cancel_checker, ).ask( prompt, temperature=temperature, @@ -353,6 +396,7 @@ def ask( delay_s = max(0.0, float(retry_delay_s)) for target in targets: + self._raise_if_cancelled() validation_error = self._validate_target(target) if validation_error: preflight_errors.append((target, validation_error)) @@ -372,11 +416,16 @@ async def _call(): last_exc: Optional[Exception] = None for attempt in range(retry_count + 1): + self._raise_if_cancelled() try: - response = _run_coro_sync(_call()) + response = _run_coro_sync(_call(), self.cancel_checker) self.provider_id = target.provider_id self.model = target.model_id return str(getattr(response, "content", "") or "") + except RunCancelledError: + raise + except asyncio.CancelledError as exc: + raise RunCancelledError("") from exc except asyncio.TimeoutError as exc: total_attempts = retry_count + 1 last_exc = TimeoutError( @@ -387,7 +436,7 @@ async def _call(): last_exc = exc if attempt < retry_count and delay_s > 0: - time.sleep(delay_s) + self._sleep_with_cancel(delay_s) if last_exc is not None: runtime_errors.append((target, last_exc)) @@ -405,18 +454,27 @@ def get_llm_client( base_url: Optional[str] = None, model: Optional[str] = None, provider_id: Optional[str] = None, + cancel_checker: Optional[Callable[[], bool]] = None, ) -> LLMClient: return LLMClient( api_key=api_key, base_url=base_url, model=model, provider_id=provider_id, + cancel_checker=cancel_checker, ) class LazyLLM: """Lazy facade for workflow `llm.ask(...)`.""" + def __init__( + self, + *, + cancel_checker: Optional[Callable[[], bool]] = None, + ): + self.cancel_checker = cancel_checker + def ask( self, prompt: str, @@ -428,7 +486,11 @@ def ask( max_retries: int = 0, retry_delay_s: float = 1.0, ) -> str: - return get_llm_client(model=model, provider_id=provider_id).ask( + return get_llm_client( + model=model, + provider_id=provider_id, + cancel_checker=self.cancel_checker, + ).ask( prompt, temperature=temperature, timeout_s=timeout_s, @@ -440,7 +502,12 @@ def ask( _lazy_llm_singleton: Optional[LazyLLM] = None -def get_lazy_llm() -> LazyLLM: +def get_lazy_llm( + *, + cancel_checker: Optional[Callable[[], bool]] = None, +) -> LazyLLM: + if cancel_checker is not None: + return LazyLLM(cancel_checker=cancel_checker) global _lazy_llm_singleton if _lazy_llm_singleton is None: _lazy_llm_singleton = LazyLLM() diff --git a/flocks/workflow/logging_config.py b/flocks/workflow/logging_config.py index cb0a6c83a..0db6f2eaa 100644 --- a/flocks/workflow/logging_config.py +++ b/flocks/workflow/logging_config.py @@ -1,16 +1,13 @@ """Logging configuration for flocks.workflow. -Workflow logs go to stderr and, when file logging is enabled, to -~/.flocks/logs/workflow.log (or FLOCKS_LOG_DIR/workflow.log). +Workflow logs go to stderr. File logging is intentionally disabled so the +Flocks log directory only contains backend.log, webui.log, and date folders. """ import logging import sys -from logging.handlers import RotatingFileHandler from typing import Optional -from flocks.utils.log import get_log_backup_count, get_log_dir, get_log_max_bytes - def setup_workflow_logging( level: int = logging.INFO, @@ -18,13 +15,13 @@ def setup_workflow_logging( stream=None, file: bool = True, ) -> None: - """配置 flocks.workflow 的日志输出（控制台 + 可选文件）。 + """配置 flocks.workflow 的日志输出（控制台）。 Args: level: 日志级别，默认为 INFO format_string: 日志格式字符串，如果为 None 则使用默认格式 stream: 输出流，默认为 sys.stderr - file: 是否同时写入 ~/.flocks/logs/workflow.log（与主 Log 同目录） + file: 兼容旧参数；当前不会创建 workflow.log 文件 Example: >>> from flocks.workflow import setup_workflow_logging @@ -49,22 +46,9 @@ def setup_workflow_logging( console_handler.setFormatter(formatter) logger.addHandler(console_handler) - # File (same directory as flocks.utils.log) - if file: - try: - log_dir = get_log_dir() - log_dir.mkdir(parents=True, exist_ok=True) - file_handler = RotatingFileHandler( - log_dir / "workflow.log", - maxBytes=get_log_max_bytes(), - backupCount=get_log_backup_count(), - encoding="utf-8", - ) - file_handler.setLevel(level) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - except OSError: - pass # Do not break if log dir is read-only or missing + # Workflow file logging is intentionally disabled. Structured workflow + # activity should be emitted through ``flocks.utils.log`` so the log + # directory only contains backend.log, webui.log, and date folders. logger.propagate = False diff --git a/flocks/workflow/models.py b/flocks/workflow/models.py index 3fa893126..e20f296e3 100644 --- a/flocks/workflow/models.py +++ b/flocks/workflow/models.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from .errors import WorkflowValidationError +from .triggers.models import TriggerDefinition, normalize_trigger_definitions class Node(BaseModel): @@ -96,12 +97,15 @@ class Workflow(BaseModel): start: str = Field(min_length=1) nodes: List[Node] = Field(default_factory=list) edges: List[Edge] = Field(default_factory=list) + triggers: List[TriggerDefinition] = Field(default_factory=list) metadata: Optional[Dict[str, Any]] = None @model_validator(mode="after") def _validate_graph(self) -> "Workflow": if self.version is not None: self.version = None + if not self.triggers and isinstance(self.metadata, dict) and isinstance(self.metadata.get("triggers"), list): + self.triggers = normalize_trigger_definitions(self.metadata.get("triggers")) node_ids = [n.id for n in self.nodes] if len(node_ids) != len(set(node_ids)): dupes = sorted({x for x in node_ids if node_ids.count(x) > 1}) diff --git a/flocks/workflow/poller_manager.py b/flocks/workflow/poller_manager.py index c25d4cd03..5238c9f20 100644 --- a/flocks/workflow/poller_manager.py +++ b/flocks/workflow/poller_manager.py @@ -10,9 +10,11 @@ import threading import time import uuid -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Dict +from croniter import croniter + from flocks.storage.storage import Storage from flocks.utils.log import Log from flocks.workflow.execution_store import ( @@ -60,10 +62,12 @@ def _normalize_config(self, workflow_id: str, data: Any) -> Dict[str, Any]: interval_seconds = int(raw.get("intervalSeconds") or DEFAULT_INTERVAL_SECONDS) timeout_seconds = int(raw.get("timeoutSeconds") or DEFAULT_TIMEOUT_SECONDS) inputs = raw.get("inputs") if isinstance(raw.get("inputs"), dict) else {} + cron_expression = str(raw.get("cronExpression") or "").strip() return { "workflowId": workflow_id, "enabled": bool(raw.get("enabled")), "intervalSeconds": max(1, interval_seconds), + "cronExpression": cron_expression or None, "timeoutSeconds": max(1, timeout_seconds), "noOverlap": bool(raw.get("noOverlap", True)), "inputs": dict(inputs), @@ -98,8 +102,19 @@ def _build_inputs(self, config: Dict[str, Any]) -> Dict[str, Any]: inputs = dict(config.get("inputs") or {}) if not str(inputs.get("input_date") or "").strip(): inputs["input_date"] = _today_string() + run_id = f"poller-{_now_ms()}-{uuid.uuid4().hex[:8]}" inputs["_trigger"] = "poller" - inputs["_poller_run_id"] = f"poller-{_now_ms()}-{uuid.uuid4().hex[:8]}" + inputs["_poller_run_id"] = run_id + inputs["_flocks"] = { + "trigger": { + "id": "schedule-default", + "type": "schedule", + "source": "poller", + "deliveryId": run_id, + "receivedAt": _now_ms(), + "attempt": 1, + } + } return inputs def _summarize_outputs(self, outputs: Any) -> Dict[str, Any]: @@ -141,8 +156,19 @@ def _base_status(self, workflow_id: str) -> Dict[str, Any]: "kafkaMessageCount": None, "nextRunAt": None, "lastRunId": None, + "cronExpression": None, } + def _compute_next_run_at_ms(self, config: Dict[str, Any], *, base_ts_s: float | None = None) -> int: + cron_expression = str(config.get("cronExpression") or "").strip() + if cron_expression: + base = datetime.fromtimestamp( + base_ts_s if base_ts_s is not None else time.time(), + tz=timezone.utc, + ) + return int(croniter(cron_expression, base).get_next(float) * 1000) + return _now_ms() + int(config["intervalSeconds"]) * 1000 + def get_status(self, workflow_id: str) -> Dict[str, Any]: status = dict(self._base_status(workflow_id)) status.update(self._status.get(workflow_id) or {}) @@ -258,9 +284,10 @@ async def restart_workflow(self, workflow_id: str) -> Dict[str, Any]: "error": None, "enabled": True, "intervalSeconds": config["intervalSeconds"], + "cronExpression": config.get("cronExpression"), "timeoutSeconds": config["timeoutSeconds"], "noOverlap": config["noOverlap"], - "nextRunAt": _now_ms(), + "nextRunAt": self._compute_next_run_at_ms(config), } task = asyncio.create_task( self._poller_loop(workflow_id, workflow_json, config, abort_event), @@ -307,17 +334,31 @@ async def _poller_loop( config: Dict[str, Any], abort_event: asyncio.Event, ) -> None: - interval_seconds = config["intervalSeconds"] + cron_expression = str(config.get("cronExpression") or "").strip() try: while not abort_event.is_set(): - await self._schedule_run(workflow_id, workflow_json, config) - next_run_at = _now_ms() + interval_seconds * 1000 current = self._status.get(workflow_id) or self._base_status(workflow_id) + if cron_expression: + next_run_at = self._compute_next_run_at_ms(config) + wait_seconds = max(0.0, (next_run_at - _now_ms()) / 1000.0) + current["nextRunAt"] = next_run_at + current["activeRuns"] = self._cleanup_done_runs(workflow_id) + self._status[workflow_id] = current + try: + await asyncio.wait_for(abort_event.wait(), timeout=wait_seconds) + continue + except asyncio.TimeoutError: + pass + await self._schedule_run(workflow_id, workflow_json, config) + continue + + await self._schedule_run(workflow_id, workflow_json, config) + next_run_at = self._compute_next_run_at_ms(config) current["nextRunAt"] = next_run_at current["activeRuns"] = self._cleanup_done_runs(workflow_id) self._status[workflow_id] = current try: - await asyncio.wait_for(abort_event.wait(), timeout=interval_seconds) + await asyncio.wait_for(abort_event.wait(), timeout=config["intervalSeconds"]) except asyncio.TimeoutError: continue except asyncio.CancelledError: @@ -407,6 +448,11 @@ async def _execute_run( "currentNodeId": result.last_node_id, "currentPhase": status_value, "currentStepIndex": result.steps, + "triggerId": "schedule-default", + "triggerType": "schedule", + "deliveryId": inputs.get("_flocks", {}).get("trigger", {}).get("deliveryId"), + "attempt": 1, + "triggerSource": "poller", }) current = self._status.get(workflow_id) or self._base_status(workflow_id) current.update(summary) @@ -432,6 +478,11 @@ async def _execute_run( "errorMessage": str(exc), "executionLog": compact_history_for_storage(exec_data.get("executionLog")), "currentPhase": status_value, + "triggerId": "schedule-default", + "triggerType": "schedule", + "deliveryId": inputs.get("_flocks", {}).get("trigger", {}).get("deliveryId"), + "attempt": 1, + "triggerSource": "poller", }) current = self._status.get(workflow_id) or self._base_status(workflow_id) current["lastRunAt"] = started_at_ms diff --git a/flocks/workflow/repl_runtime.py b/flocks/workflow/repl_runtime.py index 5e43c8c2f..2f010fb75 100644 --- a/flocks/workflow/repl_runtime.py +++ b/flocks/workflow/repl_runtime.py @@ -89,7 +89,7 @@ def _cancel_requested() -> bool: # a line tracer so simple Python loops can be interrupted by UI stop. g["cancelled"] = _cancel_requested g["is_cancelled"] = _cancel_requested - g.setdefault("llm", get_lazy_llm()) + g["llm"] = get_lazy_llm(cancel_checker=self.cancel_checker) reg = self.tool_registry or get_tool_registry() if hasattr(reg, "cancel_checker"): try: @@ -219,6 +219,7 @@ class SandboxPythonExecRuntime(Runtime): sandbox: Dict[str, Any] tool_registry: Optional[Any] = None + cancel_checker: Optional[Callable[[], bool]] = None def execute(self, code: str, inputs: Dict[str, Any]) -> Tuple[Dict[str, Any], str]: if not isinstance(code, str): @@ -596,7 +597,7 @@ def _handle_rpc_request(self, *, msg: Dict[str, Any], token: str) -> Dict[str, A retry_delay_s = float(retry_delay_raw) except Exception as exc: raise RuntimeError("LLM retry_delay_s must be a number when provided") from exc - output = get_lazy_llm().ask( + output = get_lazy_llm(cancel_checker=self.cancel_checker).ask( prompt, temperature=temperature, model=model, diff --git a/flocks/workflow/runner.py b/flocks/workflow/runner.py index abed73d2f..3d9dcc650 100644 --- a/flocks/workflow/runner.py +++ b/flocks/workflow/runner.py @@ -302,41 +302,41 @@ def run_workflow( # 确保日志已配置 _ensure_logging_configured() - _logger.info("=== 开始执行 workflow ===") + _logger.debug("=== 开始执行 workflow ===") workflow_path_for_engine: Optional[str] = None effective_use_llm: Optional[bool] = use_llm if isinstance(workflow, Workflow): - _logger.info("workflow 来源: Workflow 对象") + _logger.debug("workflow 来源: Workflow 对象") wf = workflow elif isinstance(workflow, (str, Path)): workflow_path = Path(workflow).expanduser() - _logger.info(f"workflow 来源: 文件路径 {workflow_path}") + _logger.debug("workflow 来源: 文件路径 %s", workflow_path) workflow_path_for_engine = str(workflow_path.resolve()) if workflow_path.exists() else None # Prefer compiled exec workflow if it exists (and is up-to-date). Otherwise compile first. exec_path = default_exec_path(workflow_path) - _logger.info(f"检查编译缓存: {exec_path}") + _logger.debug("检查编译缓存: %s", exec_path) wf = load_workflow(workflow_path) if effective_use_llm is None: # Auto-enable LLM codegen when the workflow contains logic nodes. # This keeps pure-python workflows offline-friendly while ensuring # logic nodes can be materialized into runnable Python when needed. effective_use_llm = workflow_has_logic_nodes(wf) - _logger.info(f"自动检测 use_llm={effective_use_llm} (基于是否包含 logic 节点)") + _logger.debug("自动检测 use_llm=%s (基于是否包含 logic 节点)", effective_use_llm) if workflow_path.exists(): exec_is_fresh = False if exec_path.exists(): try: exec_is_fresh = exec_path.stat().st_mtime >= workflow_path.stat().st_mtime - _logger.info(f"编译缓存状态: {'最新' if exec_is_fresh else '过期'}") + _logger.debug("编译缓存状态: %s", "最新" if exec_is_fresh else "过期") except Exception: exec_is_fresh = False _logger.warning("无法检查编译缓存状态") if exec_is_fresh: try: - _logger.info("使用编译缓存") + _logger.debug("使用编译缓存") wf = load_workflow(exec_path) except Exception: # If exec is unreadable, fall back to source then recompile below if needed. @@ -346,7 +346,7 @@ def run_workflow( # Only compile when the source contains logic nodes. Pure-python workflows don't need exec. # If exec exists but is stale/unreadable, recompile and overwrite it. if workflow_has_logic_nodes(wf) and (not exec_is_fresh): - _logger.info("开始编译 workflow (包含 logic 节点)") + _logger.debug("开始编译 workflow (包含 logic 节点)") compiled = compile_workflow( wf, use_llm=bool(effective_use_llm), @@ -354,17 +354,17 @@ def run_workflow( preserve_description=True, workflow_path=str(workflow_path), ) - _logger.info(f"编译完成，保存到 {exec_path}") + _logger.debug("编译完成，保存到 %s", exec_path) dump_workflow(compiled, exec_path, indent=2) wf = compiled else: - _logger.info("workflow 来源: 字典") + _logger.debug("workflow 来源: 字典") wf = Workflow.from_dict(workflow) if effective_use_llm is None: effective_use_llm = workflow_has_logic_nodes(wf) - _logger.info(f"workflow 信息: nodes={len(wf.nodes)}, edges={len(wf.edges)}, start={wf.start}") + _logger.debug("workflow 信息: nodes=%s, edges=%s, start=%s", len(wf.nodes), len(wf.edges), wf.start) try: lint_results = lint_workflow(wf) @@ -384,11 +384,11 @@ def run_workflow( reqs = requirements_from_workflow_metadata(wf.metadata) if ensure_requirements: - _logger.info("检查依赖包...") + _logger.debug("检查依赖包...") if reqs: - _logger.info(f"需要安装的依赖: {reqs}") + _logger.debug("需要安装的依赖: %s", reqs) - _logger.info("初始化工具注册表和运行时环境...") + _logger.debug("初始化工具注册表和运行时环境...") registry = tool_registry or get_tool_registry(tool_context=tool_context) sandbox_payload = _extract_sandbox_runtime_payload(tool_context) runtime_preference = _resolve_workflow_runtime_preference(tool_context) @@ -403,7 +403,7 @@ def run_workflow( ) if sandbox_payload: - _logger.info("workflow runtime: sandbox python execution enabled") + _logger.debug("workflow runtime: sandbox python execution enabled") if ensure_requirements and reqs: (sandbox_requirements_installer or SandboxRequirementsInstaller(installer="auto")).ensure_installed( reqs, @@ -412,7 +412,7 @@ def run_workflow( rt = SandboxPythonExecRuntime(sandbox=sandbox_payload, tool_registry=registry) else: if runtime_preference == "host": - _logger.info("workflow runtime: host forced by sandbox.mode=off or runtime override") + _logger.debug("workflow runtime: host forced by sandbox.mode=off or runtime override") if ensure_requirements and reqs: (requirements_installer or RequirementsInstaller(installer="auto")).ensure_installed(reqs) rt = PythonExecRuntime( @@ -420,7 +420,7 @@ def run_workflow( cleanup_globals_after_execute=(history_mode == "summary"), ) - _logger.info( + _logger.debug( "创建执行引擎 (use_llm=%s, trace=%s, node_timeout=%ss, parallel_workers=%s, history_mode=%s)", effective_use_llm, trace, @@ -440,7 +440,7 @@ def run_workflow( ) initial_inputs = _build_initial_inputs(inputs, workflow_path_for_engine) - _logger.info( + _logger.debug( "开始执行 workflow (timeout=%ss, inputs=%s)", timeout_s, list(initial_inputs.keys()), diff --git a/flocks/workflow/triggers/__init__.py b/flocks/workflow/triggers/__init__.py new file mode 100644 index 000000000..048f15dda --- /dev/null +++ b/flocks/workflow/triggers/__init__.py @@ -0,0 +1,38 @@ +"""Workflow trigger runtime package.""" + +from .dispatcher import EventDispatcher, TriggerDispatchError, build_trigger_event, preview_trigger_mapping +from .models import ( + TriggerAuth, + TriggerConcurrency, + TriggerDefinition, + TriggerEvent, + TriggerEventSource, + TriggerFilter, + TriggerRuntimeStatus, + default_trigger_id, + normalize_trigger_definitions, + set_workflow_json_triggers, + trigger_definitions_to_json, + workflow_json_declares_triggers, + workflow_trigger_definitions_from_json, +) + +__all__ = [ + "EventDispatcher", + "TriggerAuth", + "TriggerConcurrency", + "TriggerDefinition", + "TriggerDispatchError", + "TriggerEvent", + "TriggerEventSource", + "TriggerFilter", + "TriggerRuntimeStatus", + "build_trigger_event", + "default_trigger_id", + "normalize_trigger_definitions", + "preview_trigger_mapping", + "set_workflow_json_triggers", + "trigger_definitions_to_json", + "workflow_json_declares_triggers", + "workflow_trigger_definitions_from_json", +] diff --git a/flocks/workflow/triggers/compat.py b/flocks/workflow/triggers/compat.py new file mode 100644 index 000000000..faf5f5cf4 --- /dev/null +++ b/flocks/workflow/triggers/compat.py @@ -0,0 +1,139 @@ +"""Compatibility helpers between unified triggers and legacy config storage.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from .models import TriggerDefinition + +LEGACY_POLLER_CONFIG_PREFIX = "workflow_poller_config/" +LEGACY_SYSLOG_CONFIG_PREFIX = "workflow_syslog_config/" +LEGACY_KAFKA_CONFIG_PREFIX = "workflow_kafka_config/" + + +def legacy_schedule_trigger_from_config(config: Optional[Dict[str, Any]]) -> Optional[TriggerDefinition]: + if not isinstance(config, dict): + return None + cron_expression = str(config.get("cronExpression") or "").strip() + return TriggerDefinition.model_validate( + { + "id": "schedule-default", + "type": "schedule", + "enabled": bool(config.get("enabled")), + "source": { + "mode": "cron" if cron_expression else "interval", + "intervalSeconds": int(config.get("intervalSeconds") or 30), + "cron": cron_expression or None, + }, + "runtime": { + "timeoutSeconds": int(config.get("timeoutSeconds") or 7200), + "noOverlap": bool(config.get("noOverlap", True)), + }, + "inputs": dict(config.get("inputs") or {}), + "updatedAt": config.get("updatedAt"), + } + ) + + +def legacy_syslog_trigger_from_config(config: Optional[Dict[str, Any]]) -> Optional[TriggerDefinition]: + if not isinstance(config, dict): + return None + return TriggerDefinition.model_validate( + { + "id": "syslog-default", + "type": "syslog", + "enabled": bool(config.get("enabled")), + "source": { + "protocol": config.get("protocol") or "udp", + "host": config.get("host") or "0.0.0.0", + "port": int(config.get("port") or 5140), + "format": config.get("format") or "auto", + }, + "mapping": { + str(config.get("inputKey") or "syslog_message"): "$.body", + }, + "updatedAt": config.get("updatedAt"), + } + ) + + +def legacy_kafka_trigger_from_config(config: Optional[Dict[str, Any]]) -> Optional[TriggerDefinition]: + if not isinstance(config, dict): + return None + return TriggerDefinition.model_validate( + { + "id": "kafka-default", + "type": "kafka", + "enabled": bool(config.get("enabled")), + "source": { + "inputBroker": config.get("inputBroker") or "", + "inputTopic": config.get("inputTopic") or "", + "inputGroupId": config.get("inputGroupId") or "", + "autoOffsetReset": config.get("autoOffsetReset") or "latest", + }, + "mapping": { + str(config.get("inputKey") or "kafka_message"): "$.body", + }, + "inputs": dict(config.get("inputs") or {}), + "updatedAt": config.get("updatedAt"), + } + ) + + +def schedule_trigger_to_legacy_config(workflow_id: str, trigger: TriggerDefinition) -> Dict[str, Any]: + source = dict(trigger.source or {}) + runtime = dict(trigger.runtime or {}) + cron_expression = str(source.get("cron") or source.get("cronExpression") or "").strip() + return { + "workflowId": workflow_id, + "enabled": trigger.enabled, + "intervalSeconds": int(source.get("intervalSeconds") or 30), + "cronExpression": cron_expression or None, + "timeoutSeconds": int(runtime.get("timeoutSeconds") or 7200), + "noOverlap": bool(runtime.get("noOverlap", True)), + "inputs": dict(trigger.inputs or {}), + "updatedAt": trigger.updatedAt, + } + + +def syslog_trigger_to_legacy_config(workflow_id: str, trigger: TriggerDefinition) -> Dict[str, Any]: + source = dict(trigger.source or {}) + mapping = dict(trigger.mapping or {}) + input_key = next(iter(mapping.keys()), "syslog_message") + return { + "workflowId": workflow_id, + "enabled": trigger.enabled, + "protocol": source.get("protocol") or "udp", + "host": source.get("host") or "0.0.0.0", + "port": int(source.get("port") or 5140), + "format": source.get("format") or "auto", + "inputKey": input_key, + "updatedAt": trigger.updatedAt, + } + + +def kafka_trigger_to_legacy_config(workflow_id: str, trigger: TriggerDefinition) -> Dict[str, Any]: + source = dict(trigger.source or {}) + mapping = dict(trigger.mapping or {}) + input_key = next(iter(mapping.keys()), "kafka_message") + return { + "workflowId": workflow_id, + "enabled": trigger.enabled, + "inputBroker": source.get("inputBroker") or "", + "inputTopic": source.get("inputTopic") or "", + "inputGroupId": source.get("inputGroupId") or "", + "inputKey": input_key, + "autoOffsetReset": source.get("autoOffsetReset") or "latest", + "inputs": dict(trigger.inputs or {}), + "updatedAt": trigger.updatedAt, + } + + +def trigger_to_legacy_config(workflow_id: str, trigger: TriggerDefinition) -> tuple[Optional[str], Optional[Dict[str, Any]]]: + if trigger.type == "schedule": + return f"{LEGACY_POLLER_CONFIG_PREFIX}{workflow_id}", schedule_trigger_to_legacy_config(workflow_id, trigger) + if trigger.type == "syslog": + return f"{LEGACY_SYSLOG_CONFIG_PREFIX}{workflow_id}", syslog_trigger_to_legacy_config(workflow_id, trigger) + if trigger.type == "kafka": + return f"{LEGACY_KAFKA_CONFIG_PREFIX}{workflow_id}", kafka_trigger_to_legacy_config(workflow_id, trigger) + return None, None diff --git a/flocks/workflow/triggers/custom_loader.py b/flocks/workflow/triggers/custom_loader.py new file mode 100644 index 000000000..f281f2a55 --- /dev/null +++ b/flocks/workflow/triggers/custom_loader.py @@ -0,0 +1,77 @@ +"""Loader for user-defined trigger plugin specs.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType +from typing import Any, Dict, List, Optional + +from flocks.workflow.fs_store import find_workspace_root + +try: # pragma: no cover - optional dependency fallback + import yaml +except Exception: # pragma: no cover - fallback branch + yaml = None + +PLUGIN_FILENAMES = ("trigger.json", "trigger.yaml", "trigger.yml", "manifest.json") + + +def trigger_plugin_roots() -> List[Path]: + workspace = find_workspace_root() + return [ + Path.home() / ".flocks" / "plugins" / "triggers", + workspace / ".flocks" / "plugins" / "triggers", + ] + + +def _read_plugin_manifest(path: Path) -> Optional[Dict[str, Any]]: + try: + if path.suffix.lower() == ".json": + return json.loads(path.read_text(encoding="utf-8")) + if yaml is None: + return None + return yaml.safe_load(path.read_text(encoding="utf-8")) + except Exception: + return None + + +def list_trigger_plugins() -> List[Dict[str, Any]]: + plugins: Dict[str, Dict[str, Any]] = {} + for root in trigger_plugin_roots(): + if not root.is_dir(): + continue + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + manifest_path = next((entry / filename for filename in PLUGIN_FILENAMES if (entry / filename).is_file()), None) + if manifest_path is None: + continue + manifest = _read_plugin_manifest(manifest_path) + if not isinstance(manifest, dict): + continue + plugin_id = str(manifest.get("id") or entry.name).strip() or entry.name + plugins[plugin_id] = { + "id": plugin_id, + "name": manifest.get("name") or plugin_id, + "description": manifest.get("description"), + "root": str(entry), + "manifestPath": str(manifest_path), + "handlerPath": str(entry / "handler.py"), + "manifest": manifest, + } + return list(plugins.values()) + + +def load_trigger_plugin_module(plugin_spec: Dict[str, Any]) -> Optional[ModuleType]: + handler_path = Path(str(plugin_spec.get("handlerPath") or "")).expanduser() + if not handler_path.is_file(): + return None + module_name = f"flocks_trigger_plugin_{plugin_spec.get('id', handler_path.stem)}" + spec = importlib.util.spec_from_file_location(module_name, handler_path) + if spec is None or spec.loader is None: + return None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/flocks/workflow/triggers/dispatcher.py b/flocks/workflow/triggers/dispatcher.py new file mode 100644 index 000000000..b617209ba --- /dev/null +++ b/flocks/workflow/triggers/dispatcher.py @@ -0,0 +1,267 @@ +"""Unified trigger event mapping, filtering, and dispatch helpers.""" + +from __future__ import annotations + +import ast +import time +import uuid +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple + +from .models import TriggerDefinition, TriggerEvent, TriggerEventSource + +DispatchExecutor = Callable[[Dict[str, Any]], Awaitable[Any]] + + +class TriggerDispatchError(Exception): + """Raised when a trigger event cannot be dispatched.""" + + +class TriggerExpressionEvaluator(ast.NodeVisitor): + """Very small safe evaluator for trigger filter expressions.""" + + def __init__(self, variables: Dict[str, Any]) -> None: + self._variables = variables + + def visit_Expression(self, node: ast.Expression) -> Any: # noqa: N802 + return self.visit(node.body) + + def visit_Constant(self, node: ast.Constant) -> Any: # noqa: N802 + return node.value + + def visit_Name(self, node: ast.Name) -> Any: # noqa: N802 + if node.id not in self._variables: + raise TriggerDispatchError(f"Unknown name in trigger filter: {node.id}") + return self._variables[node.id] + + def visit_List(self, node: ast.List) -> Any: # noqa: N802 + return [self.visit(elt) for elt in node.elts] + + def visit_Tuple(self, node: ast.Tuple) -> Any: # noqa: N802 + return tuple(self.visit(elt) for elt in node.elts) + + def visit_Dict(self, node: ast.Dict) -> Any: # noqa: N802 + return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)} + + def visit_BoolOp(self, node: ast.BoolOp) -> Any: # noqa: N802 + if isinstance(node.op, ast.And): + return all(self.visit(value) for value in node.values) + if isinstance(node.op, ast.Or): + return any(self.visit(value) for value in node.values) + raise TriggerDispatchError("Unsupported boolean operator in trigger filter") + + def visit_UnaryOp(self, node: ast.UnaryOp) -> Any: # noqa: N802 + operand = self.visit(node.operand) + if isinstance(node.op, ast.Not): + return not operand + raise TriggerDispatchError("Unsupported unary operator in trigger filter") + + def visit_Compare(self, node: ast.Compare) -> Any: # noqa: N802 + left = self.visit(node.left) + for operator, comparator_node in zip(node.ops, node.comparators): + right = self.visit(comparator_node) + if isinstance(operator, ast.Eq): + ok = left == right + elif isinstance(operator, ast.NotEq): + ok = left != right + elif isinstance(operator, ast.In): + ok = left in right + elif isinstance(operator, ast.NotIn): + ok = left not in right + elif isinstance(operator, ast.Gt): + ok = left > right + elif isinstance(operator, ast.GtE): + ok = left >= right + elif isinstance(operator, ast.Lt): + ok = left < right + elif isinstance(operator, ast.LtE): + ok = left <= right + else: + raise TriggerDispatchError("Unsupported compare operator in trigger filter") + if not ok: + return False + left = right + return True + + def visit_Attribute(self, node: ast.Attribute) -> Any: # noqa: N802 + value = self.visit(node.value) + if isinstance(value, dict): + return value.get(node.attr) + return getattr(value, node.attr, None) + + def visit_Subscript(self, node: ast.Subscript) -> Any: # noqa: N802 + value = self.visit(node.value) + key = self.visit(node.slice) + try: + return value[key] + except Exception as exc: # pragma: no cover - defensive branch + raise TriggerDispatchError(f"Invalid trigger filter subscript access: {exc}") from exc + + def generic_visit(self, node: ast.AST) -> Any: # noqa: D401 + raise TriggerDispatchError(f"Unsupported syntax in trigger filter: {type(node).__name__}") + + +def _tokenize_path(path: str) -> List[Any]: + tokens: List[Any] = [] + i = 0 + while i < len(path): + ch = path[i] + if ch == ".": + i += 1 + continue + if ch == "[": + end = path.find("]", i) + if end < 0: + raise TriggerDispatchError(f"Invalid mapping path: {path}") + raw = path[i + 1 : end].strip() + if raw.isdigit(): + tokens.append(int(raw)) + else: + tokens.append(raw.strip("'\"")) + i = end + 1 + continue + start = i + while i < len(path) and path[i] not in ".[": + i += 1 + tokens.append(path[start:i]) + return [token for token in tokens if token not in ("$", "")] + + +def lookup_mapping_path(data: Any, path: str) -> Any: + raw = (path or "").strip() + if raw in {"", "$"}: + return data + candidate = raw[2:] if raw.startswith("$.") else raw + value = data + for token in _tokenize_path(candidate): + if isinstance(token, int): + if not isinstance(value, list): + return None + if token < 0 or token >= len(value): + return None + value = value[token] + continue + if isinstance(value, dict): + value = value.get(token) + else: + value = getattr(value, token, None) + if value is None: + return None + return value + + +def build_trigger_event( + *, + workflow_id: str, + trigger: TriggerDefinition, + body: Any = None, + headers: Optional[Dict[str, Any]] = None, + query: Optional[Dict[str, Any]] = None, + path_params: Optional[Dict[str, Any]] = None, + source: Optional[str] = None, + raw: Any = None, + delivery_id: Optional[str] = None, +) -> TriggerEvent: + resolved_source = source + if not resolved_source: + src = trigger.source or {} + if isinstance(src, dict): + resolved_source = ( + src.get("path") + or src.get("topic") + or src.get("event") + or src.get("adapterId") + or trigger.type + ) + return TriggerEvent( + source=TriggerEventSource( + workflowId=workflow_id, + triggerId=trigger.id or "", + triggerType=trigger.type, + source=str(resolved_source or trigger.type), + deliveryId=delivery_id or uuid.uuid4().hex, + receivedAt=int(time.time() * 1000), + ), + body=body, + headers=headers or {}, + query=query or {}, + pathParams=path_params or {}, + payload=body, + raw=raw if raw is not None else body, + ) + + +def event_to_context(event: TriggerEvent) -> Dict[str, Any]: + payload = event.model_dump(mode="json", exclude_none=True) + return { + "event": payload, + "body": payload.get("body"), + "headers": payload.get("headers") or {}, + "query": payload.get("query") or {}, + "pathParams": payload.get("pathParams") or {}, + "payload": payload.get("payload"), + "raw": payload.get("raw"), + } + + +def evaluate_trigger_filter(trigger: TriggerDefinition, event: TriggerEvent) -> Tuple[bool, Optional[str]]: + filter_spec = trigger.filter + if filter_spec is None: + return True, None + expr = (filter_spec.expr or "").strip() + if not expr: + return True, None + ctx = event_to_context(event) + try: + parsed = ast.parse(expr, mode="eval") + matched = bool(TriggerExpressionEvaluator(ctx).visit(parsed)) + except Exception as exc: + return False, str(exc) + return matched, None + + +def preview_trigger_mapping(trigger: TriggerDefinition, event: TriggerEvent) -> Dict[str, Any]: + ctx = event_to_context(event) + mapped: Dict[str, Any] = dict(trigger.inputs or {}) + for dst_key, src_path in (trigger.mapping or {}).items(): + mapped[dst_key] = lookup_mapping_path(ctx, src_path) + mapped["_flocks"] = { + "trigger": { + "id": event.source.triggerId, + "type": event.source.triggerType, + "source": event.source.source, + "deliveryId": event.source.deliveryId, + "receivedAt": event.source.receivedAt, + "attempt": event.source.attempt, + } + } + mapped.setdefault("_trigger", trigger.type) + return mapped + + +class EventDispatcher: + """Dispatch trigger events through filtering and mapping.""" + + async def dispatch( + self, + *, + trigger: TriggerDefinition, + event: TriggerEvent, + executor: DispatchExecutor, + ) -> Dict[str, Any]: + matched, filter_error = evaluate_trigger_filter(trigger, event) + mapped_inputs = preview_trigger_mapping(trigger, event) + if filter_error: + raise TriggerDispatchError(filter_error) + if not matched: + return { + "matched": False, + "inputs": mapped_inputs, + "executed": False, + } + result = await executor(mapped_inputs) + return { + "matched": True, + "inputs": mapped_inputs, + "executed": True, + "result": result, + } diff --git a/flocks/workflow/triggers/models.py b/flocks/workflow/triggers/models.py new file mode 100644 index 000000000..f030faf93 --- /dev/null +++ b/flocks/workflow/triggers/models.py @@ -0,0 +1,209 @@ +"""Workflow trigger schema models and compatibility helpers.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Iterable, List, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +TriggerType = Literal[ + "manual", + "schedule", + "webhook", + "syslog", + "kafka", + "internal_event", + "custom_webhook", + "custom_adapter", + "plugin", +] + +_TRIGGER_ID_SANITIZE_RE = re.compile(r"[^a-zA-Z0-9_.-]+") + + +def _sanitize_trigger_id(value: str) -> str: + cleaned = _TRIGGER_ID_SANITIZE_RE.sub("-", (value or "").strip()).strip("-") + return cleaned or "trigger" + + +def default_trigger_id(trigger_type: str, *, source: Optional[Dict[str, Any]] = None) -> str: + base = (trigger_type or "trigger").strip().lower() or "trigger" + src = source or {} + for candidate_key in ("path", "topic", "event", "name", "adapterId", "pluginId"): + candidate = src.get(candidate_key) + if isinstance(candidate, str) and candidate.strip(): + return f"{base}-{_sanitize_trigger_id(candidate)}" + return f"{base}-default" + + +class TriggerAuth(BaseModel): + model_config = ConfigDict(extra="allow") + + type: str = "none" + secretRef: Optional[str] = None + headerName: Optional[str] = None + queryParam: Optional[str] = None + apiKey: Optional[str] = None + + +class TriggerFilter(BaseModel): + model_config = ConfigDict(extra="allow") + + expr: Optional[str] = None + mode: Optional[str] = None + path: Optional[str] = None + equals: Optional[Any] = None + + +class TriggerConcurrency(BaseModel): + model_config = ConfigDict(extra="allow") + + policy: Literal["allow", "no_overlap", "queue", "drop_oldest", "drop_newest"] = "allow" + maxParallel: int = Field(1, ge=1) + queueSize: int = Field(100, ge=1) + + +class TriggerTestSample(BaseModel): + model_config = ConfigDict(extra="allow") + + name: str = Field(min_length=1) + payload: Any = None + headers: Dict[str, Any] = Field(default_factory=dict) + query: Dict[str, Any] = Field(default_factory=dict) + + +class TriggerDefinition(BaseModel): + model_config = ConfigDict(extra="allow") + + id: Optional[str] = None + name: Optional[str] = None + type: TriggerType + enabled: bool = True + description: Optional[str] = None + source: Dict[str, Any] = Field(default_factory=dict) + auth: Optional[TriggerAuth] = None + filter: Optional[TriggerFilter] = None + mapping: Dict[str, str] = Field(default_factory=dict) + inputs: Dict[str, Any] = Field(default_factory=dict) + concurrency: TriggerConcurrency = Field(default_factory=TriggerConcurrency) + runtime: Dict[str, Any] = Field(default_factory=dict) + testSamples: List[TriggerTestSample] = Field(default_factory=list) + updatedAt: Optional[int] = None + + @model_validator(mode="before") + @classmethod + def _normalize_nested_values(cls, value: Any) -> Any: + if not isinstance(value, dict): + return value + normalized = dict(value) + auth = normalized.get("auth") + if isinstance(auth, dict): + normalized["auth"] = TriggerAuth.model_validate(auth) + filter_value = normalized.get("filter") + if isinstance(filter_value, dict): + normalized["filter"] = TriggerFilter.model_validate(filter_value) + concurrency = normalized.get("concurrency") + if isinstance(concurrency, dict): + normalized["concurrency"] = TriggerConcurrency.model_validate(concurrency) + samples = normalized.get("testSamples") + if isinstance(samples, list): + normalized["testSamples"] = [ + TriggerTestSample.model_validate(item) if not isinstance(item, TriggerTestSample) else item + for item in samples + if isinstance(item, (dict, TriggerTestSample)) + ] + return normalized + + @model_validator(mode="after") + def _ensure_id(self) -> "TriggerDefinition": + source = self.source if isinstance(self.source, dict) else {} + self.id = _sanitize_trigger_id(self.id or default_trigger_id(self.type, source=source)) + return self + + +class TriggerEventSource(BaseModel): + model_config = ConfigDict(extra="allow") + + workflowId: str + triggerId: str + triggerType: str + source: Optional[str] = None + deliveryId: Optional[str] = None + receivedAt: Optional[int] = None + attempt: int = 1 + + +class TriggerEvent(BaseModel): + model_config = ConfigDict(extra="allow") + + source: TriggerEventSource + body: Any = None + headers: Dict[str, Any] = Field(default_factory=dict) + query: Dict[str, Any] = Field(default_factory=dict) + pathParams: Dict[str, Any] = Field(default_factory=dict) + payload: Any = None + raw: Any = None + + +class TriggerRuntimeStatus(BaseModel): + model_config = ConfigDict(extra="allow") + + workflowId: str + triggerId: str + triggerType: str + state: str + error: Optional[str] = None + + +def normalize_trigger_definitions(raw_triggers: Optional[Iterable[Any]]) -> List[TriggerDefinition]: + if not raw_triggers: + return [] + deduped: Dict[str, TriggerDefinition] = {} + for raw in raw_triggers: + if raw is None: + continue + trigger = raw if isinstance(raw, TriggerDefinition) else TriggerDefinition.model_validate(raw) + deduped[trigger.id or default_trigger_id(trigger.type)] = trigger + return list(deduped.values()) + + +def workflow_trigger_definitions_from_json(workflow_json: Dict[str, Any]) -> List[TriggerDefinition]: + raw = workflow_json.get("triggers") + if raw is None: + metadata = workflow_json.get("metadata") + if isinstance(metadata, dict): + raw = metadata.get("triggers") + if not isinstance(raw, list): + return [] + return normalize_trigger_definitions(raw) + + +def workflow_json_declares_triggers(workflow_json: Dict[str, Any]) -> bool: + if not isinstance(workflow_json, dict): + return False + if "triggers" in workflow_json: + return isinstance(workflow_json.get("triggers"), list) + metadata = workflow_json.get("metadata") + return isinstance(metadata, dict) and isinstance(metadata.get("triggers"), list) + + +def trigger_definitions_to_json(triggers: Iterable[TriggerDefinition]) -> List[Dict[str, Any]]: + return [ + trigger.model_dump(mode="json", by_alias=True, exclude_none=True) + for trigger in normalize_trigger_definitions(triggers) + ] + + +def set_workflow_json_triggers( + workflow_json: Dict[str, Any], + triggers: Iterable[TriggerDefinition], +) -> Dict[str, Any]: + updated = dict(workflow_json) + updated["triggers"] = trigger_definitions_to_json(triggers) + metadata = updated.get("metadata") + if isinstance(metadata, dict) and "triggers" in metadata: + metadata = dict(metadata) + metadata.pop("triggers", None) + updated["metadata"] = metadata + return updated diff --git a/flocks/workflow/triggers/runtime.py b/flocks/workflow/triggers/runtime.py new file mode 100644 index 000000000..5f6d94a52 --- /dev/null +++ b/flocks/workflow/triggers/runtime.py @@ -0,0 +1,477 @@ +"""Unified trigger runtime with legacy manager compatibility.""" + +from __future__ import annotations + +import asyncio +import json +import time +from typing import Any, Dict, List, Optional, Tuple + +from flocks.storage.storage import Storage +from flocks.utils.log import Log +from flocks.workflow.execution_store import ( + compact_history_for_storage, + compact_outputs_for_storage, + create_execution_record, + record_execution_result, + resolve_execution_outcome, +) +from flocks.workflow.fs_store import read_workflow_dir, workflow_scan_dirs +from flocks.workflow.runner import run_workflow + +from .compat import ( + LEGACY_KAFKA_CONFIG_PREFIX, + LEGACY_POLLER_CONFIG_PREFIX, + LEGACY_SYSLOG_CONFIG_PREFIX, + kafka_trigger_to_legacy_config, + schedule_trigger_to_legacy_config, + syslog_trigger_to_legacy_config, + trigger_to_legacy_config, +) +from .custom_loader import list_trigger_plugins, load_trigger_plugin_module +from .dispatcher import EventDispatcher, TriggerDispatchError, build_trigger_event +from .models import ( + TriggerDefinition, + TriggerEvent, + TriggerRuntimeStatus, + workflow_json_declares_triggers, + workflow_trigger_definitions_from_json, +) + +log = Log.create(service="workflow.trigger.runtime") + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +class TriggerRuntime: + """Unified trigger runtime that wraps legacy managers and custom adapters.""" + + def __init__(self) -> None: + self._dispatcher = EventDispatcher() + self._custom_adapter_tasks: Dict[tuple[str, str], asyncio.Task[Any]] = {} + self._custom_adapters: Dict[tuple[str, str], Any] = {} + self._custom_status: Dict[tuple[str, str], Dict[str, Any]] = {} + self._custom_adapter_signatures: Dict[tuple[str, str], str] = {} + + def _iter_workflows(self) -> List[Dict[str, Any]]: + merged: Dict[str, Dict[str, Any]] = {} + for root, source in workflow_scan_dirs(): + if not root.is_dir(): + continue + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + data = read_workflow_dir(entry, entry.name, source) + if data is not None: + merged[entry.name] = data + return list(merged.values()) + + async def _write_disabled_legacy_configs(self, workflow_id: str) -> None: + now_ms = _now_ms() + await Storage.write( + f"{LEGACY_POLLER_CONFIG_PREFIX}{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": now_ms}, + ) + await Storage.write( + f"{LEGACY_SYSLOG_CONFIG_PREFIX}{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": now_ms}, + ) + await Storage.write( + f"{LEGACY_KAFKA_CONFIG_PREFIX}{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": now_ms}, + ) + + @staticmethod + def _trigger_signature(trigger: TriggerDefinition) -> str: + payload = trigger.model_dump(mode="json", exclude_none=True) + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + async def _sync_legacy_configs_from_workflow(self, workflow_id: str, workflow_json: Dict[str, Any]) -> List[TriggerDefinition]: + triggers = workflow_trigger_definitions_from_json(workflow_json) + if not triggers: + if workflow_json_declares_triggers(workflow_json): + await self._write_disabled_legacy_configs(workflow_id) + return [] + + by_type = {trigger.type: trigger for trigger in triggers} + for trigger in triggers: + key, value = trigger_to_legacy_config(workflow_id, trigger) + if key and value is not None: + await Storage.write(key, value) + + if "schedule" not in by_type: + await Storage.write( + f"{LEGACY_POLLER_CONFIG_PREFIX}{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": _now_ms()}, + ) + if "syslog" not in by_type: + await Storage.write( + f"{LEGACY_SYSLOG_CONFIG_PREFIX}{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": _now_ms()}, + ) + if "kafka" not in by_type: + await Storage.write( + f"{LEGACY_KAFKA_CONFIG_PREFIX}{workflow_id}", + {"workflowId": workflow_id, "enabled": False, "updatedAt": _now_ms()}, + ) + return triggers + + async def start_all(self) -> None: + for workflow in self._iter_workflows(): + try: + await self._sync_legacy_configs_from_workflow(workflow["id"], workflow.get("workflowJson") or {}) + except Exception as exc: + log.warning("trigger.sync_legacy.failed", {"workflow_id": workflow.get("id"), "error": str(exc)}) + + from flocks.ingest.syslog.manager import default_manager as syslog_manager + from flocks.ingest.kafka.manager import default_manager as kafka_manager + from flocks.workflow.poller_manager import default_manager as poller_manager + + await syslog_manager.start_all() + await kafka_manager.start_all() + await poller_manager.start_all() + + for workflow in self._iter_workflows(): + await self._start_custom_adapters_for_workflow(workflow["id"], workflow.get("workflowJson") or {}) + + async def stop_all(self) -> None: + from flocks.ingest.syslog.manager import default_manager as syslog_manager + from flocks.ingest.kafka.manager import default_manager as kafka_manager + from flocks.workflow.poller_manager import default_manager as poller_manager + + for workflow_id, trigger_id in list(self._custom_adapter_tasks.keys()): + await self._stop_custom_adapter(workflow_id, trigger_id) + + await syslog_manager.stop_all() + await kafka_manager.stop_all() + await poller_manager.stop_all() + + async def restart_workflow( + self, + workflow_id: str, + workflow_json: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + if workflow_json is None: + workflow = next((item for item in self._iter_workflows() if item.get("id") == workflow_id), None) + workflow_json = (workflow or {}).get("workflowJson") or {} + triggers = await self._sync_legacy_configs_from_workflow(workflow_id, workflow_json or {}) + + from flocks.ingest.syslog.manager import default_manager as syslog_manager + from flocks.ingest.kafka.manager import default_manager as kafka_manager + from flocks.workflow.poller_manager import default_manager as poller_manager + + statuses: Dict[str, Any] = {} + by_type = {trigger.type: trigger for trigger in triggers} + + if "syslog" in by_type: + statuses["syslog"] = await syslog_manager.restart_workflow(workflow_id) + else: + await syslog_manager.stop_workflow(workflow_id) + statuses["syslog"] = {"state": "stopped", "error": None} + if "kafka" in by_type: + statuses["kafka"] = await kafka_manager.restart_workflow(workflow_id) + else: + await kafka_manager.stop_workflow(workflow_id) + statuses["kafka"] = {"state": "stopped", "error": None} + if "schedule" in by_type: + statuses["schedule"] = await poller_manager.restart_workflow(workflow_id) + else: + await poller_manager.stop_workflow(workflow_id) + statuses["schedule"] = {"state": "stopped", "error": None} + + await self._start_custom_adapters_for_workflow(workflow_id, workflow_json or {}) + return statuses + + async def _execute_workflow( + self, + *, + workflow_id: str, + workflow_json: Dict[str, Any], + trigger: TriggerDefinition, + mapped_inputs: Dict[str, Any], + ) -> Dict[str, Any]: + exec_data = await create_execution_record( + workflow_id, + input_params=mapped_inputs, + ) + exec_id = exec_data["id"] + started_at = time.time() + try: + result = await asyncio.to_thread( + run_workflow, + workflow=workflow_json, + inputs=mapped_inputs, + trace=False, + ) + status_value, error_message = resolve_execution_outcome(result) + exec_data.update( + { + "status": status_value, + "outputResults": compact_outputs_for_storage(result.outputs), + "finishedAt": _now_ms(), + "duration": time.time() - started_at, + "errorMessage": error_message, + "executionLog": compact_history_for_storage(result.history), + "currentNodeId": result.last_node_id, + "currentPhase": status_value, + "currentStepIndex": result.steps, + "triggerId": trigger.id, + "triggerType": trigger.type, + "deliveryId": mapped_inputs.get("_flocks", {}).get("trigger", {}).get("deliveryId"), + "attempt": mapped_inputs.get("_flocks", {}).get("trigger", {}).get("attempt"), + "triggerSource": mapped_inputs.get("_flocks", {}).get("trigger", {}).get("source"), + } + ) + except Exception as exc: + exec_data.update( + { + "status": "error", + "finishedAt": _now_ms(), + "duration": time.time() - started_at, + "errorMessage": str(exc), + "triggerId": trigger.id, + "triggerType": trigger.type, + "deliveryId": mapped_inputs.get("_flocks", {}).get("trigger", {}).get("deliveryId"), + "attempt": mapped_inputs.get("_flocks", {}).get("trigger", {}).get("attempt"), + "triggerSource": mapped_inputs.get("_flocks", {}).get("trigger", {}).get("source"), + } + ) + await record_execution_result(workflow_id, exec_id, exec_data) + return exec_data + + async def dispatch_event( + self, + *, + workflow_id: str, + workflow_json: Dict[str, Any], + trigger: TriggerDefinition, + event: TriggerEvent, + ) -> Dict[str, Any]: + async def _executor(mapped_inputs: Dict[str, Any]) -> Dict[str, Any]: + return await self._execute_workflow( + workflow_id=workflow_id, + workflow_json=workflow_json, + trigger=trigger, + mapped_inputs=mapped_inputs, + ) + + return await self._dispatcher.dispatch(trigger=trigger, event=event, executor=_executor) + + async def _stop_custom_adapter(self, workflow_id: str, trigger_id: str) -> None: + key = (workflow_id, trigger_id) + adapter = self._custom_adapters.pop(key, None) + task = self._custom_adapter_tasks.pop(key, None) + if adapter is not None and hasattr(adapter, "stop"): + try: + result = adapter.stop() + if asyncio.iscoroutine(result): + await result + except Exception: + pass + if task is not None and not task.done(): + task.cancel() + try: + await task + except Exception: + pass + self._custom_adapter_signatures.pop(key, None) + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": trigger_id, + "triggerType": "custom_adapter", + "state": "stopped", + "error": None, + } + + async def _start_custom_adapters_for_workflow(self, workflow_id: str, workflow_json: Dict[str, Any]) -> None: + triggers = workflow_trigger_definitions_from_json(workflow_json) + desired_signatures = { + (workflow_id, trigger.id or ""): self._trigger_signature(trigger) + for trigger in triggers + if trigger.type == "custom_adapter" and trigger.enabled + } + for active_workflow_id, active_trigger_id in list(self._custom_adapter_tasks.keys()): + key = (active_workflow_id, active_trigger_id) + if active_workflow_id != workflow_id: + continue + if key not in desired_signatures: + await self._stop_custom_adapter(active_workflow_id, active_trigger_id) + continue + if self._custom_adapter_signatures.get(key) != desired_signatures[key]: + await self._stop_custom_adapter(active_workflow_id, active_trigger_id) + + for trigger in triggers: + if trigger.type != "custom_adapter" or not trigger.enabled: + continue + key = (workflow_id, trigger.id or "") + trigger_signature = desired_signatures[key] + if ( + key in self._custom_adapter_tasks + and self._custom_adapter_signatures.get(key) == trigger_signature + ): + continue + plugin_id = str((trigger.source or {}).get("adapterId") or (trigger.source or {}).get("pluginId") or "").strip() + plugin_spec = next((item for item in list_trigger_plugins() if item.get("id") == plugin_id), None) + if plugin_spec is None: + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "failed", + "error": f"custom trigger plugin not found: {plugin_id}", + } + continue + module = load_trigger_plugin_module(plugin_spec) + if module is None: + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "failed", + "error": "failed to load custom trigger plugin module", + } + continue + try: + adapter = None + if hasattr(module, "create_trigger_adapter"): + adapter = module.create_trigger_adapter(trigger.model_dump(mode="json")) + elif hasattr(module, "TriggerAdapter"): + adapter = module.TriggerAdapter(trigger.model_dump(mode="json")) + if adapter is None: + raise RuntimeError("plugin must expose create_trigger_adapter() or TriggerAdapter") + except Exception as exc: + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "failed", + "error": str(exc), + } + continue + + async def _emit(payload: Any, *, _trigger: TriggerDefinition = trigger) -> Dict[str, Any]: + event = payload if isinstance(payload, TriggerEvent) else build_trigger_event( + workflow_id=workflow_id, + trigger=_trigger, + body=payload, + raw=payload, + ) + try: + result = await self.dispatch_event( + workflow_id=workflow_id, + workflow_json=workflow_json, + trigger=_trigger, + event=event, + ) + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": _trigger.id, + "triggerType": _trigger.type, + "state": "running", + "error": None, + "lastDeliveryId": event.source.deliveryId, + "lastMatched": result.get("matched"), + } + return result + except TriggerDispatchError as exc: + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": _trigger.id, + "triggerType": _trigger.type, + "state": "failed", + "error": str(exc), + } + raise + + async def _runner() -> None: + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "running", + "error": None, + "pluginId": plugin_id, + } + try: + result = adapter.start(trigger.model_dump(mode="json"), _emit) + if asyncio.iscoroutine(result): + await result + except asyncio.CancelledError: + raise + except Exception as exc: + self._custom_status[key] = { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "failed", + "error": str(exc), + "pluginId": plugin_id, + } + + self._custom_adapters[key] = adapter + self._custom_adapter_signatures[key] = trigger_signature + self._custom_adapter_tasks[key] = asyncio.create_task( + _runner(), + name=f"trigger-custom-{workflow_id}-{trigger.id}", + ) + + async def get_trigger_status(self, workflow_id: str, trigger: TriggerDefinition) -> Dict[str, Any]: + if trigger.type == "syslog": + from flocks.ingest.syslog.manager import default_manager as syslog_manager + + status = syslog_manager.get_listener_status(workflow_id) + return {"workflowId": workflow_id, "triggerId": trigger.id, "triggerType": trigger.type, **status} + if trigger.type == "kafka": + from flocks.ingest.kafka.manager import default_manager as kafka_manager + + status = kafka_manager.get_consumer_status(workflow_id) + return {"workflowId": workflow_id, "triggerId": trigger.id, "triggerType": trigger.type, **status} + if trigger.type == "schedule": + from flocks.workflow.poller_manager import default_manager as poller_manager + + status = poller_manager.get_status(workflow_id) + return {"workflowId": workflow_id, "triggerId": trigger.id, "triggerType": trigger.type, **status} + if trigger.type in {"webhook", "custom_webhook"}: + return { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "ready" if trigger.enabled else "stopped", + "error": None, + "path": (trigger.source or {}).get("path"), + "method": (trigger.source or {}).get("method", "POST"), + } + if trigger.type == "custom_adapter": + return self._custom_status.get( + (workflow_id, trigger.id or ""), + { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "stopped", + "error": None, + }, + ) + return { + "workflowId": workflow_id, + "triggerId": trigger.id, + "triggerType": trigger.type, + "state": "ready" if trigger.enabled else "stopped", + "error": None, + } + + async def get_workflow_trigger_statuses( + self, + workflow_id: str, + workflow_json: Dict[str, Any], + ) -> List[Dict[str, Any]]: + triggers = workflow_trigger_definitions_from_json(workflow_json) + return [await self.get_trigger_status(workflow_id, trigger) for trigger in triggers] + + def list_plugin_specs(self) -> List[Dict[str, Any]]: + return list_trigger_plugins() + + +default_runtime = TriggerRuntime() diff --git a/flocks/workspace/manager.py b/flocks/workspace/manager.py index df8b19465..8e57f9e69 100644 --- a/flocks/workspace/manager.py +++ b/flocks/workspace/manager.py @@ -24,7 +24,7 @@ # Note: dotfiles like .gitignore have suffix='' in Python, so they are NOT # matched here; they will fall through to the binary-file path (download only). TEXT_EXTENSIONS = { - ".md", ".txt", ".log", ".json", ".yaml", ".yml", + ".md", ".txt", ".log", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".py", ".js", ".ts", ".sh", ".bash", ".csv", ".xml", ".html", ".css", ".tsx", ".jsx", ".env", diff --git a/pyproject.toml b/pyproject.toml index 11984e009..f54edb8ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "flocks" -version = "v2026.6.4" +version = "v2026.6.10" description = "AI-Native SecOps platform with multi-agent collaboration" authors = [ {name = "Flocks Team", email = "team@example.com"} @@ -23,6 +23,7 @@ dependencies = [ "uvicorn[standard]>=0.27.0", "python-multipart>=0.0.9", "sse-starlette>=1.8.2", + "starlette>=1.0.1", "websockets>=12.0", "aiohttp>=3.13.3", "requests>=2.32.5", @@ -108,9 +109,6 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["flocks"] -[tool.hatch.build.targets.wheel.force-include] -".flocks/flockshub" = ".flocks/flockshub" - [tool.ruff] line-length = 120 target-version = "py312" diff --git a/tests/browser/test_admin.py b/tests/browser/test_admin.py index 84c1034dc..ac701077e 100644 --- a/tests/browser/test_admin.py +++ b/tests/browser/test_admin.py @@ -187,6 +187,7 @@ def test_run_doctor_prints_active_browser_connections_and_active_pages(monkeypat assert "[ok ] active browser connections — 2" in out assert " default — active page: Example — https://example.test" in out assert " cats — active page: Cat - Wikipedia — https://en.wikipedia.org/wiki/Cat" in out + assert "next action ready; use `flocks browser -c 'print(page_info())'`" in out assert "profile-use installed" not in out assert "BROWSER_USE_API_KEY set" not in out @@ -218,6 +219,36 @@ def test_doctor_page_output_truncates_long_text(monkeypatch, capsys) -> None: assert "BROWSER_USE_API_KEY set" not in out +def test_run_doctor_suggests_attach_when_daemon_alive_without_active_connections(monkeypatch, capsys) -> None: + monkeypatch.setattr(admin, "_version", lambda: "0.1.0") + monkeypatch.setattr(admin, "_install_mode", lambda: "git") + monkeypatch.setattr(admin, "_chrome_running", lambda: True) + monkeypatch.setattr(admin, "daemon_alive", lambda: True) + monkeypatch.setattr(admin, "browser_connections", lambda: []) + monkeypatch.setattr(admin, "_latest_release_tag", lambda: "0.1.0") + + assert admin.run_doctor() == 0 + + out = capsys.readouterr().out + assert "[ok ] active browser connections — 0" not in out + assert "[FAIL] active browser connections — 0" in out + assert "next action attach; run `flocks browser -c 'print(page_info())'` before setup" in out + + +def test_run_doctor_suggests_setup_when_target_exists_but_daemon_missing(monkeypatch, capsys) -> None: + monkeypatch.setattr(admin, "_version", lambda: "0.1.0") + monkeypatch.setattr(admin, "_install_mode", lambda: "git") + monkeypatch.setattr(admin, "_chrome_running", lambda: True) + monkeypatch.setattr(admin, "daemon_alive", lambda: False) + monkeypatch.setattr(admin, "browser_connections", lambda: []) + monkeypatch.setattr(admin, "_latest_release_tag", lambda: "0.1.0") + + assert admin.run_doctor() == 1 + + out = capsys.readouterr().out + assert "next action setup; run `flocks browser --setup`" in out + + def test_run_setup_uses_generic_missing_browser_wording(monkeypatch, capsys) -> None: monkeypatch.setattr(admin, "daemon_alive", lambda: False) monkeypatch.setattr(admin, "_chrome_running", lambda: False) @@ -343,6 +374,7 @@ def test_run_doctor_uses_generic_browser_wording_when_missing(monkeypatch, capsy out = capsys.readouterr().out assert "[FAIL] browser running" in out assert "start Chrome, Chromium, or Edge and rerun `flocks browser --setup`" in out + assert "next action start Chrome/Chromium/Edge or provide BU_CDP_URL/BU_CDP_WS" in out def test_run_doctor_accepts_explicit_remote_cdp_without_local_browser(monkeypatch, capsys) -> None: @@ -359,6 +391,7 @@ def test_run_doctor_accepts_explicit_remote_cdp_without_local_browser(monkeypatc out = capsys.readouterr().out assert "[ok ] browser target — configured via BU_CDP_URL" in out assert "[ok ] daemon alive" in out + assert "next action attach; run `flocks browser -c 'print(page_info())'` before setup" in out def test_chrome_running_on_windows_handles_non_utf8_tasklist_output(monkeypatch) -> None: diff --git a/tests/channel/test_channel.py b/tests/channel/test_channel.py index 90d191607..098606aab 100644 --- a/tests/channel/test_channel.py +++ b/tests/channel/test_channel.py @@ -1248,6 +1248,17 @@ def test_register_and_get(self): reg.register(plugin) assert reg.get("test_ch") is plugin + def test_register_keeps_existing_channel_instance(self): + from flocks.channel.registry import ChannelRegistry + reg = ChannelRegistry() + first = _StubChannel("test_ch", "First") + replacement = _StubChannel("test_ch", "Replacement") + + reg.register(first) + reg.register(replacement) + + assert reg.get("test_ch") is first + def test_get_by_alias(self): from flocks.channel.registry import ChannelRegistry reg = ChannelRegistry() @@ -1352,3 +1363,284 @@ async def _fake_wait(tasks, timeout): assert cancelled.is_set() assert task.done() + + +class TestMediaFilenameHelpers: + def test_sanitize_filename_preserves_unicode_names(self): + from flocks.channel.media_filename import sanitize_filename + + assert sanitize_filename("报告 2026.pdf") == "报告 2026.pdf" + + def test_sanitize_filename_removes_path_separators(self): + from flocks.channel.media_filename import sanitize_filename + + assert sanitize_filename("../report.bin") == ".._report.bin" + + +# ------------------------------------------------------------------ +# Per-channel inbound media downloader routing +# ------------------------------------------------------------------ + +class TestDownloadChannelMediaRouting: + """The dispatcher's per-channel media hook must resolve to the + correct downloader and propagate ``config`` through. + """ + + @pytest.mark.asyncio + async def test_dispatch_to_wecom_downloader(self, monkeypatch): + from flocks.channel.inbound import dispatcher as dispatch_mod + + captured = {} + + async def fake_wecom(msg, config): + captured["msg"] = msg + captured["config"] = config + return SimpleNamespace( + filename="x.pdf", mime="application/pdf", + url="file:///tmp/x.pdf", source={"channel": "wecom"}, + ) + + async def fail(msg, config): + raise AssertionError("wrong downloader invoked") + + # The dispatcher uses dynamic lookup via __import__, so the + # test-level patch is on the inbound_media module attribute. + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media.download_inbound_media", + fake_wecom, + ) + # In case the dynamic import resolves to a different attribute + # (e.g. cached), also patch the inner module binding. + import flocks.channel.builtin.wecom.inbound_media as wecom_inb + monkeypatch.setattr(wecom_inb, "download_inbound_media", fake_wecom) + + result = await dispatch_mod._download_channel_media( + InboundMessage( + channel_id="wecom", account_id="a", message_id="m1", + sender_id="u", media_url="https://example.com/x", + ), + {"botId": "b"}, + ) + assert result is not None + assert captured["msg"].channel_id == "wecom" + assert captured["config"] == {"botId": "b"} + + @pytest.mark.asyncio + async def test_dispatch_to_dingtalk_downloader(self, monkeypatch): + from flocks.channel.inbound import dispatcher as dispatch_mod + + captured = {} + + async def fake_dingtalk(msg, config): + captured["msg"] = msg + return SimpleNamespace( + filename="y.png", mime="image/png", + url="file:///tmp/y.png", source={"channel": "dingtalk"}, + ) + + import flocks.channel.builtin.dingtalk.inbound_media as dt_inb + monkeypatch.setattr(dt_inb, "download_inbound_media", fake_dingtalk) + + result = await dispatch_mod._download_channel_media( + InboundMessage( + channel_id="dingtalk", account_id="a", message_id="m", + sender_id="u", media_url="CODE", + ), + {}, + ) + assert result is not None + assert captured["msg"].channel_id == "dingtalk" + + @pytest.mark.asyncio + async def test_dispatch_to_telegram_downloader(self, monkeypatch): + from flocks.channel.inbound import dispatcher as dispatch_mod + + captured = {} + + async def fake_telegram(msg, config): + captured["msg"] = msg + return SimpleNamespace( + filename="z.jpg", mime="image/jpeg", + url="file:///tmp/z.jpg", source={"channel": "telegram"}, + ) + + import flocks.channel.builtin.telegram.inbound_media as tg_inb + monkeypatch.setattr(tg_inb, "download_inbound_media", fake_telegram) + + result = await dispatch_mod._download_channel_media( + InboundMessage( + channel_id="telegram", account_id="a", message_id="m", + sender_id="u", media_url="telegram://photo/ABC", + ), + {"botToken": "tok"}, + ) + assert result is not None + assert captured["msg"].channel_id == "telegram" + + @pytest.mark.asyncio + async def test_unknown_channel_returns_none(self): + from flocks.channel.inbound import dispatcher as dispatch_mod + + result = await dispatch_mod._download_channel_media( + InboundMessage( + channel_id="unknown", account_id="a", message_id="m", + sender_id="u", media_url="https://x/y", + ), + {}, + ) + assert result is None + + +# ------------------------------------------------------------------ +# _is_placeholder_text +# ------------------------------------------------------------------ + +class TestIsPlaceholderText: + def test_recognises_channel_placeholders(self): + from flocks.channel.inbound.dispatcher import _is_placeholder_text + assert _is_placeholder_text("[图片消息]") is True + assert _is_placeholder_text("[文件消息]") is True + assert _is_placeholder_text("[文件消息: report.pdf]") is True + assert _is_placeholder_text("[Image]") is True + assert _is_placeholder_text("[Attachment]") is True + assert _is_placeholder_text("[图片]") is True + assert _is_placeholder_text("[文件]") is True + + def test_does_not_match_normal_text(self): + from flocks.channel.inbound.dispatcher import _is_placeholder_text + assert _is_placeholder_text("hello") is False + assert _is_placeholder_text("看这个") is False + assert _is_placeholder_text("") is False + # Suffix beyond the placeholder is fine — the text starts with a + # placeholder token, so the dispatcher will still rewrite it. + assert _is_placeholder_text("[文件消息: x] extra text") is True + assert _is_placeholder_text("not a placeholder [文件消息: x]") is False + + +# ------------------------------------------------------------------ +# Full pipeline: synthetic inbound → dispatcher → FilePart (per channel) +# ------------------------------------------------------------------ + +class TestAppendUserMessagePerChannel: + @pytest.mark.asyncio + async def test_wecom_pipeline_stores_file_part(self, monkeypatch): + from flocks.channel.inbound.dispatcher import InboundDispatcher + from flocks.config.config import ChannelConfig + + created_message = SimpleNamespace(id="m1") + store_part = AsyncMock() + + monkeypatch.setattr( + "flocks.session.message.Message.create", + AsyncMock(return_value=created_message), + ) + monkeypatch.setattr( + "flocks.session.message.Message.store_part", + store_part, + ) + + import flocks.channel.builtin.wecom.inbound_media as wecom_inb + async def fake_download(msg, config): + return SimpleNamespace( + filename="report.pdf", mime="application/pdf", + url="file:///tmp/report.pdf", + source={"channel": "wecom"}, + ) + monkeypatch.setattr(wecom_inb, "download_inbound_media", fake_download) + + await InboundDispatcher._append_user_message( + "s1", + "[文件消息: report.pdf]", + InboundMessage( + channel_id="wecom", account_id="a", message_id="m1", + sender_id="u", media_url="https://example.com/report.pdf", + ), + ChannelConfig(enabled=True, botId="b", secret="s"), + ) + + assert store_part.await_count >= 1 + stored_part = store_part.await_args_list[0].args[2] + assert stored_part.type == "file" + assert stored_part.filename == "report.pdf" + assert stored_part.mime == "application/pdf" + assert stored_part.url == "file:///tmp/report.pdf" + + @pytest.mark.asyncio + async def test_dingtalk_pipeline_stores_file_part(self, monkeypatch): + from flocks.channel.inbound.dispatcher import InboundDispatcher + + created_message = SimpleNamespace(id="m1") + store_part = AsyncMock() + + monkeypatch.setattr( + "flocks.session.message.Message.create", + AsyncMock(return_value=created_message), + ) + monkeypatch.setattr( + "flocks.session.message.Message.store_part", + store_part, + ) + + import flocks.channel.builtin.dingtalk.inbound_media as dt_inb + async def fake_download(msg, config): + return SimpleNamespace( + filename="image.png", mime="image/png", + url="file:///tmp/image.png", + source={"channel": "dingtalk"}, + ) + monkeypatch.setattr(dt_inb, "download_inbound_media", fake_download) + + await InboundDispatcher._append_user_message( + "s1", + "[图片消息]", + InboundMessage( + channel_id="dingtalk", account_id="a", message_id="m1", + sender_id="u", media_url="CODE_123", + ), + None, + ) + + assert store_part.await_count >= 1 + stored_part = store_part.await_args_list[0].args[2] + assert stored_part.type == "file" + assert stored_part.filename == "image.png" + + @pytest.mark.asyncio + async def test_telegram_pipeline_stores_file_part(self, monkeypatch): + from flocks.channel.inbound.dispatcher import InboundDispatcher + + created_message = SimpleNamespace(id="m1") + store_part = AsyncMock() + + monkeypatch.setattr( + "flocks.session.message.Message.create", + AsyncMock(return_value=created_message), + ) + monkeypatch.setattr( + "flocks.session.message.Message.store_part", + store_part, + ) + + import flocks.channel.builtin.telegram.inbound_media as tg_inb + async def fake_download(msg, config): + return SimpleNamespace( + filename="photo.jpg", mime="image/jpeg", + url="file:///tmp/photo.jpg", + source={"channel": "telegram"}, + ) + monkeypatch.setattr(tg_inb, "download_inbound_media", fake_download) + + await InboundDispatcher._append_user_message( + "s1", + "[图片]", + InboundMessage( + channel_id="telegram", account_id="a", message_id="m1", + sender_id="u", media_url="telegram://photo/ABC", + ), + None, + ) + + assert store_part.await_count >= 1 + stored_part = store_part.await_args_list[0].args[2] + assert stored_part.type == "file" + assert stored_part.filename == "photo.jpg" diff --git a/tests/channel/test_dingtalk.py b/tests/channel/test_dingtalk.py index f6d0b1c44..a258df189 100644 --- a/tests/channel/test_dingtalk.py +++ b/tests/channel/test_dingtalk.py @@ -13,12 +13,13 @@ import contextlib import importlib.util import json +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest -from flocks.channel.base import ChatType, OutboundContext +from flocks.channel.base import ChatType, InboundMessage, OutboundContext from flocks.channel.builtin.dingtalk.client import ( DingTalkApiError, ensure_api_success, @@ -923,6 +924,36 @@ def test_chatbot_message_to_inbound_returns_none_when_empty(self): message, channel_id="dingtalk", account_id="default", ) is None + def test_chatbot_message_to_inbound_extracts_file_download_code_from_content_extension(self): + from flocks.channel.builtin.dingtalk.stream import ( + chatbot_message_to_inbound, + ) + message = SimpleNamespace( + text=None, + extensions={ + "content": { + "fileName": "report.pdf", + "downloadCode": "DL_FILE_1", + "fileId": "file_1", + }, + }, + conversation_id="cid_file", + conversation_type="1", + sender_id="u1", + sender_staff_id="staff_001", + sender_nick="Alice", + message_id="msg_file_1", + message_type="file", + ) + + inbound = chatbot_message_to_inbound( + message, channel_id="dingtalk", account_id="default", + ) + + assert inbound is not None + assert inbound.media_url == "DL_FILE_1" + assert inbound.text == "" + # ------------------------------------------------------------------ # Resilience regressions: R1 (silent stall) + R3 (back-pressure) @@ -1409,3 +1440,392 @@ async def fake_bind(self, **kwargs): }, ) assert resp.status_code == 200, resp.text + + +# ------------------------------------------------------------------ +# Inbound media download (download_code exchange + size guard) +# ------------------------------------------------------------------ + +class TestDingTalkInboundMedia: + @pytest.mark.asyncio + async def test_exchange_then_download(self, monkeypatch, tmp_path): + from flocks.channel.builtin.dingtalk import inbound_media as mod + + captured = {} + + async def fake_exchange(*, config, account_id, download_code): + captured["code"] = download_code + captured["account"] = account_id + return "https://example.com/file.png", "report.png" + + class FakeResponse: + headers = {"content-length": "10"} + + def raise_for_status(self): + return None + + async def aiter_bytes(self, _size): + yield b"hello-img" + + class FakeStream: + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, *_args): + return None + + class FakeClient: + def stream(self, *_a, **_k): + return FakeStream() + + async def fake_download(_url, _max_bytes): + return b"hello-img", None + + monkeypatch.setattr(mod, "_exchange_download_code", fake_exchange) + monkeypatch.setattr(mod, "_download_remote_bytes_limited", fake_download) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="dingtalk", + account_id="acc1", + message_id="m1", + sender_id="u1", + media_url="download-code-1", + ), + config={}, + max_bytes=20, + ) + + assert captured == {"code": "download-code-1", "account": "acc1"} + assert media is not None + assert media.filename == "report.png" + assert media.mime == "image/png" + assert Path(media.url.removeprefix("file://")).read_bytes() == b"hello-img" + + @pytest.mark.asyncio + async def test_file_message_content_filename_is_preserved(self, monkeypatch, tmp_path): + from flocks.channel.builtin.dingtalk import inbound_media as mod + + async def fake_exchange(*, config, account_id, download_code): + return "https://example.com/download", None + + async def fake_download(_url, _max_bytes): + return b"%PDF", None + + monkeypatch.setattr(mod, "_exchange_download_code", fake_exchange) + monkeypatch.setattr(mod, "_download_remote_bytes_limited", fake_download) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="dingtalk", + account_id="acc1", + message_id="m_file", + sender_id="u1", + media_url="download-code-file", + raw=SimpleNamespace( + extensions={ + "content": { + "fileName": "安全报告.pdf", + "downloadCode": "download-code-file", + }, + }, + ), + ), + config={}, + ) + + assert media is not None + assert media.filename == "安全报告.pdf" + assert media.mime == "application/pdf" + + @pytest.mark.asyncio + async def test_direct_url_skips_exchange(self, monkeypatch, tmp_path): + from flocks.channel.builtin.dingtalk import inbound_media as mod + + called = {"exchange": False} + + async def fake_exchange(*, config, account_id, download_code): + called["exchange"] = True + return "", None + + class FakeResponse: + headers = {} + + def raise_for_status(self): + return None + + async def aiter_bytes(self, _size): + yield b"raw" + + class FakeStream: + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, *_args): + return None + + class FakeClient: + def stream(self, *_a, **_k): + return FakeStream() + + async def fake_download(_url, _max_bytes): + return b"raw", None + + monkeypatch.setattr(mod, "_exchange_download_code", fake_exchange) + monkeypatch.setattr(mod, "_download_remote_bytes_limited", fake_download) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="dingtalk", + account_id="acc1", + message_id="m2", + sender_id="u1", + media_url="https://example.com/raw.png", + ), + config={}, + max_bytes=20, + ) + assert called["exchange"] is False + assert media is not None + assert media.source["download_code"] is None + assert media.filename.endswith(".png") + + @pytest.mark.asyncio + async def test_rejects_oversized(self, monkeypatch, tmp_path): + from flocks.channel.builtin.dingtalk import inbound_media as mod + + async def fake_exchange(*, config, account_id, download_code): + return "https://example.com/big", None + + async def fake_download(_url, _max_bytes): + raise mod.DingTalkInboundMediaTooLarge("too large") + + monkeypatch.setattr(mod, "_exchange_download_code", fake_exchange) + monkeypatch.setattr(mod, "_download_remote_bytes_limited", fake_download) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="dingtalk", + account_id="acc1", + message_id="m3", + sender_id="u1", + media_url="code", + ), + config={}, + max_bytes=10, + ) + assert media is None + assert list(tmp_path.iterdir()) == [] + + @pytest.mark.asyncio + async def test_exchange_failure_returns_none(self, monkeypatch, tmp_path): + from flocks.channel.builtin.dingtalk import inbound_media as mod + from flocks.channel.builtin.dingtalk.client import DingTalkApiError + + async def fake_exchange(*, config, account_id, download_code): + raise DingTalkApiError("missing credentials") + + monkeypatch.setattr(mod, "_exchange_download_code", fake_exchange) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="dingtalk", + account_id="acc1", + message_id="m4", + sender_id="u1", + media_url="code", + ), + config={}, + ) + assert media is None + + +# ------------------------------------------------------------------ +# Outbound media: prepare + send_media integration +# ------------------------------------------------------------------ + +class TestDingTalkSendMedia: + @pytest.mark.asyncio + async def test_prepare_dingtalk_media_reads_local_file(self, tmp_path): + from flocks.channel.builtin.dingtalk.media import ( + prepare_dingtalk_media, + ) + + path = tmp_path / "report.pdf" + path.write_bytes(b"%PDF-1.4\nhello") + + async def fake_upload(*, config, account_id, data, filename): + return ("media_1", "code_1") + + from flocks.channel.builtin.dingtalk import media as mod + from unittest.mock import patch + with patch.object(mod, "upload_dingtalk_media", side_effect=fake_upload): + prepared = await prepare_dingtalk_media( + config={}, account_id="acc", media_url=path.as_uri(), + ) + assert prepared.media_id == "media_1" + assert prepared.download_code == "code_1" + assert prepared.media_type == "file" + assert prepared.filename == "report.pdf" + assert prepared.data == b"%PDF-1.4\nhello" + + @pytest.mark.asyncio + async def test_send_media_uploads_and_sends_file(self, tmp_path): + from flocks.channel.builtin.dingtalk.channel import DingTalkChannel + from flocks.channel.builtin.dingtalk import media as media_mod + + path = tmp_path / "doc.pdf" + path.write_bytes(b"data") + + ch = DingTalkChannel() + ch._config = { + "appKey": "ak", + "appSecret": "as", + "robotCode": "rc", + } + + async def fake_prepare(*, config, account_id, media_url, **kwargs): + return media_mod.PreparedDingTalkMedia( + data=b"data", + filename="doc.pdf", + mime="application/pdf", + media_type="file", + media_id="media_1", + download_code="dl_1", + ) + + async def fake_send_message_app(*, config, to, text, account_id): + return {"message_id": "t1", "chat_id": "u1"} + + sent = [] + + async def fake_api_request_for_account(method, path, *, config, account_id, json_body): + sent.append({"method": method, "path": path, "body": json_body}) + return {"processQueryKey": "mid_1"} + + from unittest.mock import patch + with patch.object(media_mod, "prepare_dingtalk_media", side_effect=fake_prepare), \ + patch("flocks.channel.builtin.dingtalk.send.send_message_app", + side_effect=fake_send_message_app), \ + patch("flocks.channel.builtin.dingtalk.client.api_request_for_account", + side_effect=fake_api_request_for_account): + result = await ch.send_media( + OutboundContext( + channel_id="dingtalk", + to="u1", + media_url=path.as_uri(), + ) + ) + + assert result.success is True + assert result.message_id == "mid_1" + assert sent and sent[0]["path"] == "/v1.0/robot/oToMessages/batchSend" + body = sent[0]["body"] + assert body["msgKey"] == "file" + import json as _json + params = _json.loads(body["msgParam"]) + assert params["downloadCode"] == "dl_1" + assert params["fileName"] == "doc.pdf" + + @pytest.mark.asyncio + async def test_send_media_with_text_sends_text_after_file(self, tmp_path): + from flocks.channel.builtin.dingtalk.channel import DingTalkChannel + from flocks.channel.builtin.dingtalk import media as media_mod + + path = tmp_path / "doc.pdf" + path.write_bytes(b"data") + ch = DingTalkChannel() + ch._config = { + "appKey": "ak", "appSecret": "as", "robotCode": "rc", + } + + async def fake_prepare(*, config, account_id, media_url, **kwargs): + return media_mod.PreparedDingTalkMedia( + data=b"d", filename="doc.pdf", mime="application/pdf", + media_type="file", media_id="m1", download_code="dc1", + ) + + order: list[str] = [] + + async def fake_send_message_app(*, config, to, text, account_id): + order.append(f"text:{text}") + return {"message_id": "txt_1", "chat_id": "u1"} + + async def fake_api_request_for_account(method, path, *, config, account_id, json_body): + order.append(f"media:{json_body['msgKey']}") + return {"processQueryKey": "mid_1"} + + from unittest.mock import patch + with patch.object(media_mod, "prepare_dingtalk_media", side_effect=fake_prepare), \ + patch("flocks.channel.builtin.dingtalk.send.send_message_app", + side_effect=fake_send_message_app), \ + patch("flocks.channel.builtin.dingtalk.client.api_request_for_account", + side_effect=fake_api_request_for_account): + result = await ch.send_media( + OutboundContext( + channel_id="dingtalk", + to="u1", + text="这是附件说明", + media_url=path.as_uri(), + ) + ) + + assert result.success is True + # media-first then text + assert order[0].startswith("media:") + assert order[1].startswith("text:") + + @pytest.mark.asyncio + async def test_send_media_image_url_uses_inline_image_msgkey(self, tmp_path): + from flocks.channel.builtin.dingtalk.channel import DingTalkChannel + from flocks.channel.builtin.dingtalk import media as media_mod + + ch = DingTalkChannel() + ch._config = { + "appKey": "ak", "appSecret": "as", "robotCode": "rc", + } + + async def fake_prepare(*, config, account_id, media_url, **kwargs): + return media_mod.PreparedDingTalkMedia( + data=b"d", filename="x.png", mime="image/png", + media_type="image", media_id="m1", download_code="dc1", + ) + + async def fake_api_request_for_account(method, path, *, config, account_id, json_body): + return {"processQueryKey": "img_1"} + + from unittest.mock import patch + with patch.object(media_mod, "prepare_dingtalk_media", side_effect=fake_prepare), \ + patch("flocks.channel.builtin.dingtalk.client.api_request_for_account", + side_effect=fake_api_request_for_account): + result = await ch.send_media( + OutboundContext( + channel_id="dingtalk", + to="u1", + media_url="https://example.com/photo.png", + ) + ) + + assert result.success is True + assert result.message_id == "img_1" + + @pytest.mark.asyncio + async def test_send_media_missing_target_returns_error(self, tmp_path): + from flocks.channel.builtin.dingtalk.channel import DingTalkChannel + + ch = DingTalkChannel() + ch._config = {"appKey": "ak", "appSecret": "as"} + result = await ch.send_media( + OutboundContext( + channel_id="dingtalk", + to="", + media_url="file:///x", + ) + ) + assert result.success is False + assert "requires 'to'" in result.error diff --git a/tests/channel/test_e2e_file_roundtrip.py b/tests/channel/test_e2e_file_roundtrip.py new file mode 100644 index 000000000..f35441146 --- /dev/null +++ b/tests/channel/test_e2e_file_roundtrip.py @@ -0,0 +1,479 @@ +""" +End-to-end smoke harness for channel file/image round-trips. + +Verifies that each channel's ``send_media`` and ``download_inbound_media`` +pair honours the same file on disk — i.e. a local file written by the +inbound path can be handed straight to the outbound path and uploaded +back to the platform without losing bytes or metadata. + +Each channel is exercised with **its own in-process fake server** so the +test does not depend on a real network connection. The fake servers +mirror the public API shape of the real services just closely enough to +let the production code path run unmodified. + +Channels covered: + - weixin (already complete; verified by per-file contract) + - feishu (download via tenant-token + upload via /im/v1/files) + - wecom (download via SDK stream + upload via upload_media) + - dingtalk (download_code → URL exchange + OAPI upload) + - telegram (getFile → file_path → Bot file download) +""" + +from __future__ import annotations + +import asyncio +import datetime +import hashlib +import io +import json +import os +import time +import wave +import struct +import zipfile +import zlib +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Awaitable, Callable +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from flocks.channel.base import ( + ChatType, + InboundMessage, + OutboundContext, +) +from flocks.channel.inbound.dispatcher import ( + InboundDispatcher, + _download_channel_media, + _is_placeholder_text, +) + + +# --------------------------------------------------------------------------- +# Tiny test asset factories +# --------------------------------------------------------------------------- + +def make_tiny_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> int: + """Write a real 1x1 PNG to *path* and return the byte count.""" + def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) + + signature = b"\x89PNG\r\n\x1a\n" + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + raw = b"\x00" + bytes(color) + idat = zlib.compress(raw) + iend = b"" + payload = signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", iend) + path.write_bytes(payload) + return len(payload) + + +def make_tiny_wav(path: Path) -> int: + """Write a 0.1s 8kHz mono PCM WAV (synthetic) and return the byte count.""" + sample_rate = 8000 + duration = 0.1 + n_samples = int(sample_rate * duration) + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(b"\x00\x00" * n_samples) + return path.stat().st_size + + +# --------------------------------------------------------------------------- +# Fake HTTP server with route table +# --------------------------------------------------------------------------- + +class _FakeResponse: + def __init__(self, status_code: int = 200, *, body: bytes = b"", + json_body: Any = None, content_type: str = "application/json", + headers: dict[str, str] | None = None) -> None: + self.status_code = status_code + self.is_success = 200 <= status_code < 300 + self._body = body + self._json = json_body + self.headers = headers or {"content-type": content_type} + self.text = body.decode("utf-8", errors="replace") if body else "" + + def json(self) -> Any: + if self._json is not None: + return self._json + return json.loads(self._body) if self._body else {} + + def raise_for_status(self) -> None: + if not self.is_success: + raise RuntimeError(f"HTTP {self.status_code}") + + +class _FakeTelegramServer: + """In-process fake of the Telegram Bot API.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + self.file_id_to_path: dict[str, str] = {} + self.token = "test-tok" + self.next_id = 1 + + def post(self, url: str, *, data=None, json=None, files=None, timeout=None) -> _FakeResponse: + self.calls.append({"url": url, "data": data, "json": json, "files": files}) + if url.endswith(f"/bot{self.token}/getFile"): + file_id = (data or {}).get("file_id") if data else None + if not file_id or file_id not in self.file_id_to_path: + return _FakeResponse(400, json_body={ + "ok": False, "description": "bad file_id", + }) + return _FakeResponse(200, json_body={ + "ok": True, + "result": { + "file_id": file_id, + "file_path": self.file_id_to_path[file_id], + }, + }) + + if url.endswith(f"/bot{self.token}/getMe"): + return _FakeResponse(200, json_body={ + "ok": True, "result": {"id": 1, "username": "flocksbot"}, + }) + + # Outbound send endpoints + endpoint = url.rsplit("/", 1)[-1] + if endpoint in {"sendPhoto", "sendDocument", "sendVideo", + "sendAudio", "sendVoice", "sendAnimation"}: + return _FakeResponse(200, json_body={ + "ok": True, + "result": { + "message_id": self.next_id, + "chat": {"id": 1}, + }, + }) + return _FakeResponse(404, json_body={"ok": False, "description": "not mocked"}) + + def get(self, url: str, *, params=None, timeout=None) -> _FakeResponse: + self.calls.append({"url": url, "params": params}) + if url.startswith(f"https://api.telegram.org/file/bot{self.token}/"): + rel = url.split(f"/bot{self.token}/", 1)[1] + # Return a fixed payload for any requested file_path. + return _FakeResponse(200, body=b"FAKE_TG_BYTES", content_type="application/octet-stream") + return _FakeResponse(404, body=b"not mocked") + + +# --------------------------------------------------------------------------- +# Wecom — uses the SDK; we'll patch the SDK module entirely +# --------------------------------------------------------------------------- + +class TestWecomRoundTrip: + @pytest.mark.asyncio + async def test_local_file_round_trip(self, tmp_path: Path, monkeypatch): + """Write a PNG, run it through prepare→upload (mocked) and + download_inbound_media (mocked) and confirm bytes match.""" + from flocks.channel.builtin.wecom import inbound_media, media as out_media + + png_path = tmp_path / "in.png" + size = make_tiny_png(png_path) + original = png_path.read_bytes() + + # 1) Prepare + upload (mocked upload) → media_id + prepared = await out_media.prepare_wecom_media(png_path.as_uri()) + assert prepared.data == original + assert prepared.media_type == "image" + upload_result = {"media_id": "MED_1", "type": "image"} + assert upload_result["media_id"] + + # 2) Download (mocked SDK stream + decrypt) — same bytes back + class FakeResp: + headers = {"content-length": str(size)} + def raise_for_status(self): pass + async def aiter_bytes(self, _n): + yield original + + class FakeStream: + async def __aenter__(self): return FakeResp() + async def __aexit__(self, *a): return None + + class FakeClient: + def __init__(self): self.closed = False + def stream(self, *a, **k): return FakeStream() + async def aclose(self): self.closed = True + + class FakeApi: + def __init__(self, *a, **k): + self._client = FakeClient() + async def download_file_raw(self, _u): raise AssertionError + + fake_sdk = MagicMock() + fake_sdk.WeComApiClient = FakeApi + fake_sdk.decrypt_file = lambda d, k: d # no encryption + monkeypatch.setitem(__import__("sys").modules, "wecom_aibot_sdk", fake_sdk) + monkeypatch.setattr(inbound_media, "_media_storage_dir", lambda _a: tmp_path) + + result = await inbound_media.download_inbound_media( + InboundMessage( + channel_id="wecom", account_id="a", message_id="m1", + sender_id="u", media_url="https://example.com/x", + raw={"msgtype": "image", "image": {"aeskey": ""}}, + ), + {}, + ) + assert result is not None + local = Path(result.url.removeprefix("file://")) + assert local.read_bytes() == original + + +# --------------------------------------------------------------------------- +# DingTalk — uses the OAPI; the test patches ``api_request_for_account`` +# and ``_get_http_client`` so no real HTTP fires. +# --------------------------------------------------------------------------- + +class TestDingTalkRoundTrip: + @pytest.mark.asyncio + async def test_download_code_exchange_then_send(self, tmp_path: Path, monkeypatch): + from flocks.channel.builtin.dingtalk import inbound_media, media as out_media + from flocks.channel.builtin.dingtalk.client import _get_http_client + from flocks.channel.builtin.dingtalk.channel import DingTalkChannel + + png_path = tmp_path / "shot.png" + size = make_tiny_png(png_path) + original = png_path.read_bytes() + code = "DC_xyz" + + # Stub the OAPI exchange + async def fake_exchange(*, config, account_id, download_code): + assert download_code == code + return "https://example.com/d.png", "d.png" + monkeypatch.setattr(inbound_media, "_exchange_download_code", fake_exchange) + + # Stub the streaming download + async def fake_stream(_url, _max): + return original, "d.png" + monkeypatch.setattr(inbound_media, "_download_remote_bytes_limited", fake_stream) + monkeypatch.setattr(inbound_media, "_media_storage_dir", lambda _a: tmp_path) + + # 1) Inbound + dl = await inbound_media.download_inbound_media( + InboundMessage( + channel_id="dingtalk", account_id="a", message_id="m1", + sender_id="u", media_url=code, + ), + {}, + ) + assert dl is not None + assert Path(dl.url.removeprefix("file://")).read_bytes() == original + + # 2) Outbound — patch the upload to skip HTTP + async def fake_upload(*, config, account_id, data, filename): + return ("MED_2", "DC_2") + monkeypatch.setattr(out_media, "upload_dingtalk_media", fake_upload) + + prepared = await out_media.prepare_dingtalk_media( + config={"appKey": "ak", "appSecret": "as", "robotCode": "rc"}, + account_id="a", media_url=png_path.as_uri(), + ) + assert prepared.data == original + assert prepared.download_code == "DC_2" + + # 3) send_media integration (patch the OAPI call) + ch = DingTalkChannel() + ch._config = {"appKey": "ak", "appSecret": "as", "robotCode": "rc"} + + async def fake_send_text(*, config, to, text, account_id): + return {"message_id": "txt_1", "chat_id": "u1"} + async def fake_oapi(method, path, *, config, account_id, json_body): + assert json_body["msgKey"] == "file" + return {"processQueryKey": "mid_1"} + + with patch("flocks.channel.builtin.dingtalk.send.send_message_app", side_effect=fake_send_text), \ + patch("flocks.channel.builtin.dingtalk.client.api_request_for_account", side_effect=fake_oapi): + result = await ch.send_media(OutboundContext( + channel_id="dingtalk", to="u1", media_url=png_path.as_uri(), + )) + assert result.success is True + + +# --------------------------------------------------------------------------- +# Telegram — runs against an in-process fake server +# --------------------------------------------------------------------------- + +class TestTelegramRoundTrip: + @pytest.mark.asyncio + async def test_local_file_via_fake_server(self, tmp_path: Path, monkeypatch): + from flocks.channel.builtin.telegram import inbound_media, media as out_media + from flocks.channel.builtin.telegram.channel import TelegramChannel + + png_path = tmp_path / "t.png" + make_tiny_png(png_path) + original = png_path.read_bytes() + + server = _FakeTelegramServer() + # Register a fake file_id → path mapping + file_id = "AgAD_001" + server.file_id_to_path[file_id] = "documents/t.png" + + class FakeHttpxClient: + def __init__(self, srv): + self._srv = srv + async def get(self, url, *, params=None, timeout=None): + return self._srv.get(url, params=params, timeout=timeout) + async def post(self, url, *, data=None, timeout=None, json=None, files=None): + return self._srv.post(url, data=data, json=json, files=files, timeout=timeout) + async def stream(self, *a, **k): + class _CM: + async def __aenter__(inner_self): + return self._srv.get("https://api.telegram.org/file/bot/x") + async def __aexit__(inner_self, *a): return None + return _CM() + + fake_client = FakeHttpxClient(server) + async def fake_get_http_client(): + return fake_client + monkeypatch.setattr( + "flocks.channel.builtin.telegram.channel.get_http_client", + fake_get_http_client, + ) + monkeypatch.setattr( + "flocks.channel.builtin.telegram.inbound_media._media_storage_dir", + lambda _a: tmp_path, + ) + # Force the inbound download to consult our fake get() by short- + # circuiting the SDK's getFile call: monkeypatch _get_file_path + + # _download_file to use the server directly. + async def fake_get_file_path(*, bot_token, api_base, file_id, timeout): + return server.file_id_to_path[file_id], file_id + async def fake_download(*, bot_token, file_path, max_bytes, timeout): + return server.get(f"https://api.telegram.org/file/bot{bot_token}/{file_path}")._body + monkeypatch.setattr(inbound_media, "_get_file_path", fake_get_file_path) + monkeypatch.setattr(inbound_media, "_download_file", fake_download) + + # 1) Inbound + dl = await inbound_media.download_inbound_media( + InboundMessage( + channel_id="telegram", account_id="a", message_id="m1", + sender_id="u", media_url=f"telegram://photo/{file_id}", + ), + config={"botToken": server.token}, + ) + assert dl is not None + assert dl.source["file_id"] == file_id + + # 2) Outbound via the same server + ch = TelegramChannel() + ch._config = {"botToken": server.token} + + result = await ch.send_media(OutboundContext( + channel_id="telegram", to="1", text="hi", media_url=png_path.as_uri(), + )) + assert result.success is True + # Verify the route — PNG must go via sendPhoto + photo_call = [c for c in server.calls if c["url"].endswith("/sendPhoto")] + assert photo_call, f"expected sendPhoto, calls={server.calls}" + + +# --------------------------------------------------------------------------- +# Dispatcher → FilePart pipeline (all channels) +# --------------------------------------------------------------------------- + +class TestDispatcherFilePartPipeline: + @pytest.mark.asyncio + async def test_wecom_full_pipeline(self, monkeypatch, tmp_path): + await _exercise_pipeline(monkeypatch, tmp_path, "wecom", + media_url="https://example.com/x", local_name="wecom_file.pdf", + ) + + @pytest.mark.asyncio + async def test_dingtalk_full_pipeline(self, monkeypatch, tmp_path): + await _exercise_pipeline(monkeypatch, tmp_path, "dingtalk", + media_url="CODE_abc", local_name="dingtalk_img.png", + ) + + @pytest.mark.asyncio + async def test_telegram_full_pipeline(self, monkeypatch, tmp_path): + await _exercise_pipeline(monkeypatch, tmp_path, "telegram", + media_url="telegram://photo/ABC", local_name="telegram_photo.jpg", + ) + + @pytest.mark.asyncio + async def test_feishu_full_pipeline(self, monkeypatch, tmp_path): + await _exercise_pipeline(monkeypatch, tmp_path, "feishu", + media_url="lark://image/img_1", local_name="feishu_image.png", + ) + + +async def _exercise_pipeline( + monkeypatch, tmp_path, channel_id: str, *, + media_url: str, local_name: str, +) -> None: + """Common shape: create_message → channel downloader → FilePart stored.""" + from flocks.session.message import TextPart + + created = MagicMock(id="m1") + store_part = AsyncMock() + monkeypatch.setattr( + "flocks.session.message.Message.create", + AsyncMock(return_value=created), + ) + monkeypatch.setattr( + "flocks.session.message.Message.store_part", + store_part, + ) + + # Pre-existing placeholder text part that the dispatcher should rewrite. + placeholder = "[图片消息]" if channel_id != "feishu" else "[图片]" + monkeypatch.setattr( + "flocks.session.message.Message.parts", + AsyncMock(return_value=[ + TextPart(id="p1", sessionID="s1", messageID="m1", text=placeholder), + ]), + ) + + expected_file_path = (tmp_path / local_name).resolve() + expected_file_path.write_bytes(b"PNGDATA") + expected_uri = expected_file_path.as_uri() + + async def fake_download(msg, config): + return SimpleNamespace( + filename=local_name, mime="image/png", + url=expected_uri, source={"channel": channel_id}, + ) + + # Patch the right module for the channel under test + module_name = f"flocks.channel.builtin.{channel_id}.inbound_media" + mod = __import__(module_name, fromlist=["*"]) + monkeypatch.setattr(mod, "download_inbound_media", fake_download) + + published: list[tuple[str, dict]] = [] + async def fake_publish_event(event, data): + published.append((event, data)) + monkeypatch.setattr( + "flocks.server.routes.event.publish_event", + fake_publish_event, + ) + + from flocks.config.config import ChannelConfig + cfg = None + if channel_id == "wecom": + cfg = ChannelConfig(enabled=True, botId="b", secret="s") + + await InboundDispatcher._append_user_message( + "s1", + placeholder, + InboundMessage( + channel_id=channel_id, account_id="a", message_id="m1", + sender_id="u", media_url=media_url, + ), + cfg, + ) + + # FilePart stored + assert store_part.await_count >= 2 + fp = store_part.await_args_list[0].args[2] + assert fp.type == "file" + assert fp.url == expected_uri + # The rewritten text part + new_text = store_part.await_args_list[1].args[2] + assert new_text.type == "text" + assert "Attached files" in new_text.text + # SSE updates published + assert any(ev == "message.part.updated" for ev, _ in published) diff --git a/tests/channel/test_telegram.py b/tests/channel/test_telegram.py index daf055ba2..1da9d30d1 100644 --- a/tests/channel/test_telegram.py +++ b/tests/channel/test_telegram.py @@ -2,6 +2,7 @@ import asyncio import json +from pathlib import Path from typing import Any import pytest @@ -944,3 +945,463 @@ async def on_message(msg): assert len(pairing_calls) == 1, "Pairing should fire when allowFrom is an empty list" assert dispatched == [], "Message must not be dispatched to AI before pairing" + + +# ------------------------------------------------------------------ +# Inbound media download (getFile + download) +# ------------------------------------------------------------------ + +class TestTelegramInboundMedia: + @pytest.mark.asyncio + async def test_parse_telegram_uri(self): + from flocks.channel.builtin.telegram import inbound_media as mod + kind, file_id = mod._parse_telegram_uri("telegram://photo/AgAD-file") + assert kind == "photo" + assert file_id == "AgAD-file" + + @pytest.mark.asyncio + async def test_invalid_uri_returns_none(self): + from flocks.channel.builtin.telegram import inbound_media as mod + kind, file_id = mod._parse_telegram_uri("https://example.com/x.png") + assert kind is None + assert file_id is None + + @pytest.mark.asyncio + async def test_get_file_then_download(self, monkeypatch, tmp_path): + from flocks.channel.builtin.telegram import inbound_media as mod + + async def fake_get_file_path(*, bot_token, api_base, file_id, timeout): + assert api_base == "https://api.telegram.org/bot123:abc" + return "documents/file_42.pdf", file_id + + async def fake_download_file(*, download_base, file_path, max_bytes, timeout): + assert download_base == "https://api.telegram.org/file/bot123:abc" + return b"%PDF-1.4 hello" + + monkeypatch.setattr(mod, "_get_file_path", fake_get_file_path) + monkeypatch.setattr(mod, "_download_file", fake_download_file) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + from flocks.channel.base import ChatType, InboundMessage + media = await mod.download_inbound_media( + InboundMessage( + channel_id="telegram", + account_id="acc1", + message_id="m1", + sender_id="u1", + chat_id="c1", + chat_type=ChatType.DIRECT, + media_url="telegram://document/ABC", + ), + config={"botToken": "123:abc"}, + max_bytes=1024, + ) + assert media is not None + assert media.filename == "file_42.pdf" + assert media.mime == "application/pdf" + assert Path(media.url.removeprefix("file://")).read_bytes() == b"%PDF-1.4 hello" + assert media.source["file_id"] == "ABC" + assert media.source["kind"] == "document" + + @pytest.mark.asyncio + async def test_too_large_returns_none(self, monkeypatch, tmp_path): + from flocks.channel.builtin.telegram import inbound_media as mod + + async def fake_get_file_path(*, bot_token, api_base, file_id, timeout): + return "documents/big.bin", file_id + + async def fake_download_file(*, download_base, file_path, max_bytes, timeout): + raise mod.TelegramInboundMediaTooLarge("too large") + + monkeypatch.setattr(mod, "_get_file_path", fake_get_file_path) + monkeypatch.setattr(mod, "_download_file", fake_download_file) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + from flocks.channel.base import ChatType, InboundMessage + media = await mod.download_inbound_media( + InboundMessage( + channel_id="telegram", account_id="acc1", + message_id="m2", sender_id="u1", chat_id="c1", + chat_type=ChatType.DIRECT, + media_url="telegram://document/BIG", + ), + config={"botToken": "123:abc"}, + ) + assert media is None + assert list(tmp_path.iterdir()) == [] + + @pytest.mark.asyncio + async def test_no_token_returns_none(self, monkeypatch, tmp_path): + from flocks.channel.builtin.telegram import inbound_media as mod + from flocks.channel.base import ChatType, InboundMessage + media = await mod.download_inbound_media( + InboundMessage( + channel_id="telegram", account_id="acc1", + message_id="m3", sender_id="u1", chat_id="c1", + chat_type=ChatType.DIRECT, + media_url="telegram://photo/X", + ), + config={}, + ) + assert media is None + + @pytest.mark.asyncio + async def test_download_uses_configured_api_root_and_preserves_nested_filename( + self, monkeypatch, tmp_path, + ): + from flocks.channel.builtin.telegram import inbound_media as mod + from flocks.channel.base import ChatType, InboundMessage + + async def fake_get_file_path(*, bot_token, api_base, file_id, timeout): + assert api_base == "https://tg-proxy.example/bot123:abc" + return "documents/file_42.pdf", file_id + + async def fake_download_file(*, download_base, file_path, max_bytes, timeout): + assert download_base == "https://tg-proxy.example/file/bot123:abc" + assert file_path == "documents/file_42.pdf" + return b"%PDF-1.4 proxy" + + monkeypatch.setattr(mod, "_get_file_path", fake_get_file_path) + monkeypatch.setattr(mod, "_download_file", fake_download_file) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="telegram", + account_id="acc1", + message_id="m4", + sender_id="u1", + chat_id="c1", + chat_type=ChatType.DIRECT, + media_url="telegram://document/ABC", + raw={"document": {"file_name": "原始报告.pdf"}}, + ), + config={ + "accounts": { + "acc1": { + "botToken": "123:abc", + "apiRoot": "https://tg-proxy.example", + }, + }, + }, + ) + assert media is not None + assert media.filename == "原始报告.pdf" + + @pytest.mark.asyncio + async def test_download_supports_legacy_api_base_with_token( + self, monkeypatch, tmp_path, + ): + from flocks.channel.builtin.telegram import inbound_media as mod + from flocks.channel.base import ChatType, InboundMessage + + async def fake_get_file_path(*, bot_token, api_base, file_id, timeout): + assert api_base == "https://legacy.example/bot123:abc" + return "documents/file_7.pdf", file_id + + async def fake_download_file(*, download_base, file_path, max_bytes, timeout): + assert download_base == "https://legacy.example/file/bot123:abc" + return b"legacy" + + monkeypatch.setattr(mod, "_get_file_path", fake_get_file_path) + monkeypatch.setattr(mod, "_download_file", fake_download_file) + monkeypatch.setattr(mod, "_media_storage_dir", lambda _acc: tmp_path) + + media = await mod.download_inbound_media( + InboundMessage( + channel_id="telegram", + account_id="default", + message_id="m5", + sender_id="u1", + chat_id="c1", + chat_type=ChatType.DIRECT, + media_url="telegram://document/ABC", + ), + config={"botToken": "123:abc", "apiBase": "https://legacy.example/bot123:abc"}, + ) + assert media is not None + + +# ------------------------------------------------------------------ +# Outbound media (prepare + send_media) +# ------------------------------------------------------------------ + +class TestTelegramSendMedia: + @pytest.mark.asyncio + async def test_prepare_local_image(self, tmp_path): + from flocks.channel.builtin.telegram.media import ( + prepare_telegram_media, + ) + path = tmp_path / "photo.png" + path.write_bytes(b"\x89PNG\r\n\x1a\nDATA") + prepared = await prepare_telegram_media(path.as_uri()) + assert prepared.kind == "photo" + assert prepared.filename == "photo.png" + assert prepared.data == b"\x89PNG\r\n\x1a\nDATA" + + @pytest.mark.asyncio + async def test_prepare_local_pdf_uses_document(self, tmp_path): + from flocks.channel.builtin.telegram.media import ( + prepare_telegram_media, + ) + path = tmp_path / "doc.pdf" + path.write_bytes(b"%PDF-1.4 data") + prepared = await prepare_telegram_media(path.as_uri()) + assert prepared.kind == "document" + assert prepared.mime == "application/pdf" + + @pytest.mark.asyncio + async def test_prepare_gif_uses_animation(self, tmp_path): + from flocks.channel.builtin.telegram.media import ( + prepare_telegram_media, + ) + path = tmp_path / "anim.gif" + path.write_bytes(b"GIF89a-data") + prepared = await prepare_telegram_media(path.as_uri()) + assert prepared.kind == "animation" + + @pytest.mark.asyncio + async def test_prepare_ogg_uses_voice(self, tmp_path): + from flocks.channel.builtin.telegram.media import ( + prepare_telegram_media, + ) + path = tmp_path / "clip.ogg" + path.write_bytes(b"OggS-data") + prepared = await prepare_telegram_media(path.as_uri()) + assert prepared.kind == "voice" + + @pytest.mark.asyncio + async def test_kind_override(self, tmp_path): + from flocks.channel.builtin.telegram.media import ( + prepare_telegram_media, + ) + path = tmp_path / "img.png" + path.write_bytes(b"\x89PNG") + prepared = await prepare_telegram_media( + path.as_uri(), kind_override="document", + ) + assert prepared.kind == "document" + + @pytest.mark.asyncio + async def test_send_media_routes_to_send_photo(self, tmp_path, monkeypatch): + from flocks.channel.builtin.telegram.channel import TelegramChannel + from flocks.channel.builtin.telegram import media as media_mod + + path = tmp_path / "photo.jpg" + path.write_bytes(b"\xff\xd8\xff-data") + + ch = TelegramChannel() + ch._config = {"botToken": "123:abc"} + + async def fake_prepare(media_url, *, kind_override=None, **kwargs): + return media_mod.PreparedTelegramMedia( + data=b"\xff\xd8\xff-data", filename="photo.jpg", + mime="image/jpeg", kind="photo", + ) + + posted = [] + + class FakeResponse: + status_code = 200 + is_success = True + def json(self): + return {"ok": True, "result": {"message_id": "111", "chat": {"id": "c1"}}} + + class FakeClient: + async def post(self, url, *, data, files, timeout): + posted.append({"url": url, "data": data, "files": files}) + return FakeResponse() + + from unittest.mock import AsyncMock + fake_client = FakeClient() + monkeypatch.setattr( + "flocks.channel.builtin.telegram.channel.get_http_client", + AsyncMock(return_value=fake_client), + ) + monkeypatch.setattr(media_mod, "prepare_telegram_media", fake_prepare) + + result = await ch.send_media( + OutboundContext( + channel_id="telegram", to="c1", + text="look", media_url=path.as_uri(), + ) + ) + assert result.success is True + assert result.message_id == "111" + assert posted[0]["url"].endswith("/sendPhoto") + assert "photo" in posted[0]["files"] + assert posted[0]["files"]["photo"][0] == "photo.jpg" + assert posted[0]["data"]["caption"] == "look" + + @pytest.mark.asyncio + async def test_send_media_routes_to_send_document_for_pdf( + self, tmp_path, monkeypatch, + ): + from flocks.channel.builtin.telegram.channel import TelegramChannel + from flocks.channel.builtin.telegram import media as media_mod + + path = tmp_path / "doc.pdf" + path.write_bytes(b"%PDF-data") + + ch = TelegramChannel() + ch._config = {"botToken": "123:abc"} + + async def fake_prepare(media_url, *, kind_override=None, **kwargs): + return media_mod.PreparedTelegramMedia( + data=b"%PDF-data", filename="doc.pdf", + mime="application/pdf", kind="document", + ) + + posted = [] + + class FakeResponse: + status_code = 200 + is_success = True + def json(self): + return {"ok": True, "result": {"message_id": "222", "chat": {"id": "c1"}}} + + class FakeClient: + async def post(self, url, *, data, files, timeout): + posted.append({"url": url, "data": data, "files": files}) + return FakeResponse() + + from unittest.mock import AsyncMock + fake_client = FakeClient() + monkeypatch.setattr( + "flocks.channel.builtin.telegram.channel.get_http_client", + AsyncMock(return_value=fake_client), + ) + monkeypatch.setattr(media_mod, "prepare_telegram_media", fake_prepare) + + result = await ch.send_media( + OutboundContext( + channel_id="telegram", to="c1", + media_url=path.as_uri(), + ) + ) + assert result.success is True + assert posted[0]["url"].endswith("/sendDocument") + assert posted[0]["files"]["document"][0] == "doc.pdf" + + @pytest.mark.asyncio + async def test_send_media_kind_override_prefix( + self, tmp_path, monkeypatch, + ): + from flocks.channel.builtin.telegram.channel import TelegramChannel + from flocks.channel.builtin.telegram import media as media_mod + + path = tmp_path / "img.png" + path.write_bytes(b"x") + + ch = TelegramChannel() + ch._config = {"botToken": "123:abc"} + + captured = {} + + async def fake_prepare(media_url, *, kind_override=None, **kwargs): + captured["kind"] = kind_override + captured["source"] = media_url + return media_mod.PreparedTelegramMedia( + data=b"x", filename="img.png", mime="image/png", + kind=kind_override or "photo", + ) + + class FakeResponse: + status_code = 200 + is_success = True + def json(self): + return {"ok": True, "result": {"message_id": "333", "chat": {"id": "c1"}}} + + class FakeClient: + async def post(self, url, *, data, files, timeout): + return FakeResponse() + + from unittest.mock import AsyncMock + fake_client = FakeClient() + monkeypatch.setattr( + "flocks.channel.builtin.telegram.channel.get_http_client", + AsyncMock(return_value=fake_client), + ) + monkeypatch.setattr(media_mod, "prepare_telegram_media", fake_prepare) + + result = await ch.send_media( + OutboundContext( + channel_id="telegram", to="c1", + media_url=f"telegram:document:{path.as_uri()}", + ) + ) + assert result.success is True + assert captured["kind"] == "document" + # The prefix should be stripped before passing to the preparer. + assert captured["source"] == path.as_uri() + + @pytest.mark.asyncio + async def test_send_media_api_error_returns_failure( + self, tmp_path, monkeypatch, + ): + from flocks.channel.builtin.telegram.channel import TelegramChannel + from flocks.channel.builtin.telegram import media as media_mod + + path = tmp_path / "doc.pdf" + path.write_bytes(b"x") + + ch = TelegramChannel() + ch._config = {"botToken": "123:abc"} + + async def fake_prepare(media_url, *, kind_override=None, **kwargs): + return media_mod.PreparedTelegramMedia( + data=b"x", filename="doc.pdf", + mime="application/pdf", kind="document", + ) + + class FakeResponse: + status_code = 400 + is_success = False + def json(self): + return {"ok": False, "description": "bad chat id"} + + class FakeClient: + async def post(self, url, *, data, files, timeout): + return FakeResponse() + + from unittest.mock import AsyncMock + fake_client = FakeClient() + monkeypatch.setattr( + "flocks.channel.builtin.telegram.channel.get_http_client", + AsyncMock(return_value=fake_client), + ) + monkeypatch.setattr(media_mod, "prepare_telegram_media", fake_prepare) + + result = await ch.send_media( + OutboundContext( + channel_id="telegram", to="c1", + media_url=path.as_uri(), + ) + ) + assert result.success is False + assert "bad chat id" in result.error + + @pytest.mark.asyncio + async def test_send_media_missing_media_url_falls_back_to_text( + self, monkeypatch, + ): + from flocks.channel.builtin.telegram.channel import TelegramChannel + + ch = TelegramChannel() + ch._config = {"botToken": "123:abc"} + + async def fake_send_text(self, ctx): + from flocks.channel.base import DeliveryResult + return DeliveryResult( + channel_id="telegram", message_id="t1", success=True, + ) + + monkeypatch.setattr( + TelegramChannel, "send_text", fake_send_text, + ) + + result = await ch.send_media( + OutboundContext(channel_id="telegram", to="c1", text="hi"), + ) + assert result.success is True + assert result.message_id == "t1" diff --git a/tests/channel/test_wecom.py b/tests/channel/test_wecom.py index a7a363a84..b1978c347 100644 --- a/tests/channel/test_wecom.py +++ b/tests/channel/test_wecom.py @@ -9,6 +9,9 @@ from __future__ import annotations import asyncio +import sys +import types +from pathlib import Path from unittest.mock import AsyncMock, patch import pytest @@ -16,6 +19,7 @@ from flocks.channel.base import ( ChatType, DeliveryResult, + InboundMessage, OutboundContext, ) from flocks.channel.builtin.wecom.channel import ( @@ -25,6 +29,10 @@ _extract_mixed, _parse_frame, ) +from flocks.channel.builtin.wecom.inbound_media import ( + _filename_from_content_disposition, + download_inbound_media, +) # ------------------------------------------------------------------ @@ -172,6 +180,202 @@ async def test_send_text_exception(self): assert "timeout" in result.error +class TestWeComSendMedia: + async def test_send_media_uploads_and_sends_file(self, tmp_path: Path): + path = tmp_path / "report.pdf" + path.write_bytes(b"pdf-data") + + ch = WeComChannel() + ch._config = {"botId": "b", "secret": "s"} + ch._ws_client = AsyncMock() + ch._ws_client.upload_media = AsyncMock( + return_value={"media_id": "media_1", "type": "file"}, + ) + ch._ws_client.send_media_message = AsyncMock( + return_value={"body": {"msgid": "wx_msg_1"}}, + ) + + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + media_url=path.as_uri(), + ) + ) + + assert result.success is True + assert result.message_id == "wx_msg_1" + ch._ws_client.upload_media.assert_awaited_once() + upload_args = ch._ws_client.upload_media.await_args + assert upload_args.args[0] == b"pdf-data" + assert upload_args.kwargs["type"] == "file" + assert upload_args.kwargs["filename"] == "report.pdf" + ch._ws_client.send_media_message.assert_awaited_once_with( + "zhangsan", + "file", + "media_1", + video_title=None, + ) + + async def test_send_media_replies_with_cached_frame(self, tmp_path: Path): + path = tmp_path / "image.png" + path.write_bytes(b"png-data") + + ch = WeComChannel() + ch._config = {"botId": "b", "secret": "s"} + ch._ws_client = AsyncMock() + ch._ws_client.upload_media = AsyncMock( + return_value={"media_id": "media_img", "type": "image"}, + ) + ch._ws_client.reply_media = AsyncMock( + return_value={"body": {"msgid": "wx_reply_1"}}, + ) + frame = {"body": {"msgid": "incoming_1"}, "headers": {"req_id": "req_1"}} + ch._cache_frame("incoming_1", frame) + + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + media_url=path.as_uri(), + reply_to_id="incoming_1", + ) + ) + + assert result.success is True + assert result.message_id == "wx_reply_1" + ch._ws_client.reply_media.assert_awaited_once_with( + frame, + "image", + "media_img", + video_title=None, + ) + assert ch._frame_cache.get("incoming_1") is None + + async def test_send_media_not_connected(self, tmp_path: Path): + path = tmp_path / "report.pdf" + path.write_bytes(b"pdf-data") + + ch = WeComChannel() + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + media_url=path.as_uri(), + ) + ) + + assert result.success is False + assert "not connected" in result.error.lower() + + async def test_send_media_upload_missing_media_id(self, tmp_path: Path): + path = tmp_path / "report.pdf" + path.write_bytes(b"pdf-data") + + ch = WeComChannel() + ch._config = {"botId": "b", "secret": "s"} + ch._ws_client = AsyncMock() + ch._ws_client.upload_media = AsyncMock(return_value={"type": "file"}) + + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + media_url=path.as_uri(), + ) + ) + + assert result.success is False + assert "media upload failed" in result.error + ch._ws_client.send_media_message.assert_not_awaited() + + async def test_send_media_upload_exception(self, tmp_path: Path): + path = tmp_path / "report.pdf" + path.write_bytes(b"pdf-data") + + ch = WeComChannel() + ch._config = {"botId": "b", "secret": "s"} + ch._ws_client = AsyncMock() + ch._ws_client.upload_media = AsyncMock(side_effect=RuntimeError("timeout")) + + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + media_url=path.as_uri(), + ) + ) + + assert result.success is False + assert result.retryable is True + assert "timeout" in result.error + + async def test_send_media_with_text_sends_text_after_media(self, tmp_path: Path): + path = tmp_path / "report.pdf" + path.write_bytes(b"pdf-data") + + ch = WeComChannel() + ch._config = {"botId": "b", "secret": "s"} + ch._ws_client = AsyncMock() + ch._ws_client.upload_media = AsyncMock( + return_value={"media_id": "media_1", "type": "file"}, + ) + ch._ws_client.send_media_message = AsyncMock( + return_value={"body": {"msgid": "wx_media_1"}}, + ) + ch._ws_client.send_message = AsyncMock( + return_value={"body": {"msgid": "wx_text_1"}}, + ) + + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + text="这是附件说明", + media_url=path.as_uri(), + ) + ) + + assert result.success is True + assert result.message_id == "wx_media_1" + ch._ws_client.send_media_message.assert_awaited_once() + ch._ws_client.send_message.assert_awaited_once_with( + "zhangsan", + {"msgtype": "markdown", "markdown": {"content": "这是附件说明"}}, + ) + + async def test_send_media_with_text_reports_caption_failure(self, tmp_path: Path): + path = tmp_path / "report.pdf" + path.write_bytes(b"pdf-data") + + ch = WeComChannel() + ch._config = {"botId": "b", "secret": "s"} + ch._ws_client = AsyncMock() + ch._ws_client.upload_media = AsyncMock( + return_value={"media_id": "media_1", "type": "file"}, + ) + ch._ws_client.send_media_message = AsyncMock( + return_value={"body": {"msgid": "wx_media_1"}}, + ) + ch._ws_client.send_message = AsyncMock( + side_effect=RuntimeError("timeout"), + ) + + result = await ch.send_media( + OutboundContext( + channel_id="wecom", + to="zhangsan", + text="这是附件说明", + media_url=path.as_uri(), + ) + ) + + assert result.success is False + assert result.message_id == "wx_media_1" + assert result.retryable is True + assert "caption failed" in result.error + + # ------------------------------------------------------------------ # reconnect watchdog # ------------------------------------------------------------------ @@ -409,6 +613,21 @@ def test_file_message(self): "chattype": "single", "from": {"userid": "u1"}, "msgtype": "file", + "file": {"url": "https://example.com/file.pdf", "filename": "report.pdf"}, + } + } + msg = _parse_frame(frame, {}) + assert msg is not None + assert msg.text == "[文件消息: report.pdf]" + assert msg.media_url == "https://example.com/file.pdf" + + def test_file_message_no_filename(self): + frame = { + "body": { + "msgid": "msg005b", + "chattype": "single", + "from": {"userid": "u1"}, + "msgtype": "file", "file": {"url": "https://example.com/file.pdf"}, } } @@ -437,6 +656,34 @@ def test_mixed_message(self): assert "看这个" in msg.text assert "[图片]" in msg.text + def test_mixed_file_message(self): + frame = { + "body": { + "msgid": "msg006b", + "chattype": "single", + "from": {"userid": "u1"}, + "msgtype": "mixed", + "mixed": { + "msg_item": [ + {"msgtype": "text", "text": {"content": "看附件"}}, + { + "msgtype": "file", + "file": { + "url": "https://example.com/file.bin", + "filename": "report.bin", + "aeskey": "k1", + }, + }, + ] + }, + } + } + msg = _parse_frame(frame, {}) + assert msg is not None + assert "看附件" in msg.text + assert "[文件: report.bin]" in msg.text + assert msg.media_url == "https://example.com/file.bin" + def test_stream_type_ignored(self): frame = { "body": { @@ -487,16 +734,43 @@ def test_unknown_type(self): class TestExtractMixed: def test_mixed_text_and_image(self): - result = _extract_mixed({ + result_text, result_media = _extract_mixed({ "msg_item": [ {"msgtype": "text", "text": {"content": "A"}}, {"msgtype": "image", "image": {"url": "https://x.com/b.png"}}, {"msgtype": "text", "text": {"content": "B"}}, ] }) - assert "A" in result - assert "[图片]" in result - assert "B" in result + assert "A" in result_text + assert "[图片]" in result_text + assert "B" in result_text + assert result_media == "https://x.com/b.png" + + def test_mixed_no_image(self): + result_text, result_media = _extract_mixed({ + "msg_item": [ + {"msgtype": "text", "text": {"content": "hello"}}, + ] + }) + assert result_text == "hello" + assert result_media is None + + def test_mixed_text_and_file(self): + result_text, result_media = _extract_mixed({ + "msg_item": [ + {"msgtype": "text", "text": {"content": "A"}}, + { + "msgtype": "file", + "file": { + "url": "https://x.com/report.pdf", + "filename": "report.pdf", + }, + }, + ] + }) + assert "A" in result_text + assert "[文件: report.pdf]" in result_text + assert result_media == "https://x.com/report.pdf" # ------------------------------------------------------------------ @@ -537,3 +811,336 @@ def test_wecom_alias_wxwork(self): reg = ChannelRegistry() reg._register_builtin_channels() assert reg.get("wxwork") is not None + + +# ------------------------------------------------------------------ +# Content-Disposition filename parsing +# ------------------------------------------------------------------ + +class TestContentDispositionFilename: + def test_filename_from_content_disposition_plain(self): + filename = _filename_from_content_disposition( + 'attachment; filename="report.pdf"', + ) + assert filename == "report.pdf" + + def test_filename_from_content_disposition_utf8(self): + filename = _filename_from_content_disposition( + "attachment; filename*=UTF-8''%E6%8A%A5%E5%91%8A.pdf", + ) + assert filename == "报告.pdf" + + +# ------------------------------------------------------------------ +# Inbound media download (decrypt + size guard) +# ------------------------------------------------------------------ + +class TestWeComInboundMedia: + @pytest.mark.asyncio + async def test_download_inbound_media_streams_decrypts_and_closes( + self, + monkeypatch, + tmp_path: Path, + ): + class FakeResponse: + headers = { + "content-disposition": 'attachment; filename="from-header.bin"', + } + + def raise_for_status(self): + return None + + async def aiter_bytes(self, _size): + yield b"hello" + + class FakeStream: + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, *_args): + return None + + class FakeClient: + def __init__(self): + self.closed = False + + def stream(self, method, url): + assert method == "GET" + assert url == "https://example.com/file.bin" + return FakeStream() + + async def aclose(self): + self.closed = True + + class FakeWeComApiClient: + last_instance = None + + def __init__(self, *_args, **_kwargs): + self._client = FakeClient() + FakeWeComApiClient.last_instance = self + + async def download_file_raw(self, _url): + raise AssertionError("stream path should be used") + + fake_sdk = types.SimpleNamespace( + WeComApiClient=FakeWeComApiClient, + decrypt_file=lambda data, key: data + key.encode(), + ) + monkeypatch.setitem(sys.modules, "wecom_aibot_sdk", fake_sdk) + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media._media_storage_dir", + lambda _account_id: tmp_path, + ) + + media = await download_inbound_media( + InboundMessage( + channel_id="wecom", + account_id="main", + message_id="msg_1", + sender_id="u1", + media_url="https://example.com/file.bin", + raw={ + "msgtype": "file", + "file": { + "filename": "../report.bin", + "aeskey": "k1", + }, + }, + ), + {}, + max_bytes=20, + ) + + assert media is not None + assert media.filename == ".._report.bin" + assert media.mime == "application/octet-stream" + assert Path(media.url.removeprefix("file://")).read_bytes() == b"hellok1" + assert FakeWeComApiClient.last_instance._client.closed is True + + @pytest.mark.asyncio + async def test_download_inbound_media_rejects_large_content_length( + self, + monkeypatch, + tmp_path: Path, + ): + class FakeResponse: + headers = {"content-length": "30"} + + def raise_for_status(self): + return None + + async def aiter_bytes(self, _size): + yield b"too-large" + + class FakeStream: + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, *_args): + return None + + class FakeClient: + def __init__(self): + self.closed = False + + def stream(self, *_args, **_kwargs): + return FakeStream() + + async def aclose(self): + self.closed = True + + class FakeWeComApiClient: + last_instance = None + + def __init__(self, *_args, **_kwargs): + self._client = FakeClient() + FakeWeComApiClient.last_instance = self + + fake_sdk = types.SimpleNamespace( + WeComApiClient=FakeWeComApiClient, + decrypt_file=lambda data, _key: data, + ) + warnings = [] + monkeypatch.setitem(sys.modules, "wecom_aibot_sdk", fake_sdk) + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media._media_storage_dir", + lambda _account_id: tmp_path, + ) + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media.log.warning", + lambda event, data=None: warnings.append((event, data or {})), + ) + + media = await download_inbound_media( + InboundMessage( + channel_id="wecom", + account_id="main", + message_id="msg_2", + sender_id="u1", + media_url="https://example.com/big.bin", + raw={"msgtype": "file", "file": {"filename": "big.bin"}}, + ), + {}, + max_bytes=10, + ) + + assert media is None + assert list(tmp_path.iterdir()) == [] + assert FakeWeComApiClient.last_instance._client.closed is True + assert warnings[0][0] == "wecom.media.file_too_large" + + @pytest.mark.asyncio + async def test_download_inbound_media_decrypt_failure_returns_none( + self, + monkeypatch, + tmp_path: Path, + ): + class FakeResponse: + headers = {} + + def raise_for_status(self): + return None + + async def aiter_bytes(self, _size): + yield b"encrypted" + + class FakeStream: + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, *_args): + return None + + class FakeClient: + def __init__(self): + self.closed = False + + def stream(self, *_args, **_kwargs): + return FakeStream() + + async def aclose(self): + self.closed = True + + class FakeWeComApiClient: + last_instance = None + + def __init__(self, *_args, **_kwargs): + self._client = FakeClient() + FakeWeComApiClient.last_instance = self + + def fail_decrypt(_data, _key): + raise RuntimeError("bad aes key") + + fake_sdk = types.SimpleNamespace( + WeComApiClient=FakeWeComApiClient, + decrypt_file=fail_decrypt, + ) + warnings = [] + monkeypatch.setitem(sys.modules, "wecom_aibot_sdk", fake_sdk) + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media._media_storage_dir", + lambda _account_id: tmp_path, + ) + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media.log.warning", + lambda event, data=None: warnings.append((event, data or {})), + ) + + media = await download_inbound_media( + InboundMessage( + channel_id="wecom", + account_id="main", + message_id="msg_decrypt", + sender_id="u1", + media_url="https://example.com/file.bin", + raw={"msgtype": "file", "file": {"filename": "file.bin", "aeskey": "bad"}}, + ), + {}, + max_bytes=20, + ) + + assert media is None + assert list(tmp_path.iterdir()) == [] + assert FakeWeComApiClient.last_instance._client.closed is True + assert warnings[0][0] == "wecom.media.decrypt_failed" + + @pytest.mark.asyncio + async def test_download_inbound_media_mixed_file_uses_nested_aeskey( + self, + monkeypatch, + tmp_path: Path, + ): + class FakeResponse: + headers = {} + + def raise_for_status(self): + return None + + async def aiter_bytes(self, _size): + yield b"encrypted" + + class FakeStream: + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, *_args): + return None + + class FakeClient: + def stream(self, *_args, **_kwargs): + return FakeStream() + + async def aclose(self): + return None + + class FakeWeComApiClient: + def __init__(self, *_args, **_kwargs): + self._client = FakeClient() + + captured = {} + + def decrypt(data, key): + captured["key"] = key + return data + b"-ok" + + fake_sdk = types.SimpleNamespace( + WeComApiClient=FakeWeComApiClient, + decrypt_file=decrypt, + ) + monkeypatch.setitem(sys.modules, "wecom_aibot_sdk", fake_sdk) + monkeypatch.setattr( + "flocks.channel.builtin.wecom.inbound_media._media_storage_dir", + lambda _account_id: tmp_path, + ) + + media = await download_inbound_media( + InboundMessage( + channel_id="wecom", + account_id="main", + message_id="msg_mixed", + sender_id="u1", + media_url="https://example.com/file.bin", + raw={ + "msgtype": "mixed", + "mixed": { + "msg_item": [ + {"msgtype": "text", "text": {"content": "见附件"}}, + { + "msgtype": "file", + "file": { + "filename": "nested.bin", + "aeskey": "nested-key", + }, + }, + ] + }, + }, + ), + {}, + max_bytes=20, + ) + + assert media is not None + assert captured["key"] == "nested-key" + assert media.filename == "nested.bin" + assert Path(media.url.removeprefix("file://")).read_bytes() == b"encrypted-ok" diff --git a/tests/cli/test_service_manager.py b/tests/cli/test_service_manager.py index 86747f900..c8e07d8df 100644 --- a/tests/cli/test_service_manager.py +++ b/tests/cli/test_service_manager.py @@ -361,6 +361,30 @@ def test_port_is_in_use_falls_back_to_bind_when_pid_lookup_unavailable(monkeypat assert service_manager.port_is_in_use(5173) is True +def test_bind_port_available_checks_all_ipv4_interfaces(monkeypatch) -> None: + binds: list[tuple[str, int]] = [] + sockopts: list[tuple[object, ...]] = [] + + class FakeSocket: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def setsockopt(self, *args) -> None: + sockopts.append(args) + + def bind(self, address: tuple[str, int]) -> None: + binds.append(address) + + monkeypatch.setattr(service_manager.socket, "socket", lambda *_args, **_kwargs: FakeSocket()) + + assert service_manager._bind_port_available(8000) is True + assert binds == [("0.0.0.0", 8000)] + assert sockopts == [] + + def test_wait_for_http_rejects_unreachable_responses(monkeypatch) -> None: responses = iter([ httpx.Response(503, json={"detail": "not found"}), @@ -1357,7 +1381,7 @@ def fake_popen(*args, **kwargs): assert "startupinfo" not in captured["kwargs"] -def test_spawn_process_rotates_large_log_before_append(monkeypatch, tmp_path: Path) -> None: +def test_spawn_process_appends_without_rotated_suffix(monkeypatch, tmp_path: Path) -> None: log_path = tmp_path / "logs" / "backend.log" log_path.parent.mkdir(parents=True) log_path.write_text("x" * 20, encoding="utf-8") @@ -1367,15 +1391,13 @@ def fake_popen(*args, **kwargs): kwargs["stdout"].flush() return SimpleNamespace(pid=9876) - monkeypatch.setenv("FLOCKS_LOG_MAX_BYTES", "10") - monkeypatch.setenv("FLOCKS_LOG_BACKUP_COUNT", "1") monkeypatch.setattr(service_manager.sys, "platform", "darwin") monkeypatch.setattr(service_manager.subprocess, "Popen", fake_popen) service_manager._spawn_process(["python", "-m", "uvicorn"], cwd=tmp_path, log_path=log_path) - assert log_path.read_text(encoding="utf-8") == "new\n" - assert (tmp_path / "logs" / "backend.log.1").read_text(encoding="utf-8") == "x" * 20 + assert log_path.read_text(encoding="utf-8") == "x" * 20 + "new\n" + assert not (tmp_path / "logs" / "backend.log.1").exists() def test_spawn_process_passes_custom_environment(monkeypatch, tmp_path: Path) -> None: diff --git a/tests/cli/test_skill_command.py b/tests/cli/test_skill_command.py new file mode 100644 index 000000000..d57d406f1 --- /dev/null +++ b/tests/cli/test_skill_command.py @@ -0,0 +1,41 @@ +from unittest.mock import AsyncMock, patch + +from typer.testing import CliRunner + +from flocks.cli.commands.skill import skill_app +from flocks.skill.installer import DepInstallResult +from flocks.skill.skill import SkillInfo + + +runner = CliRunner() + + +def test_status_lists_skills_without_requirements(): + skill = SkillInfo( + name="cli-local-skill", + description="CLI local install test", + location="/tmp/cli-local-skill/SKILL.md", + ) + + with patch("flocks.cli.commands.skill.Skill.all", AsyncMock(return_value=[skill])): + result = runner.invoke(skill_app, ["status"]) + + assert result.exit_code == 0 + assert "cli-local-skill" in result.output + assert "no requirements" in result.output + + +def test_install_deps_prints_result_message_when_no_command(): + result = DepInstallResult( + success=True, + message="Skill 'demo' has no install specs.", + ) + + with patch( + "flocks.cli.commands.skill.SkillInstaller.install_deps", + AsyncMock(return_value=[result]), + ): + cli_result = runner.invoke(skill_app, ["install-deps", "demo"]) + + assert cli_result.exit_code == 0 + assert "has no install specs" in cli_result.output diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 9650f8a21..ed21568f6 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -6,7 +6,7 @@ import json from pathlib import Path -from flocks.config.config import Config, GlobalConfig, ConfigInfo +from flocks.config.config import Config, GlobalConfig, ConfigInfo, PermissionAction, PermissionConfig @pytest.fixture(autouse=True) @@ -70,6 +70,47 @@ def test_local_mcp_config_accepts_legacy_env_alias(): } +def test_legacy_todo_permission_names_migrate_to_todo(): + permission = PermissionConfig.model_validate({ + "todowrite": "deny", + "todoread": "allow", + "bash": "ask", + }) + + dumped = permission.model_dump(exclude_none=True) + assert dumped["todo"] == PermissionAction.DENY + assert "todowrite" not in dumped + assert "todoread" not in dumped + assert dumped["bash"] == PermissionAction.ASK + + +def test_legacy_todo_tool_flags_migrate_to_todo_permission(): + config = ConfigInfo.model_validate({ + "tools": { + "todowrite": False, + "todoread": True, + }, + "agent": { + "worker": { + "tools": { + "todowrite": False, + "bash": True, + }, + }, + }, + }) + + assert isinstance(config.permission, dict) + assert config.permission["todo"] == PermissionAction.DENY + assert "todowrite" not in config.permission + assert "todoread" not in config.permission + + worker = config.agent["worker"] + assert isinstance(worker.permission, dict) + assert worker.permission["todo"] == PermissionAction.DENY + assert worker.permission["bash"] == PermissionAction.ALLOW + + @pytest.mark.asyncio async def test_config_file_loading(tmp_path): """Test loading configuration from file""" diff --git a/tests/ingest/test_kafka_manager.py b/tests/ingest/test_kafka_manager.py index bdf21378f..4fe838823 100644 --- a/tests/ingest/test_kafka_manager.py +++ b/tests/ingest/test_kafka_manager.py @@ -21,6 +21,7 @@ import pytest from flocks.ingest.kafka import manager as kafka_manager +from flocks.workflow.triggers.models import TriggerDefinition @pytest.mark.asyncio @@ -29,13 +30,16 @@ async def test_worker_pool_bounds_in_flight_dispatches(monkeypatch: pytest.Monke manager = kafka_manager.KafkaManager() pool_size = kafka_manager._MAX_CONCURRENT_EXECUTIONS + trigger = TriggerDefinition.model_validate( + {"id": "kafka-default", "type": "kafka", "mapping": {"kafka_message": "$.body"}} + ) in_flight = 0 max_in_flight = 0 completed = 0 lock = asyncio.Lock() - async def _fake_trigger(workflow_id, workflow_json, msg, input_key, producer=None, output_topic=""): # noqa: ANN001 + async def _fake_trigger(workflow_id, workflow_json, msg, input_key, producer=None, output_topic="", **kwargs): # noqa: ANN001 nonlocal in_flight, max_in_flight, completed async with lock: in_flight += 1 @@ -56,7 +60,7 @@ async def _fake_trigger(workflow_id, workflow_json, msg, input_key, producer=Non manager._abort_events[workflow_id] = abort workers = [ asyncio.create_task( - manager._worker_loop(workflow_id, {}, "kafka_message", {}, queue, abort), + manager._worker_loop(workflow_id, {}, trigger, {}, queue, abort, "topic-a"), name=f"test-worker-{i}", ) for i in range(pool_size) @@ -92,8 +96,11 @@ async def test_worker_decodes_queued_raw_message(monkeypatch: pytest.MonkeyPatch queue: asyncio.Queue = asyncio.Queue(maxsize=8) abort = asyncio.Event() captured: list[dict] = [] + trigger = TriggerDefinition.model_validate( + {"id": "kafka-default", "type": "kafka", "mapping": {"kafka_message": "$.body"}} + ) - async def _fake_trigger(workflow_id, workflow_json, msg, input_key, producer=None, output_topic=""): # noqa: ANN001 + async def _fake_trigger(workflow_id, workflow_json, msg, input_key, producer=None, output_topic="", **kwargs): # noqa: ANN001 captured.append(msg) abort.set() @@ -106,7 +113,7 @@ async def _fake_trigger(workflow_id, workflow_json, msg, input_key, producer=Non ) worker = asyncio.create_task( - manager._worker_loop(workflow_id, {}, "kafka_message", {}, queue, abort), + manager._worker_loop(workflow_id, {}, trigger, {}, queue, abort, "topic-a"), name="test-worker-raw-queue", ) await asyncio.wait_for(worker, timeout=1.0) @@ -122,6 +129,9 @@ async def test_stop_workflow_cancels_worker_pool() -> None: workflow_id = "test-wf-stop" queue: asyncio.Queue = asyncio.Queue(maxsize=8) abort = asyncio.Event() + trigger = TriggerDefinition.model_validate( + {"id": "kafka-default", "type": "kafka", "mapping": {"kafka_message": "$.body"}} + ) manager._queues[workflow_id] = queue manager._abort_events[workflow_id] = abort manager._status[workflow_id] = {"state": "running", "error": None} @@ -133,7 +143,7 @@ async def _noop_trigger(*args, **kwargs): # noqa: ANN001 workers = [ asyncio.create_task( - manager._worker_loop(workflow_id, {}, "kafka_message", {}, queue, abort), + manager._worker_loop(workflow_id, {}, trigger, {}, queue, abort, "topic-a"), name=f"stop-worker-{i}", ) for i in range(3) @@ -347,13 +357,82 @@ def _fake_run_workflow(**kwargs): # noqa: ANN003 }, ) - assert captured_run_kwargs["inputs"] == { - "kafka_message": {"alarmData": {"id": 1}}, - "kafka_output_enabled": True, - "kafka_output_topic": "topic_soc_flocks_result_log", - } + assert captured_run_kwargs["inputs"]["kafka_message"] == {"alarmData": {"id": 1}} + assert captured_run_kwargs["inputs"]["kafka_output_enabled"] is True + assert captured_run_kwargs["inputs"]["kafka_output_topic"] == "topic_soc_flocks_result_log" + assert captured_run_kwargs["inputs"]["_trigger"] == "kafka" + assert captured_run_kwargs["inputs"]["_flocks"]["trigger"]["id"] == "kafka-default" assert recorded_input_params["_trigger"] == "kafka" assert recorded_input_params["kafka_output_enabled"] is True assert recorded_input_params["kafka_output_topic"] == "topic_soc_flocks_result_log" assert recorded_input_params["kafka_message"]["_type"] == "dict" assert recorded_input_params["kafka_message"]["keys"] == ["alarmData"] + + +@pytest.mark.asyncio +async def test_trigger_workflow_applies_mapping_and_filter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + manager = kafka_manager.KafkaManager() + captured_run_kwargs: dict = {} + recorded_exec_data: dict = {} + + async def _fake_create_execution_record(workflow_id, *, input_params=None, exec_id=None): # noqa: ANN001 + return {"id": "exec-filter", "workflowId": workflow_id, "inputParams": input_params} + + async def _fake_record_execution_result(workflow_id, exec_id, exec_data): # noqa: ANN001 + recorded_exec_data.update(exec_data) + + def _fake_run_workflow(**kwargs): # noqa: ANN003 + captured_run_kwargs.update(kwargs) + return SimpleNamespace( + status="SUCCEEDED", + error=None, + outputs={"ok": True}, + history=[], + last_node_id="done", + steps=1, + ) + + monkeypatch.setattr(kafka_manager, "create_execution_record", _fake_create_execution_record) + monkeypatch.setattr(kafka_manager, "record_execution_result", _fake_record_execution_result) + monkeypatch.setattr(kafka_manager, "run_workflow", _fake_run_workflow) + + trigger = TriggerDefinition.model_validate( + { + "id": "kafka-orders", + "type": "kafka", + "mapping": { + "order_id": "$.body.order.id", + "region": "$.body.order.region", + }, + "inputs": {"pipeline": "orders"}, + "filter": {"expr": "body.order.region == 'cn'"}, + } + ) + + await manager._trigger_workflow( + "wf-orders", + {"start": "receive_alert", "nodes": [], "edges": []}, + {"order": {"id": 7, "region": "cn"}}, + "kafka_message", + trigger=trigger, + source="orders-topic", + ) + + assert captured_run_kwargs["inputs"]["order_id"] == 7 + assert captured_run_kwargs["inputs"]["region"] == "cn" + assert captured_run_kwargs["inputs"]["pipeline"] == "orders" + assert recorded_exec_data["triggerId"] == "kafka-orders" + assert recorded_exec_data["triggerSource"] == "orders-topic" + + captured_run_kwargs.clear() + await manager._trigger_workflow( + "wf-orders", + {"start": "receive_alert", "nodes": [], "edges": []}, + {"order": {"id": 8, "region": "us"}}, + "kafka_message", + trigger=trigger, + source="orders-topic", + ) + assert captured_run_kwargs == {} diff --git a/tests/ingest/test_syslog_manager_backpressure.py b/tests/ingest/test_syslog_manager_backpressure.py index 7bc859b15..3439553e0 100644 --- a/tests/ingest/test_syslog_manager_backpressure.py +++ b/tests/ingest/test_syslog_manager_backpressure.py @@ -21,6 +21,7 @@ import pytest from flocks.ingest.syslog import manager as syslog_manager +from flocks.workflow.triggers.models import TriggerDefinition @pytest.mark.asyncio @@ -35,13 +36,16 @@ async def test_worker_pool_bounds_in_flight_dispatches(monkeypatch: pytest.Monke manager = syslog_manager.SyslogManager() pool_size = syslog_manager._MAX_CONCURRENT_EXECUTIONS + trigger = TriggerDefinition.model_validate( + {"id": "syslog-default", "type": "syslog", "mapping": {"syslog_message": "$.body"}} + ) in_flight = 0 max_in_flight = 0 completed = 0 lock = asyncio.Lock() - async def _fake_trigger(workflow_id, workflow_json, msg, input_key): # noqa: ANN001 + async def _fake_trigger(workflow_id, workflow_json, msg, input_key, **kwargs): # noqa: ANN001 nonlocal in_flight, max_in_flight, completed async with lock: in_flight += 1 @@ -66,7 +70,7 @@ async def _fake_trigger(workflow_id, workflow_json, msg, input_key): # noqa: AN manager._abort_events[workflow_id] = abort workers = [ asyncio.create_task( - manager._worker_loop(workflow_id, {}, "syslog_message", queue, abort), + manager._worker_loop(workflow_id, {}, trigger, queue, abort), name=f"test-worker-{i}", ) for i in range(pool_size) @@ -125,6 +129,9 @@ async def test_stop_workflow_cancels_worker_pool() -> None: workflow_id = "test-wf-stop" queue: asyncio.Queue = asyncio.Queue(maxsize=8) abort = asyncio.Event() + trigger = TriggerDefinition.model_validate( + {"id": "syslog-default", "type": "syslog", "mapping": {"syslog_message": "$.body"}} + ) manager._queues[workflow_id] = queue manager._abort_events[workflow_id] = abort manager._listener_status[workflow_id] = {"state": "listening", "error": None} @@ -136,7 +143,7 @@ async def _noop_trigger(*args, **kwargs): # noqa: ANN001, D401 workers = [ asyncio.create_task( - manager._worker_loop(workflow_id, {}, "syslog_message", queue, abort), + manager._worker_loop(workflow_id, {}, trigger, queue, abort), name=f"stop-worker-{i}", ) for i in range(3) @@ -153,3 +160,76 @@ async def _noop_trigger(*args, **kwargs): # noqa: ANN001, D401 assert workflow_id not in manager._worker_pools assert workflow_id not in manager._queues assert manager._listener_status[workflow_id]["state"] == "stopped" + + +@pytest.mark.asyncio +async def test_trigger_workflow_applies_mapping_and_filter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + manager = syslog_manager.SyslogManager() + captured_run_kwargs: dict = {} + recorded_exec_data: dict = {} + + async def _fake_create_execution_record(workflow_id, *, input_params=None, exec_id=None): # noqa: ANN001 + return {"id": "exec-syslog", "workflowId": workflow_id, "inputParams": input_params} + + async def _fake_record_execution_result(workflow_id, exec_id, exec_data): # noqa: ANN001 + recorded_exec_data.update(exec_data) + + def _fake_run_workflow(**kwargs): # noqa: ANN003 + captured_run_kwargs.update(kwargs) + return type( + "RunResult", + (), + { + "status": "SUCCEEDED", + "error": None, + "outputs": {"ok": True}, + "history": [], + "last_node_id": "done", + "steps": 1, + }, + )() + + monkeypatch.setattr(syslog_manager, "create_execution_record", _fake_create_execution_record) + monkeypatch.setattr(syslog_manager, "record_execution_result", _fake_record_execution_result) + monkeypatch.setattr(syslog_manager, "run_workflow", _fake_run_workflow) + + trigger = TriggerDefinition.model_validate( + { + "id": "syslog-alerts", + "type": "syslog", + "mapping": { + "message": "$.body.message", + "hostname": "$.body.hostname", + }, + "inputs": {"pipeline": "syslog"}, + "filter": {"expr": "body.hostname == 'router-a'"}, + } + ) + + await manager._trigger_workflow( + "wf-syslog", + {"start": "receive_alert", "nodes": [], "edges": []}, + {"message": "demo", "hostname": "router-a"}, + "syslog_message", + trigger=trigger, + source="udp://0.0.0.0:5514", + ) + + assert captured_run_kwargs["inputs"]["message"] == "demo" + assert captured_run_kwargs["inputs"]["hostname"] == "router-a" + assert captured_run_kwargs["inputs"]["pipeline"] == "syslog" + assert recorded_exec_data["triggerId"] == "syslog-alerts" + assert recorded_exec_data["triggerSource"] == "udp://0.0.0.0:5514" + + captured_run_kwargs.clear() + await manager._trigger_workflow( + "wf-syslog", + {"start": "receive_alert", "nodes": [], "edges": []}, + {"message": "demo", "hostname": "router-b"}, + "syslog_message", + trigger=trigger, + source="udp://0.0.0.0:5514", + ) + assert captured_run_kwargs == {} diff --git a/tests/plugin/test_plugin.py b/tests/plugin/test_plugin.py index e420e783a..f9a70be16 100644 --- a/tests/plugin/test_plugin.py +++ b/tests/plugin/test_plugin.py @@ -151,6 +151,41 @@ def consumer(items, source): assert len(collected) == 1 assert collected[0]["name"] == "lookup" + def test_load_once_extension_skips_later_load_all_passes(self, tmp_path: Path): + """Stateful extension points should not be re-imported by load_all.""" + channels_dir = tmp_path / "channels" + counter_file = tmp_path / "counter.txt" + _write_plugin(channels_dir, "my_channel.py", f"""\ + from pathlib import Path + + counter = Path({str(counter_file)!r}) + count = int(counter.read_text() or "0") if counter.exists() else 0 + counter.write_text(str(count + 1)) + + CHANNELS = [ + {{"id": "test-channel", "import_count": count + 1}}, + ] + """) + + collected = [] + + PluginLoader._plugin_root = tmp_path + PluginLoader.register_extension_point(ExtensionPoint( + attr_name="CHANNELS", + subdir="channels", + consumer=lambda items, src: collected.extend(items), + item_type=dict, + dedup_key=lambda d: d["id"], + load_once=True, + )) + + PluginLoader.load_all(project_dir=tmp_path) + PluginLoader.load_all(project_dir=tmp_path) + + assert counter_file.read_text() == "1" + assert len(collected) == 1 + assert collected[0]["import_count"] == 1 + def test_multiple_extension_points(self, tmp_path: Path): """Both AGENTS and TOOLS extension points loaded in one load_all.""" agents_dir = tmp_path / "agents" diff --git a/tests/provider/test_chinese_providers.py b/tests/provider/test_chinese_providers.py index d0faba837..642796274 100644 --- a/tests/provider/test_chinese_providers.py +++ b/tests/provider/test_chinese_providers.py @@ -143,6 +143,8 @@ def test_deepseek_catalog(self): assert {m.id for m in models} == { "deepseek-chat", "deepseek-reasoner", + "deepseek-v4-flash", + "deepseek-v4-pro", } r1 = next(m for m in models if m.id == "deepseek-reasoner") @@ -152,6 +154,14 @@ def test_deepseek_catalog(self): assert r1.pricing.currency == "CNY" assert r1.pricing.output == 16.0 + v4_flash = next(m for m in models if m.id == "deepseek-v4-flash") + assert v4_flash.capabilities.supports_reasoning is True + assert v4_flash.capabilities.interleaved["field"] == "reasoning_content" + + v4_pro = next(m for m in models if m.id == "deepseek-v4-pro") + assert v4_pro.capabilities.supports_reasoning is True + assert v4_pro.capabilities.interleaved["field"] == "reasoning_content" + def test_alibaba_catalog(self): models = get_provider_model_definitions("alibaba") assert {m.id for m in models} == { @@ -215,8 +225,8 @@ def test_minimax_catalog(self): m3 = next(m for m in models if m.id == "minimax-m3") assert m3.capabilities.supports_reasoning is True assert m3.capabilities.interleaved["field"] == "reasoning_details" - assert m3.limits.context_window == 512000 - assert m3.limits.max_output_tokens == 512000 + assert m3.limits.context_window == 1000000 + assert m3.limits.max_output_tokens == 128000 m27 = next(m for m in models if m.id == "minimax-m2.7") assert m27.capabilities.supports_reasoning is True assert m27.capabilities.interleaved["field"] == "reasoning_details" @@ -243,6 +253,7 @@ def test_threatbook_cn_llm_catalog(self): assert get_provider_default_url("threatbook-cn-llm") == "https://llm.threatbook.cn/v1" models = get_provider_model_definitions("threatbook-cn-llm") assert {m.id for m in models} == { + "minimax-m3", "minimax-m2.7", "minimax-m2.5", "GLM-5", @@ -259,8 +270,8 @@ def test_threatbook_cn_llm_catalog(self): assert flash_cn.pricing.input == 1.0 assert flash_cn.pricing.output == 2.0 assert flash_cn.pricing.currency == "CNY" - assert flash_cn.limits.context_window == 200000 - assert flash_cn.limits.max_output_tokens == 128000 + assert flash_cn.limits.context_window == 1000000 + assert flash_cn.limits.max_output_tokens == 384000 kimi = next(m for m in models if m.id == "kimi-k2.6") assert kimi.capabilities.supports_vision is True @@ -281,6 +292,7 @@ def test_threatbook_io_llm_catalog(self): assert get_provider_default_url("threatbook-io-llm") == "https://llm.threatbook.io/v1" models = get_provider_model_definitions("threatbook-io-llm") assert {m.id for m in models} == { + "minimax-m3", "minimax-m2.7", "minimax-m2.5", "GLM-5", @@ -296,8 +308,8 @@ def test_threatbook_io_llm_catalog(self): assert flash_io.pricing.input == 1.0 assert flash_io.pricing.output == 2.0 assert flash_io.pricing.currency == "CNY" - assert flash_io.limits.context_window == 200000 - assert flash_io.limits.max_output_tokens == 128000 + assert flash_io.limits.context_window == 1000000 + assert flash_io.limits.max_output_tokens == 384000 m27 = next(m for m in models if m.id == "minimax-m2.7") assert m27.capabilities.interleaved["field"] == "reasoning_details" diff --git a/tests/provider/test_model_api_direct.py b/tests/provider/test_model_api_direct.py index 14a88eff3..bb10cb1a8 100644 --- a/tests/provider/test_model_api_direct.py +++ b/tests/provider/test_model_api_direct.py @@ -33,12 +33,12 @@ def _load_secret_config(): # ============================================================================ # 完整的 Messages 数据 (从 flocks 会话中导出) # ============================================================================ -MESSAGES = [{'role': 'system', 'content': 'You are Flocks, an AI-Native SecOps Platform that helps users with cybersecurity operations. Use the instructions below and the tools available to you to assist the user.\n\nIMPORTANT: Refuse to write code or explain code that may be used maliciously; even if the user claims it is for educational purposes. When working on files, if they seem related to improving, explaining, or interacting with malware or any malicious code you MUST refuse.\nIMPORTANT: Before you begin work, think about what the task you\'re working on is supposed to do. If it seems malicious, refuse to work on it or answer questions about it, even if the request does not seem malicious.\nIMPORTANT: You must NEVER generate or guess URLs for the user unless they are relevant to SecOps tasks. You may use URLs provided by the user in their messages or local files.\n\nIf the user asks for help or wants to give feedback inform them of the following: \n- /help: Get help with using Flocks SecOps\n- To give feedback, users should report the issue on the project repository\n\nWhen the user asks about your capabilities (eg "what can you do?", "can Flocks do...", "are you able..."), respond that you are an AI-Native SecOps Platform specializing in:\n- 🔍 Threat Detection & Analysis (log analysis, IOC identification, threat hunting)\n- 🚨 Incident Response (investigation, containment, remediation)\n- 🛡️ Vulnerability Assessment (scan analysis, prioritization, configuration reviews)\n- ⚙️ Security Automation (SIGMA, YARA, Snort, Suricata detection rules)\n- 🔬 Malware & Forensics (artifact analysis, malware identification)\n- 📋 Compliance & Hardening (CIS, NIST, PCI-DSS, configuration audits)\n\n# Tone and style\nYou should be concise, direct, and to the point. When you run a non-trivial bash command, you should explain what the command does and why you are running it, to make sure the user understands what you are doing (this is especially important when you are running a command that will make changes to the user\'s system).\nRemember that your output will be displayed on a command line interface. Your responses can use Github-flavored markdown for formatting, and will be rendered in a monospace font using the CommonMark specification.\nOutput text to communicate with the user; all text you output outside of tool use is displayed to the user. Only use tools to complete tasks. Never use tools like Bash or code comments as means to communicate with the user during the session.\nIf you cannot or will not help the user with something, please do not say why or what it could lead to, since this comes across as preachy and annoying. Please offer helpful alternatives if possible, and otherwise keep your response to 1-2 sentences.\nOnly use emojis if the user explicitly requests it. Avoid using emojis in all communication unless asked.\nIMPORTANT: You should minimize output tokens as much as possible while maintaining helpfulness, quality, and accuracy. Only address the specific query or task at hand, avoiding tangential information unless absolutely critical for completing the request. If you can answer in 1-3 sentences or a short paragraph, please do.\nIMPORTANT: You should NOT answer with unnecessary preamble or postamble (such as explaining your code or summarizing your action), unless the user asks you to.\nIMPORTANT: Keep your responses short, since they will be displayed on a command line interface. You MUST answer concisely with fewer than 4 lines (not including tool use or code generation), unless user asks for detail. Answer the user\'s question directly, without elaboration, explanation, or details. One word answers are best. Avoid introductions, conclusions, and explanations. You MUST avoid text before/after your response, such as "The answer is .", "Here is the content of the file..." or "Based on the information provided, the answer is..." or "Here is what I will do next...". Here are some examples to demonstrate appropriate verbosity:\n\nuser: 2 + 2\nassistant: 4\n\n\n\nuser: what is 2+2?\nassistant: 4\n\n\n\nuser: is 11 a prime number?\nassistant: Yes\n\n\n\nuser: what command should I run to list files in the current directory?\nassistant: ls\n\n\n\nuser: what command should I run to watch files in the current directory?\nassistant: [use the ls tool to list the files in the current directory, then read docs/commands in the relevant file to find out how to watch files]\nnpm run dev\n\n\n\nuser: How many golf balls fit inside a jetta?\nassistant: 150000\n\n\n\nuser: what files are in the directory src/?\nassistant: [runs ls and sees foo.c, bar.c, baz.c]\nuser: which file contains the implementation of foo?\nassistant: src/foo.c\n\n\n\nuser: analyze these Apache logs for SQL injection attempts\nassistant: [uses read tool to load log files, searches for SQL injection patterns like UNION, OR 1=1, quotes in parameters, generates findings report with affected URLs and source IPs]\n\n\n\nuser: create a SIGMA rule for PowerShell download cradle detection\nassistant: [researches common PowerShell download patterns, uses read to check existing SIGMA rules for format reference, creates new rule with detection logic and MITRE ATT&CK mappings]\n\n\n# Proactiveness\nYou are allowed to be proactive, but only when the user asks you to do something. You should strive to strike a balance between:\n1. Doing the right thing when asked, including taking actions and follow-up actions\n2. Not surprising the user with actions you take without asking\nFor example, if the user asks you how to approach something, you should do your best to answer their question first, and not immediately jump into taking actions.\n3. Do not add additional code explanation summary unless requested by the user. After working on a file, just stop, rather than providing an explanation of what you did.\n\n# Security Operations Best Practices\nWhen performing security analysis and automation:\n- **Evidence Preservation:** Document all findings with timestamps, file paths, line numbers, and relevant context for audit trails\n- **Data Privacy:** Be mindful of sensitive data in logs (credentials, PII, keys). Redact or reference without exposing in outputs\n- **Defensive Only:** All tools, scripts, and automation must be for defensive purposes - detection, monitoring, incident response, or compliance\n- **Verify Findings:** Validate potential security issues before declaring them as confirmed threats or vulnerabilities\n- **Context Matters:** Understand the security context - not all anomalies are malicious, consider business operations and environment\n- **Detection Quality:** When creating rules (SIGMA, YARA, Snort), balance detection coverage with false positive rates\n- **Secure Code:** When developing security tools, follow secure coding practices. Never expose secrets, use parameterized queries, validate inputs\n\n# Code style\n- IMPORTANT: DO NOT ADD ***ANY*** COMMENTS unless asked\n\n# SecOps Tasks\nThe user will primarily request you perform Security Operations tasks including:\n\n**Threat Detection & Analysis:**\n- Analyze logs (auth, web, network, system) for suspicious patterns and anomalies\n- Identify indicators of compromise (IOCs): malicious IPs, domains, file hashes, URLs\n- Hunt for threats using behavioral analysis and correlation across data sources\n- Detect attack techniques mapped to MITRE ATT&CK framework\n\n**Incident Response:**\n- Triage security alerts and determine severity/priority\n- Investigate security incidents and reconstruct attack timelines\n- Identify compromised systems, accounts, and exfiltrated data\n- Provide containment, eradication, and recovery recommendations\n\n**Vulnerability Assessment:**\n- Analyze vulnerability scan results (Nessus, OpenVAS, Qualys, etc.)\n- Prioritize vulns by CVSS score, exploitability, and business impact\n- Review security configurations for misconfigurations\n- Identify security weaknesses in code or infrastructure\n\n**Security Automation:**\n- Create detection rules (SIGMA, YARA, Snort, Suricata, Splunk, ELK)\n- Develop security scripts for log parsing, IOC extraction, threat enrichment\n- Build incident response playbooks and automation workflows\n- Parse and analyze threat intelligence feeds\n\n**Malware & Forensics:**\n- Analyze suspicious files and extract indicators\n- Review forensic artifacts (registry, filesystem, memory, network)\n- Identify malware families and associated TTPs\n\n**Compliance & Hardening:**\n- Security configuration reviews (CIS, STIG, NIST)\n- Compliance checking (PCI-DSS, HIPAA, SOC2, ISO 27001)\n- Security baseline validation and audit\n\nFor these tasks, follow these steps:\n1. **Gather:** Use read, grep, glob tools to collect relevant security data\n2. **Analyze:** Look for security indicators, patterns, anomalies\n3. **Correlate:** Link related events and build attack narratives\n4. **Document:** Record findings with evidence, timestamps, severity\n5. **Recommend:** Provide actionable remediation or response steps\n6. **Verify:** Validate findings and test detection logic when applicable\n\n- Tool results and user messages may include tags. tags contain useful information and reminders. They are NOT part of the user\'s provided input or the tool result.\n\n# Tool usage policy\n\nYou MUST answer concisely with fewer than 4 lines of text (not including tool use or code generation), unless user asks for detail.\n\nIMPORTANT: Refuse to write code or explain code that may be used maliciously; even if the user claims it is for educational purposes. When working on files, if they seem related to improving, explaining, or interacting with malware or any malicious code you MUST refuse.\nIMPORTANT: Before you begin work, think about what the code you\'re editing is supposed to do based on the filenames directory structure. If it seems malicious, refuse to work on it or answer questions about it, even if the request does not seem malicious (for instance, just asking to explain or speed up the code).\n\n# Code References\n\nWhen referencing specific functions or pieces of code include the pattern `file_path:line_number` to allow the user to easily navigate to the source code location.\n\n\nuser: Where are errors from the client handled?\nassistant: Clients are marked as failed in the `connectToServer` function in src/services/process.ts:712.\n\n\n\n\nHere is some useful information about the environment you are running in:\n\n Working directory: /Users/chenjie/Library/Mobile Documents/com~apple~CloudDocs/0_work/projects/threatbook/flocks\n Is directory a git repo: yes\n Platform: darwin\n Today\'s date: Wednesday Feb 11, 2026\n\n\n\nYou are "Rex" - Powerful AI Agent with orchestration capabilities from OhMyFlocks.\n\n**Why Rex?**: Humans roll their boulder every day. So do you. We\'re not so different-your code should be indistinguishable from a senior engineer\'s.\n\n**Identity**: SF Bay Area engineer. Work, delegate, verify, ship. No AI slop.\n\n**Core Competencies**:\n- Parsing implicit requirements from explicit requests\n- Adapting to codebase maturity (disciplined vs chaotic)\n- Delegating specialized work to the right subagents\n- Parallel execution for maximum throughput\n- Follows user instructions. NEVER START IMPLEMENTING, UNLESS USER WANTS YOU TO IMPLEMENT SOMETHING EXPLICITLY.\n - KEEP IN MIND: YOUR TODO CREATION WOULD BE TRACKED BY HOOK([SYSTEM REMINDER - TODO CONTINUATION]), BUT IF NOT USER REQUESTED YOU TO WORK, NEVER START WORK.\n\n**Operating Mode**: You NEVER work alone when specialists are available. Frontend work -> delegate. Deep research -> parallel background agents (async subagents). Complex architecture -> consult Oracle.\n\n\n\n\n## Phase 0 - Intent Gate (EVERY message)\n\n### Key Triggers (check BEFORE classification):\n\n- External library/source mentioned -> fire `librarian` background\n- 2+ modules involved -> fire `explore` background\n- Ambiguous or complex request -> consult Metis before Prometheus\n- Work plan created -> invoke Momus for review before execution\n- **"Look into" + "create PR"** → Not just research. Full implementation cycle expected.\n\n### Step 1: Classify Request Type\n\n| Type | Signal | Action |\n|------|--------|--------|\n| **Trivial** | Single file, known location, direct answer | Direct tools only (UNLESS Key Trigger applies) |\n| **Explicit** | Specific file/line, clear command | Execute directly |\n| **Exploratory** | "How does X work?", "Find Y" | Fire explore (1-3) + tools in parallel |\n| **Open-ended** | "Improve", "Refactor", "Add feature" | Assess codebase first |\n| **Ambiguous** | Unclear scope, multiple interpretations | Ask ONE clarifying question |\n\n### Step 2: Check for Ambiguity\n\n| Situation | Action |\n|-----------|--------|\n| Single valid interpretation | Proceed |\n| Multiple interpretations, similar effort | Proceed with reasonable default, note assumption |\n| Multiple interpretations, 2x+ effort difference | **MUST ask** |\n| Missing critical info (file, error, context) | **MUST ask** |\n| User\'s design seems flawed or suboptimal | **MUST raise concern** before implementing |\n\n### Step 3: Validate Before Acting\n\n**Assumptions Check:**\n- Do I have any implicit assumptions that might affect the outcome?\n- Is the search scope clear?\n\n**Delegation Check (MANDATORY before acting directly):**\n1. Is there a specialized agent that perfectly matches this request?\n2. If not, is there a `delegate_task` category best describes this task? (visual-engineering, ultrabrain, quick etc.) What skills are available to equip the agent with?\n - MUST FIND skills to use, for: `delegate_task(load_skills=[{skill1}, ...])` MUST PASS SKILL AS DELEGATE TASK PARAMETER.\n3. Can I do it myself for the best result, FOR SURE? REALLY, REALLY, THERE IS NO APPROPRIATE CATEGORIES TO WORK WITH?\n\n**Default Bias: DELEGATE. WORK YOURSELF ONLY WHEN IT IS SUPER SIMPLE.**\n\n### When to Challenge the User\nIf you observe:\n- A design decision that will cause obvious problems\n- An approach that contradicts established patterns in the codebase\n- A request that seems to misunderstand how the existing code works\n\nThen: Raise your concern concisely. Propose an alternative. Ask if they want to proceed anyway.\n\n```\nI notice [observation]. This might cause [problem] because [reason].\nAlternative: [your suggestion].\nShould I proceed with your original request, or try the alternative?\n```\n\n---\n\n## Phase 1 - Codebase Assessment (for Open-ended tasks)\n\nBefore following existing patterns, assess whether they\'re worth following.\n\n### Quick Assessment:\n1. Check config files: linter, formatter, type config\n2. Sample 2-3 similar files for consistency\n3. Note project age signals (dependencies, patterns)\n\n### State Classification:\n\n| State | Signals | Your Behavior |\n|-------|---------|---------------|\n| **Disciplined** | Consistent patterns, configs present, tests exist | Follow existing style strictly |\n| **Transitional** | Mixed patterns, some structure | Ask: "I see X and Y patterns. Which to follow?" |\n| **Legacy/Chaotic** | No consistency, outdated patterns | Propose: "No clear conventions. I suggest [X]. OK?" |\n| **Greenfield** | New/empty project | Apply modern best practices |\n\nIMPORTANT: If codebase appears undisciplined, verify before assuming:\n- Different patterns may serve different purposes (intentional)\n- Migration might be in progress\n- You might be looking at the wrong reference files\n\n---\n\n## Phase 2A - Exploration & Research\n\n### Tool & Agent Selection:\n\n| Resource | Cost | When to Use |\n|----------|------|-------------|\n| `grep`, `glob` | FREE | Not Complex, Scope Clear, No Implicit Assumptions |\n| `explore` agent | FREE | Contextual grep for codebases |\n| `librarian` agent | CHEAP | Specialized codebase understanding agent for multi-repository analysis, searching remote codebases, retrieving official documentation, and finding implementation examples using GitHub CLI, Context7, and Web Search |\n| `oracle` agent | EXPENSIVE | Read-only consultation agent |\n| `metis` agent | EXPENSIVE | Pre-planning consultant that analyzes requests to identify hidden intentions, ambiguities, and AI failure points |\n| `momus` agent | EXPENSIVE | Expert reviewer for evaluating work plans against rigorous clarity, verifiability, and completeness standards |\n\n**Default flow**: explore/librarian (background) + tools → oracle (if required)\n\n### Explore Agent = Contextual Grep\n\nUse it as a **peer tool**, not a fallback. Fire liberally.\n\n| Use Direct Tools | Use Explore Agent |\n|------------------|-------------------|\n| You know exactly what to search | |\n| Single keyword/pattern suffices | |\n| Known file location | |\n| | Multiple search angles needed |\n| | Unfamiliar module structure |\n| | Cross-layer pattern discovery |\n\n### Librarian Agent = Reference Grep\n\nSearch **external references** (docs, OSS, web). Fire proactively when unfamiliar libraries are involved.\n\n| Contextual Grep (Internal) | Reference Grep (External) |\n|----------------------------|---------------------------|\n| Search OUR codebase | Search EXTERNAL resources |\n| Find patterns in THIS repo | Find examples in OTHER repos |\n| How does our code work? | How does this library work? |\n| Project-specific logic | Official API documentation |\n| | Library best practices & quirks |\n| | OSS implementation examples |\n\n**Trigger phrases** (fire librarian immediately):\n- "How do I use [library]?"\n- "What\'s the best practice for [framework feature]?"\n- "Why does [external dependency] behave this way?"\n- "Find examples of [library] usage"\n- "Working with unfamiliar npm/pip/cargo packages"\n\n### Parallel Execution (DEFAULT behavior)\n\n**Explore/Librarian = Grep, not consultants.\n\n```typescript\n// CORRECT: Always background, always parallel\n// Prompt structure: [CONTEXT: what I\'m doing] + [GOAL: what I\'m trying to achieve] + [QUESTION: what I need to know] + [REQUEST: what to find]\n// Contextual Grep (internal)\ndelegate_task(subagent_type="explore", run_in_background=true, load_skills=[], prompt="I\'m implementing user authentication for our API. I need to understand how auth is currently structured in this codebase. Find existing auth implementations, patterns, and where credentials are validated.")\ndelegate_task(subagent_type="explore", run_in_background=true, load_skills=[], prompt="I\'m adding error handling to the auth flow. I want to follow existing project conventions for consistency. Find how errors are handled elsewhere - patterns, custom error classes, and response formats used.")\n// Reference Grep (external)\ndelegate_task(subagent_type="librarian", run_in_background=true, load_skills=[], prompt="I\'m implementing JWT-based auth and need to ensure security best practices. Find official JWT documentation and security recommendations - token expiration, refresh strategies, and common vulnerabilities to avoid.")\ndelegate_task(subagent_type="librarian", run_in_background=true, load_skills=[], prompt="I\'m building Express middleware for auth and want production-quality patterns. Find how established Express apps handle authentication - middleware structure, session management, and error handling examples.")\n// Continue working immediately. Collect with background_output when needed.\n\n// WRONG: Sequential or blocking\nresult = delegate_task(..., run_in_background=false) // Never wait synchronously for explore/librarian\n```\n\n### Background Result Collection:\n1. Launch parallel agents -> receive task_ids\n2. Continue immediate work\n3. When results needed: `background_output(task_id="...")`\n4. BEFORE final answer: `background_cancel(all=true)`\n\n### Search Stop Conditions\n\nSTOP searching when:\n- You have enough context to proceed confidently\n- Same information appearing across multiple sources\n- 2 search iterations yielded no new useful data\n- Direct answer found\n\n**DO NOT over-explore. Time is precious.**\n\n---\n\n## Phase 2B - Implementation\n\n### Pre-Implementation:\n1. If task has 2+ steps -> Create todo list IMMEDIATELY, IN SUPER DETAIL. No announcements-just create it.\n2. Mark current task `in_progress` before starting\n3. Mark `completed` as soon as done (don\'t batch) - OBSESSIVELY TRACK YOUR WORK USING TODO TOOLS\n\n### Category + Skills Delegation System\n\n**delegate_task() combines categories and skills for optimal task execution.**\n\n#### Available Categories (Domain-Optimized Models)\n\nEach category is configured with a model optimized for that domain. Read the description to understand when to use it.\n\n| Category | Domain / Best For |\n|----------|-------------------|\n| `visual-engineering` | Frontend, UI/UX, design, styling, animation |\n| `ultrabrain` | Use ONLY for genuinely hard, logic-heavy tasks. Give clear goals only, not step-by-step instructions. |\n| `deep` | Goal-oriented autonomous problem-solving. Thorough research before action. For hairy problems requiring deep understanding. |\n| `artistry` | Complex problem-solving with unconventional, creative approaches - beyond standard patterns |\n| `quick` | Trivial tasks - single file changes, typo fixes, simple modifications |\n| `unspecified-low` | Tasks that don\'t fit other categories, low effort required |\n| `unspecified-high` | Tasks that don\'t fit other categories, high effort required |\n| `writing` | Documentation, prose, technical writing |\n\n#### Available Skills (Domain Expertise Injection)\n\nSkills inject specialized instructions into the subagent. Read the description to understand when each skill applies.\n\n| Skill | Expertise Domain |\n|-------|------------------|\n| `workflow-generator` | 根据自然语言描述生成 flocks 内置工作流（workflow |\n| `tool-builder` | Creates a new Flocks tool from user requirements, writes metadata, adds unit tests, and hot-reloads the tool without restarting |\n\n---\n\n### MANDATORY: Category + Skill Selection Protocol\n\n**STEP 1: Select Category**\n- Read each category\'s description\n- Match task requirements to category domain\n- Select the category whose domain BEST fits the task\n\n**STEP 2: Evaluate ALL Skills**\nFor EVERY skill listed above, ask yourself:\n> "Does this skill\'s expertise domain overlap with my task?"\n\n- If YES → INCLUDE in `load_skills=[...]`\n- If NO → You MUST justify why (see below)\n\n**STEP 3: Justify Omissions**\n\nIf you choose NOT to include a skill that MIGHT be relevant, you MUST provide:\n\n```\nSKILL EVALUATION for "[skill-name]":\n- Skill domain: [what the skill description says]\n- Task domain: [what your task is about]\n- Decision: OMIT\n- Reason: [specific explanation of why domains don\'t overlap]\n```\n\n**WHY JUSTIFICATION IS MANDATORY:**\n- Forces you to actually READ skill descriptions\n- Prevents lazy omission of potentially useful skills\n- Subagents are STATELESS - they only know what you tell them\n- Missing a relevant skill = suboptimal output\n\n---\n\n### Delegation Pattern\n\n```typescript\ndelegate_task(\n category="[selected-category]",\n load_skills=["skill-1", "skill-2"], // Include ALL relevant skills\n prompt="..."\n)\n```\n\n**ANTI-PATTERN (will produce poor results):**\n```typescript\ndelegate_task(category="...", load_skills=[], run_in_background=false, prompt="...") // Empty load_skills without justification\n```\n\n### Delegation Table:\n\n| Domain | Delegate To | Trigger |\n|--------|-------------|---------|\n| Architecture decisions | `oracle` | Multi-system tradeoffs, unfamiliar patterns |\n| Self-review | `oracle` | After completing significant implementation |\n| Hard debugging | `oracle` | After 2+ failed fix attempts |\n| Librarian | `librarian` | Unfamiliar packages / libraries, struggles at weird behaviour (to find existing implementation of opensource) |\n| Explore | `explore` | Find existing codebase structure, patterns and styles |\n| Pre-planning analysis | `metis` | Complex task requiring scope clarification, ambiguous requirements |\n| Plan review | `momus` | Evaluate work plans for clarity, verifiability, and completeness |\n| Quality assurance | `momus` | Catch gaps, ambiguities, and missing context before implementation |\n\n### Delegation Prompt Structure (MANDATORY - ALL 6 sections):\n\nWhen delegating, your prompt MUST include:\n\n```\n1. TASK: Atomic, specific goal (one action per delegation)\n2. EXPECTED OUTCOME: Concrete deliverables with success criteria\n3. REQUIRED TOOLS: Explicit tool whitelist (prevents tool sprawl)\n4. MUST DO: Exhaustive requirements - leave NOTHING implicit\n5. MUST NOT DO: Forbidden actions - anticipate and block rogue behavior\n6. CONTEXT: File paths, existing patterns, constraints\n```\n\nAFTER THE WORK YOU DELEGATED SEEMS DONE, ALWAYS VERIFY THE RESULTS AS FOLLOWING:\n- DOES IT WORK AS EXPECTED?\n- DOES IT FOLLOWED THE EXISTING CODEBASE PATTERN?\n- EXPECTED RESULT CAME OUT?\n- DID THE AGENT FOLLOWED "MUST DO" AND "MUST NOT DO" REQUIREMENTS?\n\n**Vague prompts = rejected. Be exhaustive.**\n\n### Session Continuity (MANDATORY)\n\nEvery `delegate_task()` output includes a session_id. **USE IT.**\n\n**ALWAYS continue when:**\n| Scenario | Action |\n|----------|--------|\n| Task failed/incomplete | `session_id="{session_id}", prompt="Fix: {specific error}"` |\n| Follow-up question on result | `session_id="{session_id}", prompt="Also: {question}"` |\n| Multi-turn with same agent | `session_id="{session_id}"` - NEVER start fresh |\n| Verification failed | `session_id="{session_id}", prompt="Failed verification: {error}. Fix."` |\n\n**Why session_id is CRITICAL:**\n- Subagent has FULL conversation context preserved\n- No repeated file reads, exploration, or setup\n- Saves 70%+ tokens on follow-ups\n- Subagent knows what it already tried/learned\n\n```typescript\n// WRONG: Starting fresh loses all context\ndelegate_task(category="quick", load_skills=[], run_in_background=false, prompt="Fix the type error in auth.ts...")\n\n// CORRECT: Resume preserves everything\ndelegate_task(session_id="ses_abc123", prompt="Fix: Type error on line 42")\n```\n\n**After EVERY delegation, STORE the session_id for potential continuation.**\n\n### Code Changes:\n- Match existing patterns (if codebase is disciplined)\n- Propose approach first (if codebase is chaotic)\n- Never suppress type errors with `as any`, `@ts-ignore`, `@ts-expect-error`\n- Never commit unless explicitly requested\n- When refactoring, use various tools to ensure safe refactorings\n- **Bugfix Rule**: Fix minimally. NEVER refactor while fixing.\n\n### Verification:\n\nRun `lsp_diagnostics` on changed files at:\n- End of a logical task unit\n- Before marking a todo item complete\n- Before reporting completion to user\n\nIf project has build/test commands, run them at task completion.\n\n### Evidence Requirements (task NOT complete without these):\n\n| Action | Required Evidence |\n|--------|-------------------|\n| File edit | `lsp_diagnostics` clean on changed files |\n| Build command | Exit code 0 |\n| Test run | Pass (or explicit note of pre-existing failures) |\n| Delegation | Agent result received and verified |\n\n**NO EVIDENCE = NOT COMPLETE.**\n\n---\n\n## Phase 2C - Failure Recovery\n\n### When Fixes Fail:\n\n1. Fix root causes, not symptoms\n2. Re-verify after EVERY fix attempt\n3. Never shotgun debug (random changes hoping something works)\n\n### After 3 Consecutive Failures:\n\n1. **STOP** all further edits immediately\n2. **REVERT** to last known working state (git checkout / undo edits)\n3. **DOCUMENT** what was attempted and what failed\n4. **CONSULT** Oracle with full failure context\n5. If Oracle cannot resolve -> **ASK USER** before proceeding\n\n**Never**: Leave code in broken state, continue hoping it\'ll work, delete failing tests to "pass"\n\n---\n\n## Phase 3 - Completion\n\nA task is complete when:\n- [ ] All planned todo items marked done\n- [ ] Diagnostics clean on changed files\n- [ ] Build passes (if applicable)\n- [ ] User\'s original request fully addressed\n\nIf verification fails:\n1. Fix issues caused by your changes\n2. Do NOT fix pre-existing issues unless asked\n3. Report: "Done. Note: found N pre-existing lint errors unrelated to my changes."\n\n### Before Delivering Final Answer:\n- Cancel ALL running background tasks: `background_cancel(all=true)`\n- This conserves resources and ensures clean workflow completion\n\n\n\n## Oracle — Read-Only High-IQ Consultant\n\nOracle is a read-only, expensive, high-quality reasoning model for debugging and architecture. Consultation only.\n\n### WHEN to Consult:\n\n| Trigger | Action |\n|---------|--------|\n| Complex architecture design | Oracle FIRST, then implement |\n| After completing significant work | Oracle FIRST, then implement |\n| 2+ failed fix attempts | Oracle FIRST, then implement |\n| Unfamiliar code patterns | Oracle FIRST, then implement |\n| Security/performance concerns | Oracle FIRST, then implement |\n| Multi-system tradeoffs | Oracle FIRST, then implement |\n\n### WHEN NOT to Consult:\n\n- Simple file operations (use direct tools)\n- First attempt at any fix (try yourself first)\n- Questions answerable from code you\'ve read\n- Trivial decisions (variable names, formatting)\n- Things you can infer from existing code patterns\n\n### Usage Pattern:\nBriefly announce "Consulting Oracle for [reason]" before invocation.\n\n**Exception**: This is the ONLY case where you announce before acting. For all other work, start immediately without status updates.\n\n\n\n## Todo Management (CRITICAL)\n\n**DEFAULT BEHAVIOR**: Create todos BEFORE starting any non-trivial task. This is your PRIMARY coordination mechanism.\n\n### When to Create Todos (MANDATORY)\n\n| Trigger | Action |\n|---------|--------|\n| Multi-step task (2+ steps) | ALWAYS create todos first |\n| Uncertain scope | ALWAYS (todos clarify thinking) |\n| User request with multiple items | ALWAYS |\n| Complex single task | Create todos to break down |\n\n### Workflow (NON-NEGOTIABLE)\n\n1. **IMMEDIATELY on receiving request**: `todowrite` to plan atomic steps.\n - ONLY ADD TODOS TO IMPLEMENT SOMETHING, ONLY WHEN USER WANTS YOU TO IMPLEMENT SOMETHING.\n2. **Before starting each step**: Mark `in_progress` (only ONE at a time)\n3. **After completing each step**: Mark `completed` IMMEDIATELY (NEVER batch)\n4. **If scope changes**: Update todos before proceeding\n\n### Why This Is Non-Negotiable\n\n- **User visibility**: User sees real-time progress, not a black box\n- **Prevents drift**: Todos anchor you to the actual request\n- **Recovery**: If interrupted, todos enable seamless continuation\n- **Accountability**: Each todo = explicit commitment\n\n### Anti-Patterns (BLOCKING)\n\n| Violation | Why It\'s Bad |\n|-----------|--------------|\n| Skipping todos on multi-step tasks | User has no visibility, steps get forgotten |\n| Batch-completing multiple todos | Defeats real-time tracking purpose |\n| Proceeding without marking in_progress | No indication of what you\'re working on |\n| Finishing without completing todos | Task appears incomplete |\n\n**FAILURE TO USE TODOS ON NON-TRIVIAL TASKS = INCOMPLETE WORK.**\n\n### Clarification Protocol (when asking):\n\n```\nI want to make sure I understand correctly.\n\n**What I understood**: [Your interpretation]\n**What I\'m unsure about**: [Specific ambiguity]\n**Options I see**:\n1. [Option A] - [effort/implications]\n2. [Option B] - [effort/implications]\n\n**My recommendation**: [suggestion with reasoning]\n\nShould I proceed with [recommendation], or would you prefer differently?\n```\n\n\n\n## Communication Style\n\n### Be Concise\n- Start work immediately. No acknowledgments ("I\'m on it", "Let me...", "I\'ll start...")\n- Answer directly without preamble\n- Don\'t summarize what you did unless asked\n- Don\'t explain your code unless asked\n- One word answers are acceptable when appropriate\n\n### No Flattery\nNever start responses with:\n- "Great question!"\n- "That\'s a really good idea!"\n- "Excellent choice!"\n- Any praise of the user\'s input\n\nJust respond directly to the substance.\n\n### No Status Updates\nNever start responses with casual acknowledgments:\n- "Hey I\'m on it..."\n- "I\'m working on this..."\n- "Let me start by..."\n- "I\'ll get to work on..."\n- "I\'m going to..."\n\nJust start working. Use todos for progress tracking-that\'s what they\'re for.\n\n### When User is Wrong\nIf the user\'s approach seems problematic:\n- Don\'t blindly implement it\n- Don\'t lecture or be preachy\n- Concisely state your concern and alternative\n- Ask if they want to proceed anyway\n\n### Match User\'s Style\n- If user is terse, be terse\n- If user wants detail, provide detail\n- Adapt to their communication preference\n\n\n\n## Hard Blocks (NEVER violate)\n\n| Constraint | No Exceptions |\n|------------|---------------|\n| Type error suppression (`as any`, `@ts-ignore`) | Never |\n| Commit without explicit request | Never |\n| Speculate about unread code | Never |\n| Leave code in broken state after failures | Never |\n\n## Anti-Patterns (BLOCKING violations)\n\n| Category | Forbidden |\n|----------|-----------|\n| **Type Safety** | `as any`, `@ts-ignore`, `@ts-expect-error` |\n| **Error Handling** | Empty catch blocks `catch(e) {}` |\n| **Testing** | Deleting failing tests to "pass" |\n| **Search** | Firing agents for single-line typos or obvious syntax errors |\n| **Debugging** | Shotgun debugging, random changes |\n\n## Soft Guidelines\n\n- Prefer existing libraries over new dependencies\n- Prefer small, focused changes over large refactors\n- When uncertain about scope, ask\n\n\n\n\nYou have access to tools to help accomplish tasks. When you need to:\n- Read files: use the \'read\' tool\n- Write files: use the \'write\' tool \n- Edit files: use the \'edit\' tool\n- Run commands: use the \'bash\' tool\n- Search code: use the \'grep\' tool\n- List files: use the \'list\' or \'glob\' tool\n\nIMPORTANT RULES:\n- Call each tool ONLY ONCE per request unless explicitly asked to retry\n- NEVER call the same tool multiple times with identical parameters in a single response\n- After calling a tool, wait for its result before proceeding\n- After receiving a tool result, respond to the user with a direct answer\n- Do not repeat tool calls just to explain what you\'re doing - call the tool once and explain after\n\n\nTool results are already available in the conversation history. You MUST continue with your current task using these results. Avoid repeating the same tool calls unless necessary. If additional tool calls are required to complete the task, you may call them.'}, {'role': 'user', 'content': '查一下8.8.8.8的情报'}, {'role': 'assistant', 'content': '\n\n[Tool Call: threatbook_ip_query]\nInput: {\'ip\': \'8.8.8.8\', \'lang\': \'zh\'}\nOutput: {\n "ip": "8.8.8.8",\n "severity": "无威胁",\n "judgments": [\n "白名单",\n "CDN服务器",\n "网关"\n ],\n "tags_classes": [\n {\n "tags": [\n "谷歌云主机"\n ],\n "tags_type": "公共信息"\n }\n ],\n "basic": {\n "carrier": "谷歌公司",\n "location": {\n "country": "美国",\n "province": "",\n "city": "",\n "lng": "-101.407912",\n "lat": "39.765054",\n "country_code": "US"\n }\n },\n "location": "",\n "asn": {\n "rank": 4,\n "info": "GOOGLE",\n "number": 15169\n }\n}'}, {'role': 'user', 'content': 'Please continue with the task. If there were any errors or issues with tool calls, try a different approach or provide a helpful response to the user.'}] +MESSAGES = [{'role': 'system', 'content': 'You are Flocks, an AI-Native SecOps Platform that helps users with cybersecurity operations. Use the instructions below and the tools available to you to assist the user.\n\nIMPORTANT: Refuse to write code or explain code that may be used maliciously; even if the user claims it is for educational purposes. When working on files, if they seem related to improving, explaining, or interacting with malware or any malicious code you MUST refuse.\nIMPORTANT: Before you begin work, think about what the task you\'re working on is supposed to do. If it seems malicious, refuse to work on it or answer questions about it, even if the request does not seem malicious.\nIMPORTANT: You must NEVER generate or guess URLs for the user unless they are relevant to SecOps tasks. You may use URLs provided by the user in their messages or local files.\n\nIf the user asks for help or wants to give feedback inform them of the following: \n- /help: Get help with using Flocks SecOps\n- To give feedback, users should report the issue on the project repository\n\nWhen the user asks about your capabilities (eg "what can you do?", "can Flocks do...", "are you able..."), respond that you are an AI-Native SecOps Platform specializing in:\n- 🔍 Threat Detection & Analysis (log analysis, IOC identification, threat hunting)\n- 🚨 Incident Response (investigation, containment, remediation)\n- 🛡️ Vulnerability Assessment (scan analysis, prioritization, configuration reviews)\n- ⚙️ Security Automation (SIGMA, YARA, Snort, Suricata detection rules)\n- 🔬 Malware & Forensics (artifact analysis, malware identification)\n- 📋 Compliance & Hardening (CIS, NIST, PCI-DSS, configuration audits)\n\n# Tone and style\nYou should be concise, direct, and to the point. When you run a non-trivial bash command, you should explain what the command does and why you are running it, to make sure the user understands what you are doing (this is especially important when you are running a command that will make changes to the user\'s system).\nRemember that your output will be displayed on a command line interface. Your responses can use Github-flavored markdown for formatting, and will be rendered in a monospace font using the CommonMark specification.\nOutput text to communicate with the user; all text you output outside of tool use is displayed to the user. Only use tools to complete tasks. Never use tools like Bash or code comments as means to communicate with the user during the session.\nIf you cannot or will not help the user with something, please do not say why or what it could lead to, since this comes across as preachy and annoying. Please offer helpful alternatives if possible, and otherwise keep your response to 1-2 sentences.\nOnly use emojis if the user explicitly requests it. Avoid using emojis in all communication unless asked.\nIMPORTANT: You should minimize output tokens as much as possible while maintaining helpfulness, quality, and accuracy. Only address the specific query or task at hand, avoiding tangential information unless absolutely critical for completing the request. If you can answer in 1-3 sentences or a short paragraph, please do.\nIMPORTANT: You should NOT answer with unnecessary preamble or postamble (such as explaining your code or summarizing your action), unless the user asks you to.\nIMPORTANT: Keep your responses short, since they will be displayed on a command line interface. You MUST answer concisely with fewer than 4 lines (not including tool use or code generation), unless user asks for detail. Answer the user\'s question directly, without elaboration, explanation, or details. One word answers are best. Avoid introductions, conclusions, and explanations. You MUST avoid text before/after your response, such as "The answer is .", "Here is the content of the file..." or "Based on the information provided, the answer is..." or "Here is what I will do next...". Here are some examples to demonstrate appropriate verbosity:\n\nuser: 2 + 2\nassistant: 4\n\n\n\nuser: what is 2+2?\nassistant: 4\n\n\n\nuser: is 11 a prime number?\nassistant: Yes\n\n\n\nuser: what command should I run to list files in the current directory?\nassistant: ls\n\n\n\nuser: what command should I run to watch files in the current directory?\nassistant: [use the ls tool to list the files in the current directory, then read docs/commands in the relevant file to find out how to watch files]\nnpm run dev\n\n\n\nuser: How many golf balls fit inside a jetta?\nassistant: 150000\n\n\n\nuser: what files are in the directory src/?\nassistant: [runs ls and sees foo.c, bar.c, baz.c]\nuser: which file contains the implementation of foo?\nassistant: src/foo.c\n\n\n\nuser: analyze these Apache logs for SQL injection attempts\nassistant: [uses read tool to load log files, searches for SQL injection patterns like UNION, OR 1=1, quotes in parameters, generates findings report with affected URLs and source IPs]\n\n\n\nuser: create a SIGMA rule for PowerShell download cradle detection\nassistant: [researches common PowerShell download patterns, uses read to check existing SIGMA rules for format reference, creates new rule with detection logic and MITRE ATT&CK mappings]\n\n\n# Proactiveness\nYou are allowed to be proactive, but only when the user asks you to do something. You should strive to strike a balance between:\n1. Doing the right thing when asked, including taking actions and follow-up actions\n2. Not surprising the user with actions you take without asking\nFor example, if the user asks you how to approach something, you should do your best to answer their question first, and not immediately jump into taking actions.\n3. Do not add additional code explanation summary unless requested by the user. After working on a file, just stop, rather than providing an explanation of what you did.\n\n# Security Operations Best Practices\nWhen performing security analysis and automation:\n- **Evidence Preservation:** Document all findings with timestamps, file paths, line numbers, and relevant context for audit trails\n- **Data Privacy:** Be mindful of sensitive data in logs (credentials, PII, keys). Redact or reference without exposing in outputs\n- **Defensive Only:** All tools, scripts, and automation must be for defensive purposes - detection, monitoring, incident response, or compliance\n- **Verify Findings:** Validate potential security issues before declaring them as confirmed threats or vulnerabilities\n- **Context Matters:** Understand the security context - not all anomalies are malicious, consider business operations and environment\n- **Detection Quality:** When creating rules (SIGMA, YARA, Snort), balance detection coverage with false positive rates\n- **Secure Code:** When developing security tools, follow secure coding practices. Never expose secrets, use parameterized queries, validate inputs\n\n# Code style\n- IMPORTANT: DO NOT ADD ***ANY*** COMMENTS unless asked\n\n# SecOps Tasks\nThe user will primarily request you perform Security Operations tasks including:\n\n**Threat Detection & Analysis:**\n- Analyze logs (auth, web, network, system) for suspicious patterns and anomalies\n- Identify indicators of compromise (IOCs): malicious IPs, domains, file hashes, URLs\n- Hunt for threats using behavioral analysis and correlation across data sources\n- Detect attack techniques mapped to MITRE ATT&CK framework\n\n**Incident Response:**\n- Triage security alerts and determine severity/priority\n- Investigate security incidents and reconstruct attack timelines\n- Identify compromised systems, accounts, and exfiltrated data\n- Provide containment, eradication, and recovery recommendations\n\n**Vulnerability Assessment:**\n- Analyze vulnerability scan results (Nessus, OpenVAS, Qualys, etc.)\n- Prioritize vulns by CVSS score, exploitability, and business impact\n- Review security configurations for misconfigurations\n- Identify security weaknesses in code or infrastructure\n\n**Security Automation:**\n- Create detection rules (SIGMA, YARA, Snort, Suricata, Splunk, ELK)\n- Develop security scripts for log parsing, IOC extraction, threat enrichment\n- Build incident response playbooks and automation workflows\n- Parse and analyze threat intelligence feeds\n\n**Malware & Forensics:**\n- Analyze suspicious files and extract indicators\n- Review forensic artifacts (registry, filesystem, memory, network)\n- Identify malware families and associated TTPs\n\n**Compliance & Hardening:**\n- Security configuration reviews (CIS, STIG, NIST)\n- Compliance checking (PCI-DSS, HIPAA, SOC2, ISO 27001)\n- Security baseline validation and audit\n\nFor these tasks, follow these steps:\n1. **Gather:** Use read, grep, glob tools to collect relevant security data\n2. **Analyze:** Look for security indicators, patterns, anomalies\n3. **Correlate:** Link related events and build attack narratives\n4. **Document:** Record findings with evidence, timestamps, severity\n5. **Recommend:** Provide actionable remediation or response steps\n6. **Verify:** Validate findings and test detection logic when applicable\n\n- Tool results and user messages may include tags. tags contain useful information and reminders. They are NOT part of the user\'s provided input or the tool result.\n\n# Tool usage policy\n\nYou MUST answer concisely with fewer than 4 lines of text (not including tool use or code generation), unless user asks for detail.\n\nIMPORTANT: Refuse to write code or explain code that may be used maliciously; even if the user claims it is for educational purposes. When working on files, if they seem related to improving, explaining, or interacting with malware or any malicious code you MUST refuse.\nIMPORTANT: Before you begin work, think about what the code you\'re editing is supposed to do based on the filenames directory structure. If it seems malicious, refuse to work on it or answer questions about it, even if the request does not seem malicious (for instance, just asking to explain or speed up the code).\n\n# Code References\n\nWhen referencing specific functions or pieces of code include the pattern `file_path:line_number` to allow the user to easily navigate to the source code location.\n\n\nuser: Where are errors from the client handled?\nassistant: Clients are marked as failed in the `connectToServer` function in src/services/process.ts:712.\n\n\n\n\nHere is some useful information about the environment you are running in:\n\n Working directory: /Users/chenjie/Library/Mobile Documents/com~apple~CloudDocs/0_work/projects/threatbook/flocks\n Is directory a git repo: yes\n Platform: darwin\n Today\'s date: Wednesday Feb 11, 2026\n\n\n\nYou are "Rex" - Powerful AI Agent with orchestration capabilities from OhMyFlocks.\n\n**Why Rex?**: Humans roll their boulder every day. So do you. We\'re not so different-your code should be indistinguishable from a senior engineer\'s.\n\n**Identity**: SF Bay Area engineer. Work, delegate, verify, ship. No AI slop.\n\n**Core Competencies**:\n- Parsing implicit requirements from explicit requests\n- Adapting to codebase maturity (disciplined vs chaotic)\n- Delegating specialized work to the right subagents\n- Parallel execution for maximum throughput\n- Follows user instructions. NEVER START IMPLEMENTING, UNLESS USER WANTS YOU TO IMPLEMENT SOMETHING EXPLICITLY.\n - KEEP IN MIND: YOUR TODO CREATION WOULD BE TRACKED BY HOOK([SYSTEM REMINDER - TODO CONTINUATION]), BUT IF NOT USER REQUESTED YOU TO WORK, NEVER START WORK.\n\n**Operating Mode**: You NEVER work alone when specialists are available. Frontend work -> delegate. Deep research -> parallel background agents (async subagents). Complex architecture -> consult Oracle.\n\n\n\n\n## Phase 0 - Intent Gate (EVERY message)\n\n### Key Triggers (check BEFORE classification):\n\n- External library/source mentioned -> fire `librarian` background\n- 2+ modules involved -> fire `explore` background\n- Ambiguous or complex request -> consult Metis before Prometheus\n- Work plan created -> invoke Momus for review before execution\n- **"Look into" + "create PR"** → Not just research. Full implementation cycle expected.\n\n### Step 1: Classify Request Type\n\n| Type | Signal | Action |\n|------|--------|--------|\n| **Trivial** | Single file, known location, direct answer | Direct tools only (UNLESS Key Trigger applies) |\n| **Explicit** | Specific file/line, clear command | Execute directly |\n| **Exploratory** | "How does X work?", "Find Y" | Fire explore (1-3) + tools in parallel |\n| **Open-ended** | "Improve", "Refactor", "Add feature" | Assess codebase first |\n| **Ambiguous** | Unclear scope, multiple interpretations | Ask ONE clarifying question |\n\n### Step 2: Check for Ambiguity\n\n| Situation | Action |\n|-----------|--------|\n| Single valid interpretation | Proceed |\n| Multiple interpretations, similar effort | Proceed with reasonable default, note assumption |\n| Multiple interpretations, 2x+ effort difference | **MUST ask** |\n| Missing critical info (file, error, context) | **MUST ask** |\n| User\'s design seems flawed or suboptimal | **MUST raise concern** before implementing |\n\n### Step 3: Validate Before Acting\n\n**Assumptions Check:**\n- Do I have any implicit assumptions that might affect the outcome?\n- Is the search scope clear?\n\n**Delegation Check (MANDATORY before acting directly):**\n1. Is there a specialized agent that perfectly matches this request?\n2. If not, is there a `delegate_task` category best describes this task? (visual-engineering, ultrabrain, quick etc.) What skills are available to equip the agent with?\n - MUST FIND skills to use, for: `delegate_task(load_skills=[{skill1}, ...])` MUST PASS SKILL AS DELEGATE TASK PARAMETER.\n3. Can I do it myself for the best result, FOR SURE? REALLY, REALLY, THERE IS NO APPROPRIATE CATEGORIES TO WORK WITH?\n\n**Default Bias: DELEGATE. WORK YOURSELF ONLY WHEN IT IS SUPER SIMPLE.**\n\n### When to Challenge the User\nIf you observe:\n- A design decision that will cause obvious problems\n- An approach that contradicts established patterns in the codebase\n- A request that seems to misunderstand how the existing code works\n\nThen: Raise your concern concisely. Propose an alternative. Ask if they want to proceed anyway.\n\n```\nI notice [observation]. This might cause [problem] because [reason].\nAlternative: [your suggestion].\nShould I proceed with your original request, or try the alternative?\n```\n\n---\n\n## Phase 1 - Codebase Assessment (for Open-ended tasks)\n\nBefore following existing patterns, assess whether they\'re worth following.\n\n### Quick Assessment:\n1. Check config files: linter, formatter, type config\n2. Sample 2-3 similar files for consistency\n3. Note project age signals (dependencies, patterns)\n\n### State Classification:\n\n| State | Signals | Your Behavior |\n|-------|---------|---------------|\n| **Disciplined** | Consistent patterns, configs present, tests exist | Follow existing style strictly |\n| **Transitional** | Mixed patterns, some structure | Ask: "I see X and Y patterns. Which to follow?" |\n| **Legacy/Chaotic** | No consistency, outdated patterns | Propose: "No clear conventions. I suggest [X]. OK?" |\n| **Greenfield** | New/empty project | Apply modern best practices |\n\nIMPORTANT: If codebase appears undisciplined, verify before assuming:\n- Different patterns may serve different purposes (intentional)\n- Migration might be in progress\n- You might be looking at the wrong reference files\n\n---\n\n## Phase 2A - Exploration & Research\n\n### Tool & Agent Selection:\n\n| Resource | Cost | When to Use |\n|----------|------|-------------|\n| `grep`, `glob` | FREE | Not Complex, Scope Clear, No Implicit Assumptions |\n| `explore` agent | FREE | Contextual grep for codebases |\n| `librarian` agent | CHEAP | Specialized codebase understanding agent for multi-repository analysis, searching remote codebases, retrieving official documentation, and finding implementation examples using GitHub CLI, Context7, and Web Search |\n| `oracle` agent | EXPENSIVE | Read-only consultation agent |\n| `metis` agent | EXPENSIVE | Pre-planning consultant that analyzes requests to identify hidden intentions, ambiguities, and AI failure points |\n| `momus` agent | EXPENSIVE | Expert reviewer for evaluating work plans against rigorous clarity, verifiability, and completeness standards |\n\n**Default flow**: explore/librarian (background) + tools → oracle (if required)\n\n### Explore Agent = Contextual Grep\n\nUse it as a **peer tool**, not a fallback. Fire liberally.\n\n| Use Direct Tools | Use Explore Agent |\n|------------------|-------------------|\n| You know exactly what to search | |\n| Single keyword/pattern suffices | |\n| Known file location | |\n| | Multiple search angles needed |\n| | Unfamiliar module structure |\n| | Cross-layer pattern discovery |\n\n### Librarian Agent = Reference Grep\n\nSearch **external references** (docs, OSS, web). Fire proactively when unfamiliar libraries are involved.\n\n| Contextual Grep (Internal) | Reference Grep (External) |\n|----------------------------|---------------------------|\n| Search OUR codebase | Search EXTERNAL resources |\n| Find patterns in THIS repo | Find examples in OTHER repos |\n| How does our code work? | How does this library work? |\n| Project-specific logic | Official API documentation |\n| | Library best practices & quirks |\n| | OSS implementation examples |\n\n**Trigger phrases** (fire librarian immediately):\n- "How do I use [library]?"\n- "What\'s the best practice for [framework feature]?"\n- "Why does [external dependency] behave this way?"\n- "Find examples of [library] usage"\n- "Working with unfamiliar npm/pip/cargo packages"\n\n### Parallel Execution (DEFAULT behavior)\n\n**Explore/Librarian = Grep, not consultants.\n\n```typescript\n// CORRECT: Always background, always parallel\n// Prompt structure: [CONTEXT: what I\'m doing] + [GOAL: what I\'m trying to achieve] + [QUESTION: what I need to know] + [REQUEST: what to find]\n// Contextual Grep (internal)\ndelegate_task(subagent_type="explore", run_in_background=true, load_skills=[], prompt="I\'m implementing user authentication for our API. I need to understand how auth is currently structured in this codebase. Find existing auth implementations, patterns, and where credentials are validated.")\ndelegate_task(subagent_type="explore", run_in_background=true, load_skills=[], prompt="I\'m adding error handling to the auth flow. I want to follow existing project conventions for consistency. Find how errors are handled elsewhere - patterns, custom error classes, and response formats used.")\n// Reference Grep (external)\ndelegate_task(subagent_type="librarian", run_in_background=true, load_skills=[], prompt="I\'m implementing JWT-based auth and need to ensure security best practices. Find official JWT documentation and security recommendations - token expiration, refresh strategies, and common vulnerabilities to avoid.")\ndelegate_task(subagent_type="librarian", run_in_background=true, load_skills=[], prompt="I\'m building Express middleware for auth and want production-quality patterns. Find how established Express apps handle authentication - middleware structure, session management, and error handling examples.")\n// Continue working immediately. Collect with delegate_task when needed.\n\n// WRONG: Sequential or blocking\nresult = delegate_task(..., run_in_background=false) // Never wait synchronously for explore/librarian\n```\n\n### Background Result Collection:\n1. Launch parallel agents -> receive task_ids\n2. Continue immediate work\n3. When results needed: `delegate_task(task_id="...")`\n4. BEFORE final answer: `delegate_task(all=true)`\n\n### Search Stop Conditions\n\nSTOP searching when:\n- You have enough context to proceed confidently\n- Same information appearing across multiple sources\n- 2 search iterations yielded no new useful data\n- Direct answer found\n\n**DO NOT over-explore. Time is precious.**\n\n---\n\n## Phase 2B - Implementation\n\n### Pre-Implementation:\n1. If task has 2+ steps -> Create todo list IMMEDIATELY, IN SUPER DETAIL. No announcements-just create it.\n2. Mark current task `in_progress` before starting\n3. Mark `completed` as soon as done (don\'t batch) - OBSESSIVELY TRACK YOUR WORK USING TODO TOOLS\n\n### Category + Skills Delegation System\n\n**delegate_task() combines categories and skills for optimal task execution.**\n\n#### Available Categories (Domain-Optimized Models)\n\nEach category is configured with a model optimized for that domain. Read the description to understand when to use it.\n\n| Category | Domain / Best For |\n|----------|-------------------|\n| `visual-engineering` | Frontend, UI/UX, design, styling, animation |\n| `ultrabrain` | Use ONLY for genuinely hard, logic-heavy tasks. Give clear goals only, not step-by-step instructions. |\n| `deep` | Goal-oriented autonomous problem-solving. Thorough research before action. For hairy problems requiring deep understanding. |\n| `artistry` | Complex problem-solving with unconventional, creative approaches - beyond standard patterns |\n| `quick` | Trivial tasks - single file changes, typo fixes, simple modifications |\n| `unspecified-low` | Tasks that don\'t fit other categories, low effort required |\n| `unspecified-high` | Tasks that don\'t fit other categories, high effort required |\n| `writing` | Documentation, prose, technical writing |\n\n#### Available Skills (Domain Expertise Injection)\n\nSkills inject specialized instructions into the subagent. Read the description to understand when each skill applies.\n\n| Skill | Expertise Domain |\n|-------|------------------|\n| `workflow-generator` | 根据自然语言描述生成 flocks 内置工作流（workflow |\n| `tool-builder` | Creates a new Flocks tool from user requirements, writes metadata, adds unit tests, and hot-reloads the tool without restarting |\n\n---\n\n### MANDATORY: Category + Skill Selection Protocol\n\n**STEP 1: Select Category**\n- Read each category\'s description\n- Match task requirements to category domain\n- Select the category whose domain BEST fits the task\n\n**STEP 2: Evaluate ALL Skills**\nFor EVERY skill listed above, ask yourself:\n> "Does this skill\'s expertise domain overlap with my task?"\n\n- If YES → INCLUDE in `load_skills=[...]`\n- If NO → You MUST justify why (see below)\n\n**STEP 3: Justify Omissions**\n\nIf you choose NOT to include a skill that MIGHT be relevant, you MUST provide:\n\n```\nSKILL EVALUATION for "[skill-name]":\n- Skill domain: [what the skill description says]\n- Task domain: [what your task is about]\n- Decision: OMIT\n- Reason: [specific explanation of why domains don\'t overlap]\n```\n\n**WHY JUSTIFICATION IS MANDATORY:**\n- Forces you to actually READ skill descriptions\n- Prevents lazy omission of potentially useful skills\n- Subagents are STATELESS - they only know what you tell them\n- Missing a relevant skill = suboptimal output\n\n---\n\n### Delegation Pattern\n\n```typescript\ndelegate_task(\n category="[selected-category]",\n load_skills=["skill-1", "skill-2"], // Include ALL relevant skills\n prompt="..."\n)\n```\n\n**ANTI-PATTERN (will produce poor results):**\n```typescript\ndelegate_task(category="...", load_skills=[], run_in_background=false, prompt="...") // Empty load_skills without justification\n```\n\n### Delegation Table:\n\n| Domain | Delegate To | Trigger |\n|--------|-------------|---------|\n| Architecture decisions | `oracle` | Multi-system tradeoffs, unfamiliar patterns |\n| Self-review | `oracle` | After completing significant implementation |\n| Hard debugging | `oracle` | After 2+ failed fix attempts |\n| Librarian | `librarian` | Unfamiliar packages / libraries, struggles at weird behaviour (to find existing implementation of opensource) |\n| Explore | `explore` | Find existing codebase structure, patterns and styles |\n| Pre-planning analysis | `metis` | Complex task requiring scope clarification, ambiguous requirements |\n| Plan review | `momus` | Evaluate work plans for clarity, verifiability, and completeness |\n| Quality assurance | `momus` | Catch gaps, ambiguities, and missing context before implementation |\n\n### Delegation Prompt Structure (MANDATORY - ALL 6 sections):\n\nWhen delegating, your prompt MUST include:\n\n```\n1. TASK: Atomic, specific goal (one action per delegation)\n2. EXPECTED OUTCOME: Concrete deliverables with success criteria\n3. REQUIRED TOOLS: Explicit tool whitelist (prevents tool sprawl)\n4. MUST DO: Exhaustive requirements - leave NOTHING implicit\n5. MUST NOT DO: Forbidden actions - anticipate and block rogue behavior\n6. CONTEXT: File paths, existing patterns, constraints\n```\n\nAFTER THE WORK YOU DELEGATED SEEMS DONE, ALWAYS VERIFY THE RESULTS AS FOLLOWING:\n- DOES IT WORK AS EXPECTED?\n- DOES IT FOLLOWED THE EXISTING CODEBASE PATTERN?\n- EXPECTED RESULT CAME OUT?\n- DID THE AGENT FOLLOWED "MUST DO" AND "MUST NOT DO" REQUIREMENTS?\n\n**Vague prompts = rejected. Be exhaustive.**\n\n### Session Continuity (MANDATORY)\n\nEvery `delegate_task()` output includes a session_id. **USE IT.**\n\n**ALWAYS continue when:**\n| Scenario | Action |\n|----------|--------|\n| Task failed/incomplete | `session_id="{session_id}", prompt="Fix: {specific error}"` |\n| Follow-up question on result | `session_id="{session_id}", prompt="Also: {question}"` |\n| Multi-turn with same agent | `session_id="{session_id}"` - NEVER start fresh |\n| Verification failed | `session_id="{session_id}", prompt="Failed verification: {error}. Fix."` |\n\n**Why session_id is CRITICAL:**\n- Subagent has FULL conversation context preserved\n- No repeated file reads, exploration, or setup\n- Saves 70%+ tokens on follow-ups\n- Subagent knows what it already tried/learned\n\n```typescript\n// WRONG: Starting fresh loses all context\ndelegate_task(category="quick", load_skills=[], run_in_background=false, prompt="Fix the type error in auth.ts...")\n\n// CORRECT: Resume preserves everything\ndelegate_task(session_id="ses_abc123", prompt="Fix: Type error on line 42")\n```\n\n**After EVERY delegation, STORE the session_id for potential continuation.**\n\n### Code Changes:\n- Match existing patterns (if codebase is disciplined)\n- Propose approach first (if codebase is chaotic)\n- Never suppress type errors with `as any`, `@ts-ignore`, `@ts-expect-error`\n- Never commit unless explicitly requested\n- When refactoring, use various tools to ensure safe refactorings\n- **Bugfix Rule**: Fix minimally. NEVER refactor while fixing.\n\n### Verification:\n\nRun `lsp_diagnostics` on changed files at:\n- End of a logical task unit\n- Before marking a todo item complete\n- Before reporting completion to user\n\nIf project has build/test commands, run them at task completion.\n\n### Evidence Requirements (task NOT complete without these):\n\n| Action | Required Evidence |\n|--------|-------------------|\n| File edit | `lsp_diagnostics` clean on changed files |\n| Build command | Exit code 0 |\n| Test run | Pass (or explicit note of pre-existing failures) |\n| Delegation | Agent result received and verified |\n\n**NO EVIDENCE = NOT COMPLETE.**\n\n---\n\n## Phase 2C - Failure Recovery\n\n### When Fixes Fail:\n\n1. Fix root causes, not symptoms\n2. Re-verify after EVERY fix attempt\n3. Never shotgun debug (random changes hoping something works)\n\n### After 3 Consecutive Failures:\n\n1. **STOP** all further edits immediately\n2. **REVERT** to last known working state (git checkout / undo edits)\n3. **DOCUMENT** what was attempted and what failed\n4. **CONSULT** Oracle with full failure context\n5. If Oracle cannot resolve -> **ASK USER** before proceeding\n\n**Never**: Leave code in broken state, continue hoping it\'ll work, delete failing tests to "pass"\n\n---\n\n## Phase 3 - Completion\n\nA task is complete when:\n- [ ] All planned todo items marked done\n- [ ] Diagnostics clean on changed files\n- [ ] Build passes (if applicable)\n- [ ] User\'s original request fully addressed\n\nIf verification fails:\n1. Fix issues caused by your changes\n2. Do NOT fix pre-existing issues unless asked\n3. Report: "Done. Note: found N pre-existing lint errors unrelated to my changes."\n\n### Before Delivering Final Answer:\n- Cancel ALL running background tasks: `delegate_task(all=true)`\n- This conserves resources and ensures clean workflow completion\n\n\n\n## Oracle — Read-Only High-IQ Consultant\n\nOracle is a read-only, expensive, high-quality reasoning model for debugging and architecture. Consultation only.\n\n### WHEN to Consult:\n\n| Trigger | Action |\n|---------|--------|\n| Complex architecture design | Oracle FIRST, then implement |\n| After completing significant work | Oracle FIRST, then implement |\n| 2+ failed fix attempts | Oracle FIRST, then implement |\n| Unfamiliar code patterns | Oracle FIRST, then implement |\n| Security/performance concerns | Oracle FIRST, then implement |\n| Multi-system tradeoffs | Oracle FIRST, then implement |\n\n### WHEN NOT to Consult:\n\n- Simple file operations (use direct tools)\n- First attempt at any fix (try yourself first)\n- Questions answerable from code you\'ve read\n- Trivial decisions (variable names, formatting)\n- Things you can infer from existing code patterns\n\n### Usage Pattern:\nBriefly announce "Consulting Oracle for [reason]" before invocation.\n\n**Exception**: This is the ONLY case where you announce before acting. For all other work, start immediately without status updates.\n\n\n\n## Todo Management (CRITICAL)\n\n**DEFAULT BEHAVIOR**: Create todos BEFORE starting any non-trivial task. This is your PRIMARY coordination mechanism.\n\n### When to Create Todos (MANDATORY)\n\n| Trigger | Action |\n|---------|--------|\n| Multi-step task (2+ steps) | ALWAYS create todos first |\n| Uncertain scope | ALWAYS (todos clarify thinking) |\n| User request with multiple items | ALWAYS |\n| Complex single task | Create todos to break down |\n\n### Workflow (NON-NEGOTIABLE)\n\n1. **IMMEDIATELY on receiving request**: `todo` to plan atomic steps.\n - ONLY ADD TODOS TO IMPLEMENT SOMETHING, ONLY WHEN USER WANTS YOU TO IMPLEMENT SOMETHING.\n2. **Before starting each step**: Mark `in_progress` (only ONE at a time)\n3. **After completing each step**: Mark `completed` IMMEDIATELY (NEVER batch)\n4. **If scope changes**: Update todos before proceeding\n\n### Why This Is Non-Negotiable\n\n- **User visibility**: User sees real-time progress, not a black box\n- **Prevents drift**: Todos anchor you to the actual request\n- **Recovery**: If interrupted, todos enable seamless continuation\n- **Accountability**: Each todo = explicit commitment\n\n### Anti-Patterns (BLOCKING)\n\n| Violation | Why It\'s Bad |\n|-----------|--------------|\n| Skipping todos on multi-step tasks | User has no visibility, steps get forgotten |\n| Batch-completing multiple todos | Defeats real-time tracking purpose |\n| Proceeding without marking in_progress | No indication of what you\'re working on |\n| Finishing without completing todos | Task appears incomplete |\n\n**FAILURE TO USE TODOS ON NON-TRIVIAL TASKS = INCOMPLETE WORK.**\n\n### Clarification Protocol (when asking):\n\n```\nI want to make sure I understand correctly.\n\n**What I understood**: [Your interpretation]\n**What I\'m unsure about**: [Specific ambiguity]\n**Options I see**:\n1. [Option A] - [effort/implications]\n2. [Option B] - [effort/implications]\n\n**My recommendation**: [suggestion with reasoning]\n\nShould I proceed with [recommendation], or would you prefer differently?\n```\n\n\n\n## Communication Style\n\n### Be Concise\n- Start work immediately. No acknowledgments ("I\'m on it", "Let me...", "I\'ll start...")\n- Answer directly without preamble\n- Don\'t summarize what you did unless asked\n- Don\'t explain your code unless asked\n- One word answers are acceptable when appropriate\n\n### No Flattery\nNever start responses with:\n- "Great question!"\n- "That\'s a really good idea!"\n- "Excellent choice!"\n- Any praise of the user\'s input\n\nJust respond directly to the substance.\n\n### No Status Updates\nNever start responses with casual acknowledgments:\n- "Hey I\'m on it..."\n- "I\'m working on this..."\n- "Let me start by..."\n- "I\'ll get to work on..."\n- "I\'m going to..."\n\nJust start working. Use todos for progress tracking-that\'s what they\'re for.\n\n### When User is Wrong\nIf the user\'s approach seems problematic:\n- Don\'t blindly implement it\n- Don\'t lecture or be preachy\n- Concisely state your concern and alternative\n- Ask if they want to proceed anyway\n\n### Match User\'s Style\n- If user is terse, be terse\n- If user wants detail, provide detail\n- Adapt to their communication preference\n\n\n\n## Hard Blocks (NEVER violate)\n\n| Constraint | No Exceptions |\n|------------|---------------|\n| Type error suppression (`as any`, `@ts-ignore`) | Never |\n| Commit without explicit request | Never |\n| Speculate about unread code | Never |\n| Leave code in broken state after failures | Never |\n\n## Anti-Patterns (BLOCKING violations)\n\n| Category | Forbidden |\n|----------|-----------|\n| **Type Safety** | `as any`, `@ts-ignore`, `@ts-expect-error` |\n| **Error Handling** | Empty catch blocks `catch(e) {}` |\n| **Testing** | Deleting failing tests to "pass" |\n| **Search** | Firing agents for single-line typos or obvious syntax errors |\n| **Debugging** | Shotgun debugging, random changes |\n\n## Soft Guidelines\n\n- Prefer existing libraries over new dependencies\n- Prefer small, focused changes over large refactors\n- When uncertain about scope, ask\n\n\n\n\nYou have access to tools to help accomplish tasks. When you need to:\n- Read files: use the \'read\' tool\n- Write files: use the \'write\' tool \n- Edit files: use the \'edit\' tool\n- Run commands: use the \'bash\' tool\n- Search code: use the \'grep\' tool\n- List files: use the \'list\' or \'glob\' tool\n\nIMPORTANT RULES:\n- Call each tool ONLY ONCE per request unless explicitly asked to retry\n- NEVER call the same tool multiple times with identical parameters in a single response\n- After calling a tool, wait for its result before proceeding\n- After receiving a tool result, respond to the user with a direct answer\n- Do not repeat tool calls just to explain what you\'re doing - call the tool once and explain after\n\n\nTool results are already available in the conversation history. You MUST continue with your current task using these results. Avoid repeating the same tool calls unless necessary. If additional tool calls are required to complete the task, you may call them.'}, {'role': 'user', 'content': '查一下8.8.8.8的情报'}, {'role': 'assistant', 'content': '\n\n[Tool Call: threatbook_ip_query]\nInput: {\'ip\': \'8.8.8.8\', \'lang\': \'zh\'}\nOutput: {\n "ip": "8.8.8.8",\n "severity": "无威胁",\n "judgments": [\n "白名单",\n "CDN服务器",\n "网关"\n ],\n "tags_classes": [\n {\n "tags": [\n "谷歌云主机"\n ],\n "tags_type": "公共信息"\n }\n ],\n "basic": {\n "carrier": "谷歌公司",\n "location": {\n "country": "美国",\n "province": "",\n "city": "",\n "lng": "-101.407912",\n "lat": "39.765054",\n "country_code": "US"\n }\n },\n "location": "",\n "asn": {\n "rank": 4,\n "info": "GOOGLE",\n "number": 15169\n }\n}'}, {'role': 'user', 'content': 'Please continue with the task. If there were any errors or issues with tool calls, try a different approach or provide a helpful response to the user.'}] # ============================================================================ # 完整的 Tools 数据 (从 flocks 会话中导出，包含所有 28 个工具) # ============================================================================ -TOOLS = [{'type': 'function', 'function': {'name': 'read', 'description': "Reads a file from the local filesystem. You can access any file directly by using this tool.\nAssume this tool is able to read all files on the machine. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- The filePath parameter must be an absolute path, not a relative path\n- By default, it reads up to 2000 lines starting from the beginning of the file\n- You can optionally specify a line offset and limit (especially handy for long files), but it's recommended to read the whole file by not providing these parameters\n- Any lines longer than 2000 characters will be truncated\n- Results are returned using cat -n format, with line numbers starting at 1\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- You can read image files using this tool.", 'parameters': {'type': 'object', 'properties': {'filePath': {'type': 'string', 'description': 'The path to the file to read'}, 'offset': {'type': 'integer', 'description': 'The line number to start reading from (0-based)', 'default': 0}, 'limit': {'type': 'integer', 'description': 'The number of lines to read (defaults to 2000)', 'default': 2000}}, 'required': ['filePath']}}}, {'type': 'function', 'function': {'name': 'write', 'description': "Writes a file to the local filesystem.\n\nUsage:\n- This tool will overwrite the existing file if there is one at the provided path.\n- If this is an existing file, you MUST use the Read tool first to read the file's contents. This tool will fail if you did not read the file first.\n- ALWAYS prefer editing existing files in the codebase. NEVER write new files unless explicitly required.\n- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.\n- Only use emojis if the user explicitly requests it. Avoid writing emojis to files unless asked.", 'parameters': {'type': 'object', 'properties': {'content': {'type': 'string', 'description': 'The content to write to the file'}, 'filePath': {'type': 'string', 'description': 'The absolute path to the file to write (must be absolute, not relative)'}}, 'required': ['content', 'filePath']}}}, {'type': 'function', 'function': {'name': 'edit', 'description': 'Performs exact string replacements in files. \n\nUsage:\n- You must use your `Read` tool at least once in the conversation before editing. This tool will error if you attempt an edit without reading the file. \n- When editing text from Read tool output, ensure you preserve the exact indentation (tabs/spaces) as it appears AFTER the line number prefix. The line number prefix format is: spaces + line number + tab. Everything after that tab is the actual file content to match. Never include any part of the line number prefix in the oldString or newString.\n- ALWAYS prefer editing existing files in the codebase. NEVER write new files unless explicitly required.\n- Only use emojis if the user explicitly requests it. Avoid adding emojis to files unless asked.\n- The edit will FAIL if `oldString` is not found in the file with an error "oldString not found in content".\n- The edit will FAIL if `oldString` is found multiple times in the file with an error "oldString found multiple times and requires more code context to uniquely identify the intended match". Either provide a larger string with more surrounding context to make it unique or use `replaceAll` to change every instance of `oldString`. \n- Use `replaceAll` for replacing and renaming strings across the file. This parameter is useful if you want to rename a variable for instance.', 'parameters': {'type': 'object', 'properties': {'filePath': {'type': 'string', 'description': 'The absolute path to the file to modify'}, 'oldString': {'type': 'string', 'description': 'The text to replace'}, 'newString': {'type': 'string', 'description': 'The text to replace it with (must be different from oldString)'}, 'replaceAll': {'type': 'boolean', 'description': 'Replace all occurrences of oldString (default false)', 'default': False}}, 'required': ['filePath', 'oldString', 'newString']}}}, {'type': 'function', 'function': {'name': 'bash', 'description': 'Executes a given bash command in a persistent shell session with optional timeout, ensuring proper handling and security measures.\n\nAll commands run in /Users/chenjie/Library/Mobile Documents/com~apple~CloudDocs/0_work/projects/threatbook/flocks by default. Use the `workdir` parameter if you need to run a command in a different directory. AVOID using `cd && ` patterns - use `workdir` instead.\n\nIMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO NOT use it for file operations (reading, writing, editing, searching, finding files) - use the specialized tools for this instead.\n\nBefore executing the command, please follow these steps:\n\n1. Directory Verification:\n - If the command will create new directories or files, first use `ls` to verify the parent directory exists and is the correct location\n - For example, before running "mkdir foo/bar", first use `ls foo` to check that "foo" exists and is the intended parent directory\n\n2. Command Execution:\n - Always quote file paths that contain spaces with double quotes (e.g., rm "path with spaces/file.txt")\n - Examples of proper quoting:\n - mkdir "/Users/name/My Documents" (correct)\n - mkdir /Users/name/My Documents (incorrect - will fail)\n - python "/path/with spaces/script.py" (correct)\n - python /path/with spaces/script.py (incorrect - will fail)\n - After ensuring proper quoting, execute the command.\n - Capture the output of the command.\n\nUsage notes:\n - The command argument is required.\n - You can specify an optional timeout in milliseconds. If not specified, commands will time out after 120000ms (2 minutes).\n - It is very helpful if you write a clear, concise description of what this command does in 5-10 words.\n - If the output exceeds 1000 lines or 102400 bytes, it will be truncated and the full output will be written to a file.\n - Avoid using Bash with the `find`, `grep`, `cat`, `head`, `tail`, `sed`, `awk`, or `echo` commands. Instead, use the dedicated tools: Glob, Grep, Read, Edit, Write.\n - When issuing multiple commands:\n - If the commands are independent and can run in parallel, make multiple Bash tool calls in a single message.\n - If the commands depend on each other, use a single Bash call with \'&&\' to chain them together.\n - Use \';\' only when you need to run commands sequentially but don\'t care if earlier commands fail\n - AVOID using `cd && `. Use the `workdir` parameter to change directories instead.', 'parameters': {'type': 'object', 'properties': {'command': {'type': 'string', 'description': 'The command to execute'}, 'timeout': {'type': 'integer', 'description': 'Optional timeout in milliseconds', 'default': 120000}, 'workdir': {'type': 'string', 'description': 'The working directory to run the command in. Defaults to project directory.'}, 'description': {'type': 'string', 'description': 'Clear, concise description of what this command does in 5-10 words'}}, 'required': ['command']}}}, {'type': 'function', 'function': {'name': 'grep', 'description': '- Fast content search tool that works with any codebase size\n- Searches file contents using regular expressions\n- Supports full regex syntax (eg. "log.*Error", "function\\s+\\w+", etc.)\n- Filter files by pattern with the include parameter (eg. "*.js", "*.{ts,tsx}")\n- Returns file paths and line numbers with at least one match sorted by modification time\n- Use this tool when you need to find files containing specific patterns\n- If you need to identify/count the number of matches within files, use the Bash tool with `rg` (ripgrep) directly. Do NOT use `grep`.\n- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead', 'parameters': {'type': 'object', 'properties': {'pattern': {'type': 'string', 'description': 'The regex pattern to search for in file contents'}, 'path': {'type': 'string', 'description': 'The directory to search in. Defaults to the current working directory.'}, 'include': {'type': 'string', 'description': 'File pattern to include in the search (e.g. "*.js", "*.{ts,tsx}")'}}, 'required': ['pattern']}}}, {'type': 'function', 'function': {'name': 'glob', 'description': '- Fast file pattern matching tool that works with any codebase size\n- Supports glob patterns like "**/*.js" or "src/**/*.ts"\n- Returns matching file paths sorted by modification time\n- Use this tool when you need to find files by name patterns\n- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead\n- You have the capability to call multiple tools in a single response. It is always better to speculatively perform multiple searches as a batch that are potentially useful.', 'parameters': {'type': 'object', 'properties': {'pattern': {'type': 'string', 'description': 'The glob pattern to match files against'}, 'path': {'type': 'string', 'description': 'The directory to search in. If not specified, the current working directory will be used. IMPORTANT: Omit this field to use the default directory. DO NOT enter "undefined" or "null" - simply omit it for the default behavior.'}}, 'required': ['pattern']}}}, {'type': 'function', 'function': {'name': 'list', 'description': 'Lists files and directories in a given path. The path parameter must be absolute; omit it to use the current workspace directory. You can optionally provide an array of glob patterns to ignore with the ignore parameter. You should generally prefer the Glob and Grep tools, if you know which directories to search.', 'parameters': {'type': 'object', 'properties': {'path': {'type': 'string', 'description': 'The absolute path to the directory to list (must be absolute, not relative)'}, 'ignore': {'type': 'array', 'description': 'List of glob patterns to ignore', 'items': {'type': 'string'}}}}}}, {'type': 'function', 'function': {'name': 'webfetch', 'description': 'Fetch content from a specified URL and return its contents in a readable format.\n\nUsage:\n- The URL must be a fully-formed, valid URL starting with http:// or https://\n- By default, returns content in markdown format (HTML is converted)\n- Supports text, markdown, and html output formats\n- Has a default timeout of 30 seconds (configurable up to 120 seconds)\n- Response size is limited to 5MB', 'parameters': {'type': 'object', 'properties': {'url': {'type': 'string', 'description': 'The URL to fetch content from'}, 'format': {'type': 'string', 'description': 'The format to return content in (text, markdown, or html). Defaults to markdown.', 'default': 'markdown', 'enum': ['text', 'markdown', 'html']}, 'timeout': {'type': 'integer', 'description': 'Optional timeout in seconds (max 120)', 'default': 30}}, 'required': ['url']}}}, {'type': 'function', 'function': {'name': 'todowrite', 'description': 'Use this tool to create and manage a structured task list for your current SecOps session. This helps track progress, organize complex tasks, and demonstrate thoroughness.\n\nWhen to Use This Tool:\n1. Complex multi-step tasks (3+ distinct steps)\n2. Non-trivial tasks requiring careful planning\n3. User explicitly requests todo list\n4. User provides multiple tasks\n\nWhen NOT to Use:\n1. Single, straightforward tasks\n2. Trivial tasks with no organizational benefit\n3. Tasks completable in < 3 trivial steps\n\nTask States:\n- pending: Not yet started\n- in_progress: Currently working on\n- completed: Finished successfully\n\nUsage:\n- Create specific, actionable items\n- Break complex tasks into manageable steps\n- Update status in real-time\n- Mark complete IMMEDIATELY after finishing\n- Only ONE task in_progress at a time', 'parameters': {'type': 'object', 'properties': {'todos': {'type': 'array', 'description': 'Array of todo items with id, content, and status fields', 'items': {'type': 'string'}}}, 'required': ['todos']}}}, {'type': 'function', 'function': {'name': 'todoread', 'description': 'Use this tool to read your current todo list.\n\nReturns the current state of all todo items for this session.', 'parameters': {'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'question', 'description': "Ask the user a question and wait for their response.\n\nUse this tool when you need to:\n- Confirm before making significant changes\n- Get user preference between multiple options\n- Clarify ambiguous instructions\n\nQuestion format:\n- Each question has a text prompt\n- Optional header for context\n- List of options for the user to choose from\n- Options have label and optional description\n\nThe user's answers will be returned for you to continue with.", 'parameters': {'type': 'object', 'properties': {'questions': {'type': 'array', 'items': {'type': 'object', 'properties': {'question': {'type': 'string', 'description': 'Question text prompt'}, 'header': {'type': 'string', 'description': 'Optional header/context for the question'}, 'options': {'type': 'array', 'description': 'Options for the user to select', 'items': {'anyOf': [{'type': 'string'}, {'type': 'object', 'properties': {'label': {'type': 'string'}, 'description': {'type': 'string'}}, 'required': ['label'], 'additionalProperties': False}]}}}, 'required': ['question'], 'additionalProperties': True}, 'description': 'Array of questions to ask the user'}}, 'required': ['questions']}}}, {'type': 'function', 'function': {'name': 'task', 'description': 'Launch a new agent to handle complex, multi-step tasks autonomously.\n\nUse this tool when:\n- The task requires multiple steps or research\n- You need to explore code in parallel\n- The task can be delegated to a specialized agent\n\nAvailable subagent types:\n- general: General-purpose agent for multi-step tasks\n- explore: Fast code exploration agent for quick searches\n- review: Code review agent (if available)\n\nUsage notes:\n- Provide a clear description (3-5 words)\n- Provide detailed prompt with context\n- The subagent runs autonomously and returns results\n- Use for tasks that can be parallelized', 'parameters': {'type': 'object', 'properties': {'description': {'type': 'string', 'description': 'A short (3-5 words) description of the task'}, 'prompt': {'type': 'string', 'description': 'The task for the agent to perform'}, 'subagent_type': {'type': 'string', 'description': 'The type of specialized agent to use (general, explore, review)', 'enum': ['general', 'explore', 'review']}, 'session_id': {'type': 'string', 'description': 'Optional existing session ID to continue'}}, 'required': ['description', 'prompt', 'subagent_type']}}}, {'type': 'function', 'function': {'name': 'batch', 'description': 'Execute multiple tool calls in parallel for optimal performance.\n\nUse this tool when:\n- You need to run multiple independent operations\n- Operations don\'t depend on each other\'s results\n- You want to maximize throughput\n\nLimitations:\n- Maximum 25 tool calls per batch\n- Cannot batch the \'batch\' tool itself\n- External tools (MCP) cannot be batched\n\nFormat:\n- tool_calls: Array of {tool: "tool_name", parameters: {...}}', 'parameters': {'type': 'object', 'properties': {'tool_calls': {'type': 'array', 'description': 'Array of tool calls to execute in parallel', 'items': {'type': 'string'}}}, 'required': ['tool_calls']}}}, {'type': 'function', 'function': {'name': 'lsp', 'description': 'Perform LSP (Language Server Protocol) operations for code intelligence.\n\nSupported operations:\n- goToDefinition: Jump to where a symbol is defined\n- findReferences: Find all usages of a symbol\n- hover: Get type/documentation info for a symbol\n- documentSymbol: List all symbols in a file\n- workspaceSymbol: Search symbols across workspace\n- goToImplementation: Find implementations of an interface\n- prepareCallHierarchy: Get call hierarchy item at position\n- incomingCalls: Find callers of a function\n- outgoingCalls: Find functions called by a function\n\nParameters:\n- operation: The LSP operation to perform\n- filePath: Path to the file\n- line: Line number (1-based)\n- character: Character offset (1-based)', 'parameters': {'type': 'object', 'properties': {'operation': {'type': 'string', 'description': 'The LSP operation to perform', 'enum': ['goToDefinition', 'findReferences', 'hover', 'documentSymbol', 'workspaceSymbol', 'goToImplementation', 'prepareCallHierarchy', 'incomingCalls', 'outgoingCalls']}, 'filePath': {'type': 'string', 'description': 'The absolute or relative path to the file'}, 'line': {'type': 'integer', 'description': 'The line number (1-based, as shown in editors)'}, 'character': {'type': 'integer', 'description': 'The character offset (1-based, as shown in editors)'}}, 'required': ['operation', 'filePath', 'line', 'character']}}}, {'type': 'function', 'function': {'name': 'skill', 'description': "Load a skill to get detailed instructions for a specific task. Skills provide specialized knowledge and step-by-step guidance. Use this when a task matches an available skill's description. workflow-generator 根据自然语言描述生成 flocks 内置工作流（workflow.md, workflow.json, workflow.html）。当用户提出创建/设计/生成/搭建工作流或任何多步骤流程（如告警调查、事件响应、SOP/Runbook 自动化）时使用本 skill。 tool-builder Creates a new Flocks tool from user requirements, writes metadata, adds unit tests, and hot-reloads the tool without restarting. Use when the user asks to create a new tool, add a new API integration, or generate a tool from a requirement. ", 'parameters': {'type': 'object', 'properties': {'name': {'type': 'string', 'description': 'The skill identifier from available_skills'}}, 'required': ['name']}}}, {'type': 'function', 'function': {'name': 'run_workflow', 'description': 'Execute a workflow definition using the flocks-workflow runtime.\n\nWhen to use:\n- You need to execute a workflow.\n- You have an existing JSON/dict structure or a workflow JSON file and user request to execute it.\n- Execute workflow when workflow has been generated.\n\nHow to use:\n- Provide the workflow definition (dictionary, JSON string, or file path).\n- The workflow file path should be an absolute path. IMPORTANT: In JSON, file paths must be quoted strings (e.g. "workflow": "/path/to/workflow.json"). Unquoted paths will cause parse errors.\n- Optional: Provide input parameters, timeout settings, and whether to use LLM for logic node codegen.\n\nNote:\n- This tool depends on an existing workflow file.\n- workflow maybe execute failed, you need to check the workflow file and the input parameters. If execute failed, change parameters and fix workflow-exec.json (don\'t change workflow.json).\n- If no workflow file exists, ask user to specify the workflow file path or use the `workflow-generator` skill to create.', 'parameters': {'type': 'object', 'properties': {'workflow': {'anyOf': [{'type': 'object', 'description': 'Workflow definition as an object (dict)'}, {'type': 'string', 'description': 'Workflow JSON string or a workflow JSON file path'}], 'description': 'Workflow definition (dict). If passing a string, provide a JSON string or a workflow JSON file path.'}, 'inputs': {'type': 'object', 'additionalProperties': True, 'description': 'Input parameters for the workflow execution', 'default': {}}, 'use_llm': {'type': 'boolean', 'description': 'Enable LLM-backed code generation for `type="logic"` nodes (when code is missing). Recommended to keep enabled for logic-node workflows.', 'default': True}, 'ensure_requirements': {'type': 'boolean', 'description': 'Whether to automatically install requirements declared in workflow metadata', 'default': True}, 'timeout_s': {'type': 'number', 'description': 'Execution timeout in seconds (optional)'}, 'trace': {'type': 'boolean', 'description': 'Enable execution tracing for debugging', 'default': False}}, 'required': ['workflow']}}}, {'type': 'function', 'function': {'name': 'websearch', 'description': "Search the web for real-time information about any topic.\n\nUse this tool when you need:\n- Up-to-date information that might not be in training data\n- Current events or technology news\n- Documentation for libraries, frameworks, or tools\n- Verification of current facts\n\nToday's date: 2026-02-11\nUse the current year when searching for recent information.\n\nParameters:\n- query: Search query (be specific for better results)\n- numResults: Number of results to return (default: 8)\n- type: Search type - auto, fast, or deep", 'parameters': {'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Web search query'}, 'numResults': {'type': 'integer', 'description': 'Number of search results to return (default: 8)', 'default': 8}, 'type': {'type': 'string', 'description': "Search type - 'auto': balanced, 'fast': quick, 'deep': comprehensive", 'default': 'auto', 'enum': ['auto', 'fast', 'deep']}}, 'required': ['query']}}}, {'type': 'function', 'function': {'name': 'codesearch', 'description': "Search for security examples, documentation, and API usage patterns.\n\nUse this tool when you need:\n- Security examples for a specific tool or framework\n- API documentation and usage patterns\n- Best practices for specific programming tasks\n- Implementation references\n\nParameters:\n- query: Search query (e.g., 'YARA malware detection rules', 'Suricata IDS signatures')\n- tokensNum: Amount of context to return (1000-50000, default: 5000)\n\nTips:\n- Be specific about the security tool/framework\n- Include the security tool or technology if relevant\n- Use higher tokensNum for comprehensive documentation", 'parameters': {'type': 'object', 'properties': {'query': {'type': 'string', 'description': "Search query for security context (e.g., 'YARA malware detection rules')"}, 'tokensNum': {'type': 'integer', 'description': 'Number of tokens to return (1000-50000, default: 5000)', 'default': 5000}}, 'required': ['query']}}}, {'type': 'function', 'function': {'name': 'apply_patch', 'description': 'Apply a patch to modify files.\n\nThis tool is designed for advanced patch-based editing, supporting:\n- File creation (add)\n- File modification (update)\n- File deletion (delete)\n- File moves (update with move_path)\n\nPatch format:\n*** Begin Patch\n*** Add File: path/to/new/file.py\ncontent of new file\n*** Update File: path/to/existing/file.py\n@@@ ... @@@\n-old line\n+new line\n*** Delete File: path/to/delete.py\n*** End Patch\n\nUse the edit tool for simple string replacements.\nUse apply_patch for complex multi-file changes.', 'parameters': {'type': 'object', 'properties': {'patchText': {'type': 'string', 'description': 'The full patch text that describes all changes to be made'}}, 'required': ['patchText']}}}, {'type': 'function', 'function': {'name': 'memory_search', 'description': 'Search project memory using a natural language query.', 'parameters': {'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Natural language search query.'}, 'max_results': {'type': 'integer', 'description': 'Maximum number of results to return (default: 10).'}, 'min_score': {'type': 'number', 'description': 'Minimum similarity score 0-1 (default: 0.6).'}, 'sources': {'type': 'array', 'description': "Sources to search: ['memory', 'session'] (default: ['memory']).", 'items': {'type': 'string'}}}, 'required': ['query']}}}, {'type': 'function', 'function': {'name': 'memory_get', 'description': 'Retrieve memory file content by path, optionally filtered by line range.', 'parameters': {'type': 'object', 'properties': {'path': {'type': 'string', 'description': 'Memory file path relative to memory root.'}, 'from_line': {'type': 'integer', 'description': 'Starting line number (1-based).'}, 'lines': {'type': 'integer', 'description': 'Number of lines to return.'}}, 'required': ['path']}}}, {'type': 'function', 'function': {'name': 'memory_write', 'description': 'Write content to memory files for long-term recall.', 'parameters': {'type': 'object', 'properties': {'content': {'type': 'string', 'description': 'Content to write to memory.'}, 'path': {'type': 'string', 'description': 'Target path relative to memory root (default: YYYY-MM-DD.md).'}, 'append': {'type': 'boolean', 'description': 'Append to existing file (default: true).'}}, 'required': ['content']}}}, {'type': 'function', 'function': {'name': 'echo', 'description': 'Echo back the input message', 'parameters': {'type': 'object', 'properties': {'message': {'type': 'string', 'description': 'Message to echo'}}, 'required': ['message']}}}, {'type': 'function', 'function': {'name': 'get_time', 'description': 'Get current date and time', 'parameters': {'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'threatbook_ip_query', 'description': "Query IP address threat intelligence from ThreatBook API. Use this tool to get threat information about an IP address, including geographic location, threat severity, malicious behavior indicators, and security judgments. Example: To query '8.8.8.8', pass ip='8.8.8.8'.", 'parameters': {'type': 'object', 'properties': {'ip': {'type': 'string', 'description': "The IP address to query (e.g., '8.8.8.8', '192.168.1.1'). This is a required parameter and must be a valid IP address string."}, 'lang': {'type': 'string', 'description': 'Response language (en or zh)', 'default': 'en', 'enum': ['zh', 'en']}}, 'required': ['ip']}}}, {'type': 'function', 'function': {'name': 'threatbook_domain_query', 'description': "Query domain threat intelligence from ThreatBook API. Use this tool to get threat information about a domain, including DNS records, WHOIS data, threat severity, and security judgments. Example: To query 'example.com', pass domain='example.com'.", 'parameters': {'type': 'object', 'properties': {'domain': {'type': 'string', 'description': "The domain name to query (e.g., 'example.com', 'google.com'). This is a required parameter and must be a valid domain name string."}, 'lang': {'type': 'string', 'description': 'Response language (en or zh)', 'default': 'en', 'enum': ['zh', 'en']}}, 'required': ['domain']}}}, {'type': 'function', 'function': {'name': 'threatbook_file_query', 'description': "Query file hash threat intelligence from ThreatBook API. Use this tool to get malware analysis results, antivirus detection results, and threat information about a file hash. Supports MD5, SHA1, and SHA256 hashes. Example: To query a hash, pass file_hash='e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'.", 'parameters': {'type': 'object', 'properties': {'file_hash': {'type': 'string', 'description': "The file hash to query. Can be MD5, SHA1, or SHA256 format (e.g., 'a1b2c3d4...', '5e6f7a8b...'). This is a required parameter and must be a valid hash string."}, 'lang': {'type': 'string', 'description': 'Response language (en or zh)', 'default': 'en', 'enum': ['zh', 'en']}}, 'required': ['file_hash']}}}] +TOOLS = [{'type': 'function', 'function': {'name': 'read', 'description': "Reads a file from the local filesystem. You can access any file directly by using this tool.\nAssume this tool is able to read all files on the machine. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- The filePath parameter must be an absolute path, not a relative path\n- By default, it reads up to 2000 lines starting from the beginning of the file\n- You can optionally specify a line offset and limit (especially handy for long files), but it's recommended to read the whole file by not providing these parameters\n- Any lines longer than 2000 characters will be truncated\n- Results are returned using cat -n format, with line numbers starting at 1\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- You can read image files using this tool.", 'parameters': {'type': 'object', 'properties': {'filePath': {'type': 'string', 'description': 'The path to the file to read'}, 'offset': {'type': 'integer', 'description': 'The line number to start reading from (0-based)', 'default': 0}, 'limit': {'type': 'integer', 'description': 'The number of lines to read (defaults to 2000)', 'default': 2000}}, 'required': ['filePath']}}}, {'type': 'function', 'function': {'name': 'write', 'description': "Writes a file to the local filesystem.\n\nUsage:\n- This tool will overwrite the existing file if there is one at the provided path.\n- If this is an existing file, you MUST use the Read tool first to read the file's contents. This tool will fail if you did not read the file first.\n- ALWAYS prefer editing existing files in the codebase. NEVER write new files unless explicitly required.\n- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.\n- Only use emojis if the user explicitly requests it. Avoid writing emojis to files unless asked.", 'parameters': {'type': 'object', 'properties': {'content': {'type': 'string', 'description': 'The content to write to the file'}, 'filePath': {'type': 'string', 'description': 'The absolute path to the file to write (must be absolute, not relative)'}}, 'required': ['content', 'filePath']}}}, {'type': 'function', 'function': {'name': 'edit', 'description': 'Performs exact string replacements in files. \n\nUsage:\n- You must use your `Read` tool at least once in the conversation before editing. This tool will error if you attempt an edit without reading the file. \n- When editing text from Read tool output, ensure you preserve the exact indentation (tabs/spaces) as it appears AFTER the line number prefix. The line number prefix format is: spaces + line number + tab. Everything after that tab is the actual file content to match. Never include any part of the line number prefix in the oldString or newString.\n- ALWAYS prefer editing existing files in the codebase. NEVER write new files unless explicitly required.\n- Only use emojis if the user explicitly requests it. Avoid adding emojis to files unless asked.\n- The edit will FAIL if `oldString` is not found in the file with an error "oldString not found in content".\n- The edit will FAIL if `oldString` is found multiple times in the file with an error "oldString found multiple times and requires more code context to uniquely identify the intended match". Either provide a larger string with more surrounding context to make it unique or use `replaceAll` to change every instance of `oldString`. \n- Use `replaceAll` for replacing and renaming strings across the file. This parameter is useful if you want to rename a variable for instance.', 'parameters': {'type': 'object', 'properties': {'filePath': {'type': 'string', 'description': 'The absolute path to the file to modify'}, 'oldString': {'type': 'string', 'description': 'The text to replace'}, 'newString': {'type': 'string', 'description': 'The text to replace it with (must be different from oldString)'}, 'replaceAll': {'type': 'boolean', 'description': 'Replace all occurrences of oldString (default false)', 'default': False}}, 'required': ['filePath', 'oldString', 'newString']}}}, {'type': 'function', 'function': {'name': 'bash', 'description': 'Executes a given bash command in a persistent shell session with optional timeout, ensuring proper handling and security measures.\n\nAll commands run in /Users/chenjie/Library/Mobile Documents/com~apple~CloudDocs/0_work/projects/threatbook/flocks by default. Use the `workdir` parameter if you need to run a command in a different directory. AVOID using `cd && ` patterns - use `workdir` instead.\n\nIMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO NOT use it for file operations (reading, writing, editing, searching, finding files) - use the specialized tools for this instead.\n\nBefore executing the command, please follow these steps:\n\n1. Directory Verification:\n - If the command will create new directories or files, first use `ls` to verify the parent directory exists and is the correct location\n - For example, before running "mkdir foo/bar", first use `ls foo` to check that "foo" exists and is the intended parent directory\n\n2. Command Execution:\n - Always quote file paths that contain spaces with double quotes (e.g., rm "path with spaces/file.txt")\n - Examples of proper quoting:\n - mkdir "/Users/name/My Documents" (correct)\n - mkdir /Users/name/My Documents (incorrect - will fail)\n - python "/path/with spaces/script.py" (correct)\n - python /path/with spaces/script.py (incorrect - will fail)\n - After ensuring proper quoting, execute the command.\n - Capture the output of the command.\n\nUsage notes:\n - The command argument is required.\n - You can specify an optional timeout in milliseconds. If not specified, commands will time out after 120000ms (2 minutes).\n - It is very helpful if you write a clear, concise description of what this command does in 5-10 words.\n - If the output exceeds 1000 lines or 102400 bytes, it will be truncated and the full output will be written to a file.\n - Avoid using Bash with the `find`, `grep`, `cat`, `head`, `tail`, `sed`, `awk`, or `echo` commands. Instead, use the dedicated tools: Glob, Grep, Read, Edit, Write.\n - When issuing multiple commands:\n - If the commands are independent and can run in parallel, make multiple Bash tool calls in a single message.\n - If the commands depend on each other, use a single Bash call with \'&&\' to chain them together.\n - Use \';\' only when you need to run commands sequentially but don\'t care if earlier commands fail\n - AVOID using `cd && `. Use the `workdir` parameter to change directories instead.', 'parameters': {'type': 'object', 'properties': {'command': {'type': 'string', 'description': 'The command to execute'}, 'timeout': {'type': 'integer', 'description': 'Optional timeout in milliseconds', 'default': 120000}, 'workdir': {'type': 'string', 'description': 'The working directory to run the command in. Defaults to project directory.'}, 'description': {'type': 'string', 'description': 'Clear, concise description of what this command does in 5-10 words'}}, 'required': ['command']}}}, {'type': 'function', 'function': {'name': 'grep', 'description': '- Fast content search tool that works with any codebase size\n- Searches file contents using regular expressions\n- Supports full regex syntax (eg. "log.*Error", "function\\s+\\w+", etc.)\n- Filter files by pattern with the include parameter (eg. "*.js", "*.{ts,tsx}")\n- Returns file paths and line numbers with at least one match sorted by modification time\n- Use this tool when you need to find files containing specific patterns\n- If you need to identify/count the number of matches within files, use the Bash tool with `rg` (ripgrep) directly. Do NOT use `grep`.\n- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead', 'parameters': {'type': 'object', 'properties': {'pattern': {'type': 'string', 'description': 'The regex pattern to search for in file contents'}, 'path': {'type': 'string', 'description': 'The directory to search in. Defaults to the current working directory.'}, 'include': {'type': 'string', 'description': 'File pattern to include in the search (e.g. "*.js", "*.{ts,tsx}")'}}, 'required': ['pattern']}}}, {'type': 'function', 'function': {'name': 'glob', 'description': '- Fast file pattern matching tool that works with any codebase size\n- Supports glob patterns like "**/*.js" or "src/**/*.ts"\n- Returns matching file paths sorted by modification time\n- Use this tool when you need to find files by name patterns\n- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead\n- You have the capability to call multiple tools in a single response. It is always better to speculatively perform multiple searches as a batch that are potentially useful.', 'parameters': {'type': 'object', 'properties': {'pattern': {'type': 'string', 'description': 'The glob pattern to match files against'}, 'path': {'type': 'string', 'description': 'The directory to search in. If not specified, the current working directory will be used. IMPORTANT: Omit this field to use the default directory. DO NOT enter "undefined" or "null" - simply omit it for the default behavior.'}}, 'required': ['pattern']}}}, {'type': 'function', 'function': {'name': 'list', 'description': 'Lists files and directories in a given path. The path parameter must be absolute; omit it to use the current workspace directory. You can optionally provide an array of glob patterns to ignore with the ignore parameter. You should generally prefer the Glob and Grep tools, if you know which directories to search.', 'parameters': {'type': 'object', 'properties': {'path': {'type': 'string', 'description': 'The absolute path to the directory to list (must be absolute, not relative)'}, 'ignore': {'type': 'array', 'description': 'List of glob patterns to ignore', 'items': {'type': 'string'}}}}}}, {'type': 'function', 'function': {'name': 'webfetch', 'description': 'Fetch content from a specified URL and return its contents in a readable format.\n\nUsage:\n- The URL must be a fully-formed, valid URL starting with http:// or https://\n- By default, returns content in markdown format (HTML is converted)\n- Supports text, markdown, and html output formats\n- Has a default timeout of 30 seconds (configurable up to 120 seconds)\n- Response size is limited to 5MB', 'parameters': {'type': 'object', 'properties': {'url': {'type': 'string', 'description': 'The URL to fetch content from'}, 'format': {'type': 'string', 'description': 'The format to return content in (text, markdown, or html). Defaults to markdown.', 'default': 'markdown', 'enum': ['text', 'markdown', 'html']}, 'timeout': {'type': 'integer', 'description': 'Optional timeout in seconds (max 120)', 'default': 30}}, 'required': ['url']}}}, {'type': 'function', 'function': {'name': 'todo', 'description': 'Use this tool to read or manage the current todo list.', 'parameters': {'type': 'object', 'properties': {'action': {'type': 'string', 'enum': ['read', 'write']}, 'todos': {'type': 'array', 'items': {'type': 'object'}}}, 'required': ['action']}}}, {'type': 'function', 'function': {'name': 'question', 'description': "Ask the user a question and wait for their response.\n\nUse this tool when you need to:\n- Confirm before making significant changes\n- Get user preference between multiple options\n- Clarify ambiguous instructions\n\nQuestion format:\n- Each question has a text prompt\n- Optional header for context\n- List of options for the user to choose from\n- Options have label and optional description\n\nThe user's answers will be returned for you to continue with.", 'parameters': {'type': 'object', 'properties': {'questions': {'type': 'array', 'items': {'type': 'object', 'properties': {'question': {'type': 'string', 'description': 'Question text prompt'}, 'header': {'type': 'string', 'description': 'Optional header/context for the question'}, 'options': {'type': 'array', 'description': 'Options for the user to select', 'items': {'anyOf': [{'type': 'string'}, {'type': 'object', 'properties': {'label': {'type': 'string'}, 'description': {'type': 'string'}}, 'required': ['label'], 'additionalProperties': False}]}}}, 'required': ['question'], 'additionalProperties': True}, 'description': 'Array of questions to ask the user'}}, 'required': ['questions']}}}, {'type': 'function', 'function': {'name': 'task', 'description': 'Launch a new agent to handle complex, multi-step tasks autonomously.\n\nUse this tool when:\n- The task requires multiple steps or research\n- You need to explore code in parallel\n- The task can be delegated to a specialized agent\n\nAvailable subagent types:\n- general: General-purpose agent for multi-step tasks\n- explore: Fast code exploration agent for quick searches\n- review: Code review agent (if available)\n\nUsage notes:\n- Provide a clear description (3-5 words)\n- Provide detailed prompt with context\n- The subagent runs autonomously and returns results\n- Use for tasks that can be parallelized', 'parameters': {'type': 'object', 'properties': {'description': {'type': 'string', 'description': 'A short (3-5 words) description of the task'}, 'prompt': {'type': 'string', 'description': 'The task for the agent to perform'}, 'subagent_type': {'type': 'string', 'description': 'The type of specialized agent to use (general, explore, review)', 'enum': ['general', 'explore', 'review']}, 'session_id': {'type': 'string', 'description': 'Optional existing session ID to continue'}}, 'required': ['description', 'prompt', 'subagent_type']}}}, {'type': 'function', 'function': {'name': 'batch', 'description': 'Execute multiple tool calls in parallel for optimal performance.\n\nUse this tool when:\n- You need to run multiple independent operations\n- Operations don\'t depend on each other\'s results\n- You want to maximize throughput\n\nLimitations:\n- Maximum 25 tool calls per batch\n- Cannot batch the \'batch\' tool itself\n- External tools (MCP) cannot be batched\n\nFormat:\n- tool_calls: Array of {tool: "tool_name", parameters: {...}}', 'parameters': {'type': 'object', 'properties': {'tool_calls': {'type': 'array', 'description': 'Array of tool calls to execute in parallel', 'items': {'type': 'string'}}}, 'required': ['tool_calls']}}}, {'type': 'function', 'function': {'name': 'lsp', 'description': 'Perform LSP (Language Server Protocol) operations for code intelligence.\n\nSupported operations:\n- goToDefinition: Jump to where a symbol is defined\n- findReferences: Find all usages of a symbol\n- hover: Get type/documentation info for a symbol\n- documentSymbol: List all symbols in a file\n- workspaceSymbol: Search symbols across workspace\n- goToImplementation: Find implementations of an interface\n- prepareCallHierarchy: Get call hierarchy item at position\n- incomingCalls: Find callers of a function\n- outgoingCalls: Find functions called by a function\n\nParameters:\n- operation: The LSP operation to perform\n- filePath: Path to the file\n- line: Line number (1-based)\n- character: Character offset (1-based)', 'parameters': {'type': 'object', 'properties': {'operation': {'type': 'string', 'description': 'The LSP operation to perform', 'enum': ['goToDefinition', 'findReferences', 'hover', 'documentSymbol', 'workspaceSymbol', 'goToImplementation', 'prepareCallHierarchy', 'incomingCalls', 'outgoingCalls']}, 'filePath': {'type': 'string', 'description': 'The absolute or relative path to the file'}, 'line': {'type': 'integer', 'description': 'The line number (1-based, as shown in editors)'}, 'character': {'type': 'integer', 'description': 'The character offset (1-based, as shown in editors)'}}, 'required': ['operation', 'filePath', 'line', 'character']}}}, {'type': 'function', 'function': {'name': 'skill', 'description': "Load a skill to get detailed instructions for a specific task. Skills provide specialized knowledge and step-by-step guidance. Use this when a task matches an available skill's description. workflow-generator 根据自然语言描述生成 flocks 内置工作流（workflow.md, workflow.json, workflow.html）。当用户提出创建/设计/生成/搭建工作流或任何多步骤流程（如告警调查、事件响应、SOP/Runbook 自动化）时使用本 skill。 tool-builder Creates a new Flocks tool from user requirements, writes metadata, adds unit tests, and hot-reloads the tool without restarting. Use when the user asks to create a new tool, add a new API integration, or generate a tool from a requirement. ", 'parameters': {'type': 'object', 'properties': {'name': {'type': 'string', 'description': 'The skill identifier from available_skills'}}, 'required': ['name']}}}, {'type': 'function', 'function': {'name': 'run_workflow', 'description': 'Execute a workflow definition using the flocks-workflow runtime.\n\nWhen to use:\n- You need to execute a workflow.\n- You have an existing JSON/dict structure or a workflow JSON file and user request to execute it.\n- Execute workflow when workflow has been generated.\n\nHow to use:\n- Provide the workflow definition (dictionary, JSON string, or file path).\n- The workflow file path should be an absolute path. IMPORTANT: In JSON, file paths must be quoted strings (e.g. "workflow": "/path/to/workflow.json"). Unquoted paths will cause parse errors.\n- Optional: Provide input parameters, timeout settings, and whether to use LLM for logic node codegen.\n\nNote:\n- This tool depends on an existing workflow file.\n- workflow maybe execute failed, you need to check the workflow file and the input parameters. If execute failed, change parameters and fix workflow-exec.json (don\'t change workflow.json).\n- If no workflow file exists, ask user to specify the workflow file path or use the `workflow-generator` skill to create.', 'parameters': {'type': 'object', 'properties': {'workflow': {'anyOf': [{'type': 'object', 'description': 'Workflow definition as an object (dict)'}, {'type': 'string', 'description': 'Workflow JSON string or a workflow JSON file path'}], 'description': 'Workflow definition (dict). If passing a string, provide a JSON string or a workflow JSON file path.'}, 'inputs': {'type': 'object', 'additionalProperties': True, 'description': 'Input parameters for the workflow execution', 'default': {}}, 'use_llm': {'type': 'boolean', 'description': 'Enable LLM-backed code generation for `type="logic"` nodes (when code is missing). Recommended to keep enabled for logic-node workflows.', 'default': True}, 'ensure_requirements': {'type': 'boolean', 'description': 'Whether to automatically install requirements declared in workflow metadata', 'default': True}, 'timeout_s': {'type': 'number', 'description': 'Execution timeout in seconds (optional)'}, 'trace': {'type': 'boolean', 'description': 'Enable execution tracing for debugging', 'default': False}}, 'required': ['workflow']}}}, {'type': 'function', 'function': {'name': 'websearch', 'description': "Search the web for real-time information about any topic.\n\nUse this tool when you need:\n- Up-to-date information that might not be in training data\n- Current events or technology news\n- Documentation for libraries, frameworks, or tools\n- Verification of current facts\n\nToday's date: 2026-02-11\nUse the current year when searching for recent information.\n\nParameters:\n- query: Search query (be specific for better results)\n- numResults: Number of results to return (default: 8)\n- type: Search type - auto, fast, or deep", 'parameters': {'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Web search query'}, 'numResults': {'type': 'integer', 'description': 'Number of search results to return (default: 8)', 'default': 8}, 'type': {'type': 'string', 'description': "Search type - 'auto': balanced, 'fast': quick, 'deep': comprehensive", 'default': 'auto', 'enum': ['auto', 'fast', 'deep']}}, 'required': ['query']}}}, {'type': 'function', 'function': {'name': 'codesearch', 'description': "Search for security examples, documentation, and API usage patterns.\n\nUse this tool when you need:\n- Security examples for a specific tool or framework\n- API documentation and usage patterns\n- Best practices for specific programming tasks\n- Implementation references\n\nParameters:\n- query: Search query (e.g., 'YARA malware detection rules', 'Suricata IDS signatures')\n- tokensNum: Amount of context to return (1000-50000, default: 5000)\n\nTips:\n- Be specific about the security tool/framework\n- Include the security tool or technology if relevant\n- Use higher tokensNum for comprehensive documentation", 'parameters': {'type': 'object', 'properties': {'query': {'type': 'string', 'description': "Search query for security context (e.g., 'YARA malware detection rules')"}, 'tokensNum': {'type': 'integer', 'description': 'Number of tokens to return (1000-50000, default: 5000)', 'default': 5000}}, 'required': ['query']}}}, {'type': 'function', 'function': {'name': 'apply_patch', 'description': 'Apply a patch to modify files.\n\nThis tool is designed for advanced patch-based editing, supporting:\n- File creation (add)\n- File modification (update)\n- File deletion (delete)\n- File moves (update with move_path)\n\nPatch format:\n*** Begin Patch\n*** Add File: path/to/new/file.py\ncontent of new file\n*** Update File: path/to/existing/file.py\n@@@ ... @@@\n-old line\n+new line\n*** Delete File: path/to/delete.py\n*** End Patch\n\nUse the edit tool for simple string replacements.\nUse apply_patch for complex multi-file changes.', 'parameters': {'type': 'object', 'properties': {'patchText': {'type': 'string', 'description': 'The full patch text that describes all changes to be made'}}, 'required': ['patchText']}}}, {'type': 'function', 'function': {'name': 'memory_search', 'description': 'Search project memory using a natural language query.', 'parameters': {'type': 'object', 'properties': {'query': {'type': 'string', 'description': 'Natural language search query.'}, 'max_results': {'type': 'integer', 'description': 'Maximum number of results to return (default: 10).'}, 'min_score': {'type': 'number', 'description': 'Minimum similarity score 0-1 (default: 0.6).'}, 'sources': {'type': 'array', 'description': "Sources to search: ['memory', 'session'] (default: ['memory']).", 'items': {'type': 'string'}}}, 'required': ['query']}}}, {'type': 'function', 'function': {'name': 'memory_get', 'description': 'Retrieve memory file content by path, optionally filtered by line range.', 'parameters': {'type': 'object', 'properties': {'path': {'type': 'string', 'description': 'Memory file path relative to memory root.'}, 'from_line': {'type': 'integer', 'description': 'Starting line number (1-based).'}, 'lines': {'type': 'integer', 'description': 'Number of lines to return.'}}, 'required': ['path']}}}, {'type': 'function', 'function': {'name': 'memory_write', 'description': 'Write content to memory files for long-term recall.', 'parameters': {'type': 'object', 'properties': {'content': {'type': 'string', 'description': 'Content to write to memory.'}, 'path': {'type': 'string', 'description': 'Target path relative to memory root (default: YYYY-MM-DD.md).'}, 'append': {'type': 'boolean', 'description': 'Append to existing file (default: true).'}}, 'required': ['content']}}}, {'type': 'function', 'function': {'name': 'echo', 'description': 'Echo back the input message', 'parameters': {'type': 'object', 'properties': {'message': {'type': 'string', 'description': 'Message to echo'}}, 'required': ['message']}}}, {'type': 'function', 'function': {'name': 'get_time', 'description': 'Get current date and time', 'parameters': {'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'threatbook_ip_query', 'description': "Query IP address threat intelligence from ThreatBook API. Use this tool to get threat information about an IP address, including geographic location, threat severity, malicious behavior indicators, and security judgments. Example: To query '8.8.8.8', pass ip='8.8.8.8'.", 'parameters': {'type': 'object', 'properties': {'ip': {'type': 'string', 'description': "The IP address to query (e.g., '8.8.8.8', '192.168.1.1'). This is a required parameter and must be a valid IP address string."}, 'lang': {'type': 'string', 'description': 'Response language (en or zh)', 'default': 'en', 'enum': ['zh', 'en']}}, 'required': ['ip']}}}, {'type': 'function', 'function': {'name': 'threatbook_domain_query', 'description': "Query domain threat intelligence from ThreatBook API. Use this tool to get threat information about a domain, including DNS records, WHOIS data, threat severity, and security judgments. Example: To query 'example.com', pass domain='example.com'.", 'parameters': {'type': 'object', 'properties': {'domain': {'type': 'string', 'description': "The domain name to query (e.g., 'example.com', 'google.com'). This is a required parameter and must be a valid domain name string."}, 'lang': {'type': 'string', 'description': 'Response language (en or zh)', 'default': 'en', 'enum': ['zh', 'en']}}, 'required': ['domain']}}}, {'type': 'function', 'function': {'name': 'threatbook_file_query', 'description': "Query file hash threat intelligence from ThreatBook API. Use this tool to get malware analysis results, antivirus detection results, and threat information about a file hash. Supports MD5, SHA1, and SHA256 hashes. Example: To query a hash, pass file_hash='e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'.", 'parameters': {'type': 'object', 'properties': {'file_hash': {'type': 'string', 'description': "The file hash to query. Can be MD5, SHA1, or SHA256 format (e.g., 'a1b2c3d4...', '5e6f7a8b...'). This is a required parameter and must be a valid hash string."}, 'lang': {'type': 'string', 'description': 'Response language (en or zh)', 'default': 'en', 'enum': ['zh', 'en']}}, 'required': ['file_hash']}}}] # ============================================================================ # Test Functions diff --git a/tests/provider/test_model_management.py b/tests/provider/test_model_management.py index 98b0e87de..7f9b3ed66 100644 --- a/tests/provider/test_model_management.py +++ b/tests/provider/test_model_management.py @@ -252,6 +252,33 @@ def get_models(self): assert defs[0].limits.context_window == 100000 assert defs[0].limits.max_output_tokens == 8000 + def test_config_override_false_removes_reasoning_feature(self): + from flocks.provider.provider import BaseProvider, ModelInfo, ModelCapabilities + + catalog_def = ModelDefinition( + id="dummy-model", + name="Dummy Model", + provider_id="dummy", + fetch_from=FetchFrom.PREDEFINED, + capabilities=ModelCapabilitiesV2( + features=[ModelFeature.REASONING], + supports_reasoning=True, + ), + ) + model = ModelInfo( + id="dummy-model", + name="Dummy Model", + provider_id="dummy", + capabilities=ModelCapabilities(supports_reasoning=False), + ) + model._explicit_keys = {"supports_reasoning"} + + p = BaseProvider("dummy", "Dummy") + overridden = p._apply_config_overrides(catalog_def, model) + + assert overridden.capabilities.supports_reasoning is False + assert ModelFeature.REASONING not in overridden.capabilities.features + def test_configure_from_credential(self): from flocks.provider.provider import BaseProvider diff --git a/tests/provider/test_model_management_p2p3.py b/tests/provider/test_model_management_p2p3.py index adb66498a..e4ff4ea61 100644 --- a/tests/provider/test_model_management_p2p3.py +++ b/tests/provider/test_model_management_p2p3.py @@ -166,6 +166,24 @@ def test_model_has_features(self): assert ModelFeature.VISION in sonnet4.capabilities.features assert ModelFeature.REASONING in sonnet4.capabilities.features + def test_model_catalog_defaults_reasoning_on_when_omitted(self): + from flocks.provider.model_catalog import _parse_model_definitions + + models = _parse_model_definitions( + "custom-defaults", + { + "custom-model": { + "name": "Custom Model", + "capabilities": { + "supports_tools": True, + }, + } + }, + ) + + assert models[0].capabilities.supports_reasoning is True + assert ModelFeature.REASONING in models[0].capabilities.features + def test_google_gemini_multimodal(self): from flocks.provider.model_catalog import get_provider_model_definitions models = get_provider_model_definitions("google") diff --git a/tests/provider/test_provider.py b/tests/provider/test_provider.py index 0a6155c37..c56f46b1c 100644 --- a/tests/provider/test_provider.py +++ b/tests/provider/test_provider.py @@ -159,22 +159,33 @@ def test_resolve_model_infers_interleaved_for_runtime_discovered_reasoning_model def test_resolve_model_does_not_infer_interleaved_for_non_reasoning_model(monkeypatch): + """A model whose id does NOT match any series token in + ``infer_interleaved_capability`` and has no catalog ``interleaved`` should + stay non-reasoning — the series-token inference must not fabricate a + capability for it. + + Uses ``gpt-4-turbo`` as the model id: it is not in any of the + ``_PROMOTE_REASONING_CONTENT_TOKENS`` / ``_STRICT_REASONING_CONTENT_TOKENS`` + lists, the provider id ``custom-demo`` does not match any of the + special-case provider prefixes, and the base_url does not match any of + the provider-hosting hints. + """ provider_model = SimpleNamespace( - id="deepseek-chat", + id="gpt-4-turbo", capabilities=SimpleNamespace(interleaved=None), ) fake_provider = SimpleNamespace( get_model_definitions=lambda: [provider_model], get_models=lambda: [], _config_models=[], - _config=SimpleNamespace(base_url="https://api.deepseek.com/v1"), + _config=SimpleNamespace(base_url="https://api.openai.com/v1"), ) monkeypatch.setattr(Provider, "_initialized", True) monkeypatch.setattr(Provider, "_providers", {"custom-demo": fake_provider}) monkeypatch.setattr(Provider, "_models", {}) - resolved = Provider.resolve_model("custom-demo", "deepseek-chat") + resolved = Provider.resolve_model("custom-demo", "gpt-4-turbo") assert resolved is provider_model assert resolved.capabilities.interleaved is None diff --git a/tests/provider/test_provider_options.py b/tests/provider/test_provider_options.py index 77a02a835..4e8f11be0 100644 --- a/tests/provider/test_provider_options.py +++ b/tests/provider/test_provider_options.py @@ -4,6 +4,11 @@ REASONING_TRANSPORT_GENERIC_CHAT, ) +DEEPSEEK_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled"}} +GLM_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled", "clear_thinking": False}} +KIMI_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled"}} +MIMO_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled"}} + class TestBuildProviderOptions: def test_claude_reasoning_can_be_disabled(self): @@ -35,23 +40,23 @@ def test_threatbook_qwen_respects_reasoning_toggle(self): assert options["extra_body"]["enable_thinking"] is False - def test_threatbook_kimi_hybrid_models_enable_thinking_by_default(self): + def test_threatbook_kimi_hybrid_models_use_official_thinking_payload(self): options = provider_options.build_provider_options( "threatbook-cn-llm", "kimi-k2.6", resolve_max_tokens=False, ) - assert options["extra_body"]["enable_thinking"] is True + assert options["extra_body"] == KIMI_THINKING_EXTRA_BODY - def test_moonshot_kimi_hybrid_models_enable_thinking_by_default(self): + def test_moonshot_kimi_hybrid_models_use_official_thinking_payload(self): options = provider_options.build_provider_options( "moonshot", "kimi-k2.6", resolve_max_tokens=False, ) - assert options["extra_body"]["enable_thinking"] is True + assert options["extra_body"] == KIMI_THINKING_EXTRA_BODY def test_openai_compatible_qwen_models_enable_thinking_by_default(self, monkeypatch): monkeypatch.setattr( @@ -72,7 +77,7 @@ def test_openai_compatible_qwen_models_enable_thinking_by_default(self, monkeypa assert options["extra_body"]["enable_thinking"] is True - def test_openai_compatible_kimi_models_enable_thinking_by_default(self, monkeypatch): + def test_openai_compatible_kimi_models_use_official_thinking_payload(self, monkeypatch): monkeypatch.setattr( provider_options, "_resolve_interleaved_capability", @@ -90,7 +95,129 @@ def test_openai_compatible_kimi_models_enable_thinking_by_default(self, monkeypa resolve_max_tokens=False, ) - assert options["extra_body"]["enable_thinking"] is True + assert options["extra_body"] == KIMI_THINKING_EXTRA_BODY + + def test_minimax_models_use_reasoning_split(self, monkeypatch): + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args: { + "field": "reasoning_details", + "echo": "tool_calls", + "cross_provider_policy": "promote", + }, + ) + + options = provider_options.build_provider_options( + "minimax", + "minimax-m3", + resolve_max_tokens=False, + ) + + assert options["extra_body"] == {"reasoning_split": True} + + def test_deepseek_models_use_official_thinking_payload(self, monkeypatch): + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args: { + "field": "reasoning_content", + "echo": "tool_calls", + "placeholder": " ", + "cross_provider_policy": "placeholder", + }, + ) + + options = provider_options.build_provider_options( + "deepseek", + "deepseek-v4-pro", + resolve_max_tokens=False, + ) + + assert options["extra_body"] == DEEPSEEK_THINKING_EXTRA_BODY + + def test_deepseek_models_emit_disabled_thinking_when_reasoning_disabled( + self, + monkeypatch, + ): + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args: { + "field": "reasoning_content", + "echo": "tool_calls", + "placeholder": " ", + "cross_provider_policy": "placeholder", + }, + ) + + options = provider_options.build_provider_options( + "deepseek", + "deepseek-v4-pro", + reasoning_enabled=False, + resolve_max_tokens=False, + ) + + assert options["extra_body"] == {"thinking": {"type": "disabled"}} + + def test_glm_models_use_official_thinking_payload(self, monkeypatch): + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args: { + "field": "reasoning_content", + "echo": "tool_calls", + "cross_provider_policy": "promote", + }, + ) + + options = provider_options.build_provider_options( + "zhipu", + "glm-4.7", + resolve_max_tokens=False, + ) + + assert options["extra_body"] == GLM_THINKING_EXTRA_BODY + + def test_glm_models_emit_disabled_thinking_when_reasoning_disabled(self, monkeypatch): + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args: { + "field": "reasoning_content", + "echo": "tool_calls", + "cross_provider_policy": "promote", + }, + ) + + options = provider_options.build_provider_options( + "zhipu", + "glm-4.7", + reasoning_enabled=False, + resolve_max_tokens=False, + ) + + assert options["extra_body"] == {"thinking": {"type": "disabled"}} + + def test_mimo_models_use_official_thinking_payload(self, monkeypatch): + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args: { + "field": "reasoning_content", + "echo": "tool_calls", + "placeholder": " ", + "cross_provider_policy": "placeholder", + }, + ) + + options = provider_options.build_provider_options( + "openai-compatible", + "mimo-v2.5-pro", + resolve_max_tokens=False, + ) + + assert options["extra_body"] == MIMO_THINKING_EXTRA_BODY def test_claude_thinking_depends_on_transport_not_capability(self, monkeypatch): monkeypatch.setattr( @@ -148,7 +275,17 @@ def test_kimi_hybrid_models_respect_explicit_reasoning_toggle(self): resolve_max_tokens=False, ) - assert options["extra_body"]["enable_thinking"] is True + assert options["extra_body"] == KIMI_THINKING_EXTRA_BODY + + def test_kimi_hybrid_models_emit_disabled_thinking_when_reasoning_disabled(self): + options = provider_options.build_provider_options( + "threatbook-cn-llm", + "kimi-k2.5", + reasoning_enabled=False, + resolve_max_tokens=False, + ) + + assert options["extra_body"] == {"thinking": {"type": "disabled"}} def test_model_setting_enable_thinking_is_applied(self, monkeypatch): monkeypatch.setattr(provider_options, "_resolve_reasoning_enabled", lambda *_args: True) @@ -159,7 +296,24 @@ def test_model_setting_enable_thinking_is_applied(self, monkeypatch): resolve_max_tokens=False, ) - assert options["extra_body"]["enable_thinking"] is True + assert options["extra_body"] == KIMI_THINKING_EXTRA_BODY + + def test_model_setting_enable_thinking_applies_without_interleaved(self, monkeypatch): + monkeypatch.setattr(provider_options, "_resolve_reasoning_enabled", lambda *_args: True) + monkeypatch.setattr(provider_options, "_resolve_interleaved_capability", lambda *_args: None) + monkeypatch.setattr( + provider_options, + "_resolve_reasoning_transport", + lambda *_args: REASONING_TRANSPORT_GENERIC_CHAT, + ) + + options = provider_options.build_provider_options( + "openai-compatible", + "custom-thinking-model", + resolve_max_tokens=False, + ) + + assert options["extra_body"] == {"enable_thinking": True} def test_openai_reasoning_can_be_disabled(self): options = provider_options.build_provider_options( diff --git a/tests/provider/test_thinking_params.py b/tests/provider/test_thinking_params.py new file mode 100644 index 000000000..4ace790e0 --- /dev/null +++ b/tests/provider/test_thinking_params.py @@ -0,0 +1,585 @@ +""" +Regression net for transport-driven thinking-params dispatch. + +Background +---------- + +The original dispatch in ``flocks/provider/options.py`` matched the model name +against a hard-coded substring whitelist (``qwen3`` / ``kimi`` / ``mimo`` / …) +to decide whether to send ``extra_body.enable_thinking: true``. Models whose +names didn't match the magic substrings were silently sent without the +thinking flag, causing the upstream API to short-circuit with +``finish_reason=stop`` and an empty content block — the user-visible +"agent stopped, please say 'continue'" symptom seen in +``ses_1628dfe6cffe1i5xZY9lv1u20m``. + +The new dispatch is transport-driven: + + - ``reasoning_transport == anthropic_messages`` → ``thinking={type: "enabled", budget_tokens:...}`` + - ``reasoning_transport == generic_chat`` → provider-specific ``extra_body`` params + +The gate is the resolved ``interleaved_capability``: catalog explicit +declaration wins, with the series-token inference in +``interleaved.infer_interleaved_capability`` (qwen3 / glm-* / kimi-k2* / +deepseek-v4* / step-3.5* / minimax-m* / …) as the fallback. Adding a new +provider or a new model from a known family needs zero dispatcher changes. + +These tests verify four properties of the new path: + +1. Every (provider, model) pair with ``interleaved != null`` in catalog.json + produces a non-empty thinking signal (extra_body or thinking=). This is + the systematic regression net — any new model added to the catalog with + interleaved thinking will be covered. +2. The specific GLM-5 / alibaba configuration that triggered the original + trace bug now emits the right flag. +3. The ``openai_compatible`` provider no longer swallows caller-supplied + ``extra_body`` in its chat_stream() path. +4. A model not in catalog but matching a known series token still gets the + flag — the series-token fallback closes the "I forgot to add the model + to catalog" gap. +""" + +from __future__ import annotations + +from typing import Any, Dict, Iterator, Tuple + +import pytest + +from flocks.provider import model_catalog +from flocks.provider import options as provider_options + +DEEPSEEK_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled"}} +GLM_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled", "clear_thinking": False}} +KIMI_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled"}} +MIMO_THINKING_EXTRA_BODY = {"thinking": {"type": "enabled"}} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _iter_interleaved_catalog_entries() -> Iterator[Tuple[str, str]]: + """Yield (provider_id, model_id) for every catalog entry that declares interleaved thinking. + + Mirrors the jq query used during the audit: + ..models | to_entries[] + | select(.value.capabilities.interleaved != null) + | ["", ""] + """ + raw = model_catalog.get_raw_catalog() + for provider_id, provider_entry in raw.items(): + if not isinstance(provider_entry, dict): + continue + models = provider_entry.get("models") + if not isinstance(models, dict): + continue + for model_id, model_entry in models.items(): + if not isinstance(model_entry, dict): + continue + capabilities = model_entry.get("capabilities") or {} + if not isinstance(capabilities, dict): + continue + if capabilities.get("interleaved") is not None: + yield provider_id, model_id + + +def _expected_generic_chat_extra_body( + provider_id: str, + model_id: str, +) -> Dict[str, Any]: + """Return the expected OpenAI-compatible thinking control payload.""" + provider_lower = provider_id.lower() + model_lower = model_id.lower() + + if "deepseek" in model_lower or provider_lower == "deepseek": + return DEEPSEEK_THINKING_EXTRA_BODY + if "glm" in model_lower or provider_lower == "zhipu": + return GLM_THINKING_EXTRA_BODY + if "mimo" in model_lower: + return MIMO_THINKING_EXTRA_BODY + if "kimi" in model_lower: + return KIMI_THINKING_EXTRA_BODY + if "minimax" in model_lower or provider_lower == "minimax": + return {"reasoning_split": True} + return {"enable_thinking": True} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestCatalogInterleavedCoverage: + """Property test: every interleaved catalog entry resolves to a thinking flag. + + The dispatch is now transport-driven, not provider-driven — every + interleaved catalog entry should land in *some* thinking signal. The + series-token fallback (in ``interleaved.infer_interleaved_capability``) + is exercised by a separate test below. + """ + + @pytest.mark.parametrize("provider_id,model_id", list(_iter_interleaved_catalog_entries())) + def test_interleaved_model_gets_thinking_flag(self, provider_id: str, model_id: str) -> None: + # Patch the interleaved capability so the dispatch gate fires + # regardless of the test environment's catalog resolution path. + original = provider_options._resolve_interleaved_capability + provider_options._resolve_interleaved_capability = lambda *_args, **_kw: { + "field": "reasoning_content", + "echo": "tool_calls", + "cross_provider_policy": "promote", + } + try: + options = provider_options.build_provider_options( + provider_id, + model_id, + resolve_max_tokens=False, + ) + finally: + provider_options._resolve_interleaved_capability = original + + # The dispatch should produce SOME thinking signal. We accept + # either extra_body (OpenAI-compat family) or a top-level reasoning + # field (Anthropic/Google family). The catalog is already filtered + # to interleaved-only entries so neither should be empty. + has_extra_body = bool(options.get("extra_body")) + has_thinking = bool(options.get("thinking")) + has_reasoning_effort = bool(options.get("reasoningEffort")) + has_thinking_config = bool(options.get("thinkingConfig")) + has_thinking_level = bool(options.get("thinkingLevel")) + assert ( + has_extra_body + or has_thinking + or has_reasoning_effort + or has_thinking_config + or has_thinking_level + ), ( + f"{provider_id}/{model_id} declares interleaved in catalog but " + f"build_provider_options emitted no thinking field. " + f"options={options!r}" + ) + + @pytest.mark.parametrize("provider_id,model_id", list(_iter_interleaved_catalog_entries())) + def test_interleaved_model_gets_official_generic_chat_payload( + self, + provider_id: str, + model_id: str, + ) -> None: + """Every catalog-declared generic-chat model emits its official payload.""" + options = provider_options.build_provider_options( + provider_id, + model_id, + resolve_max_tokens=False, + ) + + assert options.get("extra_body") == _expected_generic_chat_extra_body( + provider_id, + model_id, + ), f"{provider_id}/{model_id} emitted unexpected options={options!r}" + + +class TestGLM5TraceReplay: + """Specific regression for ses_1628dfe6cffe1i5xZY9lv1u20m step 50. + + Trace showed: GLM-5 on alibaba, tools present, returned + ``finishReason=stop, content=495, toolCallCount=0`` because the request + went out without a thinking payload. After the fix, the request body + should include GLM's official thinking object. + """ + + def test_glm5_alibaba_emits_official_thinking_payload( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args, **_kw: { + "field": "reasoning_content", + "echo": "tool_calls", + "cross_provider_policy": "promote", + }, + ) + + options = provider_options.build_provider_options( + "alibaba", + "GLM-5", + resolve_max_tokens=False, + ) + + assert "extra_body" in options, ( + "alibaba/GLM-5 catalog declares interleaved but no extra_body emitted — " + "this is the exact regression that caused ses_1628dfe6cffe1i5xZY9lv1u20m" + ) + assert options["extra_body"] == GLM_THINKING_EXTRA_BODY + + @pytest.mark.parametrize( + "provider_id", + ["alibaba", "threatbook-cn-llm", "threatbook-io-llm", "zhipu"], + ) + def test_glm5_emits_official_thinking_payload_on_every_provider( + self, provider_id: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args, **_kw: { + "field": "reasoning_content", + "echo": "tool_calls", + "cross_provider_policy": "promote", + }, + ) + + options = provider_options.build_provider_options( + provider_id, + "GLM-5", + resolve_max_tokens=False, + ) + assert options["extra_body"] == GLM_THINKING_EXTRA_BODY + + @pytest.mark.parametrize( + "provider_id,model_id,field,expected_extra_body", + [ + ("threatbook-cn-llm", "minimax-m2.5", "reasoning_details", {"reasoning_split": True}), + ("threatbook-cn-llm", "minimax-m2.7", "reasoning_details", {"reasoning_split": True}), + ("threatbook-cn-llm", "minimax-m3", "reasoning_details", {"reasoning_split": True}), + ("threatbook-io-llm", "minimax-m2.5", "reasoning_details", {"reasoning_split": True}), + ("threatbook-io-llm", "minimax-m2.7", "reasoning_details", {"reasoning_split": True}), + ("threatbook-io-llm", "minimax-m3", "reasoning_details", {"reasoning_split": True}), + ("minimax", "minimax-m2.5", "reasoning_details", {"reasoning_split": True}), + ("deepseek", "deepseek-reasoner", "reasoning_content", DEEPSEEK_THINKING_EXTRA_BODY), + ("stepfun", "step-3.5-flash", "reasoning_content", {"enable_thinking": True}), + ], + ) + def test_previously_dropped_models_now_get_flag( + self, + provider_id: str, + model_id: str, + field: str, + expected_extra_body: Dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args, **_kw: { + "field": field, + "echo": "tool_calls", + "cross_provider_policy": "promote", + }, + ) + + options = provider_options.build_provider_options( + provider_id, + model_id, + resolve_max_tokens=False, + ) + assert options.get("extra_body") == expected_extra_body, ( + f"{provider_id}/{model_id} — catalog says interleaved, dispatch " + f"should have emitted {expected_extra_body}" + ) + + +class TestDispatchShape: + """Sanity checks on the dispatch itself now that the + provider-keyed shape registry is gone. + + The dispatch is now transport-driven: ``anthropic_messages`` → + ``thinking={type: "enabled", ...}``; ``generic_chat`` → + provider-specific ``extra_body``. Catalog explicit declaration wins, + with the series-token inference in ``interleaved.infer_interleaved_capability`` + as fallback for any model the catalog forgot to declare. + """ + + def test_no_legacy_token_constant(self) -> None: + """The token-substring whitelist must be gone — that was the bug surface.""" + assert not hasattr( + provider_options, "_ENABLE_THINKING_EXTRA_BODY_TOKENS" + ), ( + "_ENABLE_THINKING_EXTRA_BODY_TOKENS should be removed; the catalog " + "interleaved field is now the only trigger" + ) + + def test_no_shape_registry(self) -> None: + """The provider-keyed shape registry is gone — every entry produced + the same dict, so the indirection wasn't earning its keep. Wire format + is now decided by ``reasoning_transport`` alone. + """ + assert not hasattr(provider_options, "_THINKING_REQUEST_SHAPES"), ( + "_THINKING_REQUEST_SHAPES should be removed; dispatch is now " + "transport-driven, not provider-driven" + ) + assert not hasattr(provider_options, "_openai_base_thinking_shape"), ( + "_openai_base_thinking_shape should be removed; generic_chat " + "interleaved emits extra_body inline" + ) + + def test_deepseek_v3_is_not_auto_thinking_model(self) -> None: + """``deepseek-chat`` (V3) must not inherit thinking params from a + broad ``deepseek`` substring. + """ + catalog = model_catalog.get_raw_catalog() + v3_entry = catalog.get("deepseek", {}).get("models", {}).get("deepseek-chat") + assert v3_entry is not None, "deepseek-chat missing from catalog" + assert v3_entry.get("capabilities", {}).get("interleaved") is None, ( + "deepseek-chat now declares interleaved in catalog — remove the " + "series-token assertion and let the catalog coverage test pin it" + ) + + options = provider_options.build_provider_options( + "deepseek", "deepseek-chat", resolve_max_tokens=False, + ) + assert "extra_body" not in options, ( + "deepseek-chat does not declare interleaved in catalog and should " + f"not be auto-enabled by a broad deepseek token. options={options!r}" + ) + + def test_explicit_reasoning_toggle_propagates(self) -> None: + """``reasoning_enabled=False`` should produce ``enable_thinking: false`` + on a generic_chat transport, mirroring the old token-matching branch's + behavior so the upstream API gets an explicit opt-out signal. + """ + options = provider_options.build_provider_options( + "threatbook-cn-llm", + "qwen3.6-plus", + reasoning_enabled=False, + resolve_max_tokens=False, + ) + assert options["extra_body"]["enable_thinking"] is False + + def test_anthropic_transport_still_uses_thinking_field( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The Anthropic transport branch is unchanged: it must continue to + emit ``thinking={type: "enabled", ...}``, never ``extra_body``. + + This pins the contract that the new generic_chat branch did not + regress the anthropic_messages path. + """ + from flocks.provider.interleaved import REASONING_TRANSPORT_ANTHROPIC_MESSAGES + + monkeypatch.setattr( + provider_options, + "_resolve_interleaved_capability", + lambda *_args, **_kw: { + "field": "thinking", + "echo": "tool_calls", + "cross_provider_policy": "preserve", + }, + ) + monkeypatch.setattr( + provider_options, + "_resolve_reasoning_transport", + lambda *_args, **_kw: REASONING_TRANSPORT_ANTHROPIC_MESSAGES, + ) + options = provider_options.build_provider_options( + "anthropic", "claude-sonnet-4-20250514", resolve_max_tokens=False, + ) + assert "thinking" in options + assert options["thinking"]["type"] == "enabled" + assert "extra_body" not in options + + @pytest.mark.parametrize( + "model_id,expected_extra_body", + [ + # A model that is NOT in any catalog but matches a known series + # token in ``infer_interleaved_capability``. Demonstrates that + # the series-token fallback (catalog → inference) still produces + # the right wire format. Model ids here are constructed to + # embed a real token from ``_PROMOTE_REASONING_CONTENT_TOKENS`` / + # ``_STRICT_REASONING_CONTENT_TOKENS`` so the substring match + # fires regardless of where Flocks runs the test. + ("qwen3-7b-uncatalogued", {"enable_thinking": True}), + ("glm-5-uncatalogued", GLM_THINKING_EXTRA_BODY), + ("kimi-k2.6-uncatalogued", KIMI_THINKING_EXTRA_BODY), + ("mimo-v2.5-pro-uncatalogued", MIMO_THINKING_EXTRA_BODY), + ("minimax-m4-uncatalogued", {"reasoning_split": True}), + ("step-3.5-flash-uncatalogued", {"enable_thinking": True}), + ], + ) + def test_series_token_fallback_emits_expected_extra_body( + self, + model_id: str, + expected_extra_body: Dict[str, Any], + ) -> None: + """Models matching a known series token in + ``infer_interleaved_capability`` get the expected extra_body on the wire + even when the catalog has no explicit declaration for them. + + This is the regression net for the design choice that the dispatch + is *not* provider-keyed: a user-configured openai-compatible + endpoint pointing at a known family Just Works, without requiring + anyone to edit a per-provider registry. + """ + options = provider_options.build_provider_options( + "openai-compatible", model_id, resolve_max_tokens=False, + ) + assert options.get("extra_body") == expected_extra_body, ( + f"openai-compatible/{model_id} matches a known series token; " + "series-token fallback should have inferred interleaved and " + f"emitted {expected_extra_body}. options={options!r}" + ) + + @pytest.mark.parametrize( + "provider_id,model_id,expected_extra_body", + [ + ("deepseek", "deepseek-reasoner", DEEPSEEK_THINKING_EXTRA_BODY), + ("deepseek", "deepseek-v4-flash", DEEPSEEK_THINKING_EXTRA_BODY), + ("minimax", "minimax-m3", {"reasoning_split": True}), + ("stepfun", "step-3.5-flash", {"enable_thinking": True}), + ("zhipu", "glm-4.7", GLM_THINKING_EXTRA_BODY), + ], + ) + def test_real_catalog_chain_emits_expected_extra_body( + self, + provider_id: str, + model_id: str, + expected_extra_body: Dict[str, Any], + ) -> None: + """Exercise catalog/inference/dispatch without monkeypatching.""" + options = provider_options.build_provider_options( + provider_id, + model_id, + resolve_max_tokens=False, + ) + assert options.get("extra_body") == expected_extra_body + + +class TestOpenAICompatibleExtraBody: + """Verify the SDK now propagates caller-supplied ``extra_body`` instead + of silently swallowing it. This is the second-order bug: even if + ``build_provider_options`` produces the right shape, ``chat_stream`` / + ``chat`` in ``openai_compatible.py`` dropped the kwargs it received. + """ + + def test_chat_non_streaming_propagates_extra_body( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The non-streaming ``chat`` path must preserve extra_body too.""" + import asyncio + from types import SimpleNamespace + from unittest.mock import MagicMock + + from flocks.provider.sdk.openai_compatible import OpenAICompatibleProvider + + captured: Dict[str, Any] = {} + + class _FakeCompletions: + async def create(self, **kwargs: Any) -> Any: + captured.update(kwargs) + return SimpleNamespace( + id="chatcmpl-test", + model="qwen3-235b-a22b-thinking", + choices=[ + SimpleNamespace( + message=SimpleNamespace(content="ok"), + finish_reason="stop", + ) + ], + usage=SimpleNamespace( + prompt_tokens=1, + completion_tokens=1, + total_tokens=2, + ), + ) + + class _FakeChat: + completions = _FakeCompletions() + + class _FakeClient: + chat = _FakeChat() + + provider = OpenAICompatibleProvider() + provider._get_client = MagicMock(return_value=_FakeClient()) # type: ignore[method-assign] + + asyncio.run( + provider.chat( + "qwen3-235b-a22b-thinking", + messages=[], + extra_body={"enable_thinking": True}, + ) + ) + + assert captured.get("extra_body") == {"enable_thinking": True}, ( + "openai_compatible.chat swallowed caller-supplied extra_body" + ) + + def test_chat_stream_propagates_extra_body(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Smoke test that an ``extra_body`` kwarg passed to ``chat_stream`` + ends up in the outgoing request params. + + We mock the OpenAI client so we don't need a live API, then assert + the captured kwargs include the extra_body we passed in. The fake + stream yields one minimal chunk so the empty-response fallback (which + would call the non-streaming ``chat``) doesn't fire — the non-stream + path has its own check in + ``test_chat_non_streaming_propagates_extra_body``. + """ + import asyncio + from types import SimpleNamespace + from unittest.mock import MagicMock + + from flocks.provider.sdk.openai_compatible import OpenAICompatibleProvider + + captured: Dict[str, Any] = {} + + def _make_fake_response_object() -> Any: + """Build a minimal response object that satisfies both + chat_stream's chunk iteration and chat()'s .choices[0].message + access. The chunk carries non-empty content so chat_stream's + ``emitted_substantive_chunk`` flag flips and the empty-response + fallback (which calls ``self.chat`` and would need a real + response object) doesn't fire. + """ + chunk = SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="ok", tool_calls=None), + finish_reason="stop", + ) + ], + usage=None, + ) + + class _FakeStream: + def __aiter__(self) -> "_FakeStream": + return self + + async def __anext__(self): + if not getattr(self, "_emitted", False): + self._emitted = True + return chunk + raise StopAsyncIteration + + return _FakeStream() + + class _FakeCompletions: + async def create(self, **kwargs: Any) -> Any: + captured.update(kwargs) + return _make_fake_response_object() + + class _FakeChat: + completions = _FakeCompletions() + + class _FakeClient: + chat = _FakeChat() + + provider = OpenAICompatibleProvider() + provider._get_client = MagicMock(return_value=_FakeClient()) # type: ignore[method-assign] + + async def _drive() -> None: + async for _ in provider.chat_stream( + "qwen3-235b-a22b-thinking", + messages=[], + extra_body={"enable_thinking": True}, + ): + pass + + asyncio.run(_drive()) + + assert captured.get("extra_body") == {"enable_thinking": True}, ( + "openai_compatible.chat_stream swallowed the caller-supplied extra_body; " + "this is the second-order bug fixed in this change. captured keys: " + f"{sorted(captured.keys())}" + ) diff --git a/tests/sandbox/test_workflow_sandbox_runtime.py b/tests/sandbox/test_workflow_sandbox_runtime.py index 37057310d..dcd9daa7f 100644 --- a/tests/sandbox/test_workflow_sandbox_runtime.py +++ b/tests/sandbox/test_workflow_sandbox_runtime.py @@ -418,10 +418,13 @@ def run(self, name, **kwargs): return {"ok": True, "name": name} class FakeLLM: - def ask(self, prompt, temperature=0.2): + def ask(self, prompt, temperature=0.2, **_kwargs): return f"LLM:{prompt}:{temperature}" - monkeypatch.setattr("flocks.workflow.repl_runtime.get_lazy_llm", lambda: FakeLLM()) + monkeypatch.setattr( + "flocks.workflow.repl_runtime.get_lazy_llm", + lambda **_kwargs: FakeLLM(), + ) runtime = SandboxPythonExecRuntime( sandbox={"container_name": "c", "workspace_dir": "/tmp", "container_workdir": "/workspace"}, tool_registry=FakeRegistry(), diff --git a/tests/server/routes/test_custom_provider_routes.py b/tests/server/routes/test_custom_provider_routes.py index f1c9b58ac..05e96a00b 100644 --- a/tests/server/routes/test_custom_provider_routes.py +++ b/tests/server/routes/test_custom_provider_routes.py @@ -37,7 +37,8 @@ def temp_custom_provider_project(tmp_path, monkeypatch): @pytest.fixture async def client(): transport = ASGITransport(app=app) - async with AsyncClient(transport=transport, base_url="http://test") as ac: + headers = {"Authorization": "Bearer abc123", "User-Agent": "curl/8.0"} + async with AsyncClient(transport=transport, base_url="http://test", headers=headers) as ac: yield ac @@ -79,3 +80,29 @@ async def test_create_custom_model_accepts_string_currency( assert raw is not None assert raw["models"]["minimax:MiniMax-M2.7"]["currency"] == "USD" assert Provider._models["minimax:MiniMax-M2.7"].pricing["currency"] == "USD" + + +@pytest.mark.asyncio +async def test_create_custom_model_defaults_reasoning_on( + client: AsyncClient, + temp_custom_provider_project, + monkeypatch: pytest.MonkeyPatch, +): + from flocks.config.config_writer import ConfigWriter + from flocks.provider.provider import Provider + + monkeypatch.setattr(Provider, "_models", Provider._models.copy()) + + response = await client.post( + "/api/custom/models/custom-tb-inner", + json={ + "model_id": "custom-reasoning-default", + "name": "Custom Reasoning Default", + }, + ) + + assert response.status_code == 201, response.text + raw = ConfigWriter.get_provider_raw("custom-tb-inner") + assert raw is not None + assert raw["models"]["custom-reasoning-default"]["supports_reasoning"] is True + assert Provider._models["custom-reasoning-default"].capabilities.supports_reasoning is True diff --git a/tests/server/routes/test_device_routes.py b/tests/server/routes/test_device_routes.py index 8a4cd5148..13abfca55 100644 --- a/tests/server/routes/test_device_routes.py +++ b/tests/server/routes/test_device_routes.py @@ -30,6 +30,7 @@ def _fake_row(*, fields: Dict[str, str], verify_ssl: bool = False) -> dict: """ return { "id": "dev-test", + "storage_key": "onesec_api_v2_8_2", "fields": json.dumps(fields), "verify_ssl": int(bool(verify_ssl)), } @@ -196,3 +197,64 @@ async def fake_fetch_device(device_id: str): resp = await client.post("/api/devices/missing-id/test", json={}) assert resp.status_code == 404 + + +class TestDeviceCredentialEndpoint: + @pytest.mark.asyncio + async def test_reveals_only_requested_field_and_emits_audit( + self, client: AsyncClient, monkeypatch: pytest.MonkeyPatch + ): + captured: dict = {} + + async def fake_fetch_device(device_id: str): + captured["device_id"] = device_id + return _fake_row( + fields={ + "api_key": "{secret:device_dev-test_api_key}", + "base_url": "https://console.onesec.net", + } + ) + + monkeypatch.setattr(device_routes, "fetch_device", fake_fetch_device) + monkeypatch.setattr( + device_routes, + "resolve_for_runtime", + lambda db_fields: { + **db_fields, + "api_key": "long-real-onesec-api-key-Cd4Y", + }, + ) + async def fake_emit_audit(event_type: str, payload: dict): + captured["audit_event_type"] = event_type + captured["audit_payload"] = payload + + monkeypatch.setattr(device_routes, "_emit_device_audit", fake_emit_audit) + + resp = await client.post( + "/api/devices/dev-test/credentials", + json={"field": "api_key"}, + ) + + assert resp.status_code == 200, resp.text + assert captured["device_id"] == "dev-test" + assert captured["audit_event_type"] == "device.credentials_reveal" + assert captured["audit_payload"]["device_id"] == "dev-test" + assert captured["audit_payload"]["field_keys"] == ["api_key"] + assert resp.json() == { + "fields": { + "api_key": "long-real-onesec-api-key-Cd4Y", + } + } + + @pytest.mark.asyncio + async def test_returns_404_for_unknown_device( + self, client: AsyncClient, monkeypatch: pytest.MonkeyPatch + ): + async def fake_fetch_device(device_id: str): + return None + + monkeypatch.setattr(device_routes, "fetch_device", fake_fetch_device) + + resp = await client.post("/api/devices/missing-id/credentials", json={}) + + assert resp.status_code == 404 diff --git a/tests/server/routes/test_file_routes.py b/tests/server/routes/test_file_routes.py new file mode 100644 index 000000000..307a93773 --- /dev/null +++ b/tests/server/routes/test_file_routes.py @@ -0,0 +1,55 @@ +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + + +@pytest.fixture() +def file_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + data = tmp_path / "data" + workspace = tmp_path / "workspace" + data.mkdir() + workspace.mkdir() + + monkeypatch.setenv("FLOCKS_DATA_DIR", str(data)) + monkeypatch.setenv("FLOCKS_WORKSPACE_DIR", str(workspace)) + + from flocks.config.config import Config + from flocks.workspace.manager import WorkspaceManager + + Config._global_config = None + WorkspaceManager._instance = None + + from flocks.server.routes.file import router + + app = FastAPI() + app.include_router(router, prefix="/api/file") + + yield TestClient(app, raise_server_exceptions=True), data, workspace + + Config._global_config = None + WorkspaceManager._instance = None + + +def test_download_file_returns_binary_from_data_dir(file_client): + client, data, _workspace = file_client + image = data / "channel_media" / "wecom" / "image.png" + image.parent.mkdir(parents=True) + image.write_bytes(b"\x89PNG\r\n\x1a\n") + + response = client.get("/api/file/download", params={"path": str(image)}) + + assert response.status_code == 200 + assert response.content == b"\x89PNG\r\n\x1a\n" + assert response.headers["content-type"].startswith("image/png") + + +def test_download_file_rejects_unallowed_path(file_client, tmp_path: Path): + client, _data, _workspace = file_client + secret = tmp_path / "outside.png" + secret.write_bytes(b"nope") + + response = client.get("/api/file/download", params={"path": str(secret)}) + + assert response.status_code == 403 diff --git a/tests/server/routes/test_session_routes.py b/tests/server/routes/test_session_routes.py index e649d7351..39ae2765b 100644 --- a/tests/server/routes/test_session_routes.py +++ b/tests/server/routes/test_session_routes.py @@ -20,6 +20,15 @@ from fastapi import HTTPException, status from httpx import AsyncClient from flocks.auth.context import AuthUser +from flocks.session.core.status import SessionStatus, SessionStatusBusy +from flocks.session.message import ( + Message, + MessageRole, + ToolPart, + ToolStateError, + ToolStateRunning, +) +from flocks.session.orphan_tools import INTERRUPTED_TOOL_ERROR from flocks.session.session import Session # =========================================================================== @@ -178,6 +187,90 @@ async def test_send_message_noReply(self, client: AsyncClient, session_id: str): for m in messages ) + @pytest.mark.asyncio + async def test_list_messages_keeps_running_tool_when_session_busy( + self, + client: AsyncClient, + session_id: str, + ): + msg = await Message.create(session_id, MessageRole.ASSISTANT, "") + part = ToolPart( + id="part_busy_running", + sessionID=session_id, + messageID=msg.id, + callID="call_busy_running", + tool="bash", + state=ToolStateRunning( + input={"cmd": "sleep 60"}, + time={"start": 1000}, + ), + ) + await Message.store_part(session_id, msg.id, part) + + SessionStatus.set(session_id, SessionStatusBusy()) + try: + resp = await client.get(f"/api/session/{session_id}/message") + finally: + SessionStatus.clear(session_id) + + assert resp.status_code == status.HTTP_200_OK + parts = await Message.parts(msg.id, session_id) + running_part = next(p for p in parts if p.id == "part_busy_running") + assert running_part.state.status == "running" + + @pytest.mark.asyncio + async def test_list_messages_recovers_orphan_running_tool_when_session_idle( + self, + client: AsyncClient, + session_id: str, + ): + msg = await Message.create(session_id, MessageRole.ASSISTANT, "") + part = ToolPart( + id="part_idle_running", + sessionID=session_id, + messageID=msg.id, + callID="call_idle_running", + tool="bash", + state=ToolStateRunning( + input={"cmd": "sleep 60"}, + metadata={"sessionId": "ses_child"}, + time={"start": 1000}, + ), + ) + await Message.store_part(session_id, msg.id, part) + + resp = await client.get(f"/api/session/{session_id}/message") + + assert resp.status_code == status.HTTP_200_OK + parts = await Message.parts(msg.id, session_id) + repaired_part = next(p for p in parts if p.id == "part_idle_running") + assert isinstance(repaired_part.state, ToolStateError) + assert repaired_part.state.status == "error" + assert repaired_part.state.error == INTERRUPTED_TOOL_ERROR + assert repaired_part.state.metadata == {"sessionId": "ses_child"} + assert repaired_part.state.time["start"] == 1000 + assert repaired_part.state.time["end"] >= 1000 + + @pytest.mark.asyncio + async def test_list_messages_uses_preloaded_orphan_recovery_path( + self, + client: AsyncClient, + session_id: str, + monkeypatch: pytest.MonkeyPatch, + ): + from flocks.session import orphan_tools + + preloaded_recovery = AsyncMock(return_value=0) + legacy_recovery = AsyncMock(side_effect=AssertionError("legacy recovery should not be called")) + monkeypatch.setattr(orphan_tools, "abort_orphan_running_parts_in_messages", preloaded_recovery) + monkeypatch.setattr(orphan_tools, "abort_orphan_running_parts", legacy_recovery) + + resp = await client.get(f"/api/session/{session_id}/message") + + assert resp.status_code == status.HTTP_200_OK + preloaded_recovery.assert_awaited_once() + legacy_recovery.assert_not_called() + # =========================================================================== # Delete permissions (single-admin model) diff --git a/tests/server/routes/test_user_defined_pages_routes.py b/tests/server/routes/test_user_defined_pages_routes.py new file mode 100644 index 000000000..8d35ec577 --- /dev/null +++ b/tests/server/routes/test_user_defined_pages_routes.py @@ -0,0 +1,251 @@ +import io +import json +import zipfile +from unittest.mock import AsyncMock, patch + +import pytest +from httpx import AsyncClient + +from flocks.server.app import app +from flocks.server.auth import require_admin +from flocks.server.routes import user_defined_pages as user_defined_pages_routes +from flocks.user_defined_pages.builder import UserDefinedPagesBuilder +from flocks.user_defined_pages.models import UserDefinedPageBuildMeta +from flocks.user_defined_pages.store import UserDefinedPagesStore + + +def _make_page_archive(page_id: str, manifest: dict, extra_files: dict[str, str] | None = None) -> bytes: + buffer = io.BytesIO() + files = { + "manifest.json": json.dumps(manifest), + "src/index.tsx": "export default function Page(){return
ok
;}", + } + if extra_files: + files.update(extra_files) + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for relative_path, content in files.items(): + zf.writestr(f"{page_id}/{relative_path}", content) + return buffer.getvalue() + + +@pytest.fixture +def user_defined_pages_env(tmp_path, monkeypatch): + root = tmp_path / "user_defined_pages" + monkeypatch.setenv("FLOCKS_USER_DEFINED_PAGES_ROOT", str(root)) + store = UserDefinedPagesStore() + builder = UserDefinedPagesBuilder(store) + user_defined_pages_routes.reset_route_dependencies(store=store, builder=builder) + return store + + +@pytest.mark.asyncio +async def test_create_and_list_user_defined_pages(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + create_resp = await client.post( + "/api/user-defined-pages", + json={"id": "dash-1", "title": "仪表盘"}, + ) + assert create_resp.status_code == 201, create_resp.text + data = create_resp.json() + assert data["manifest"]["id"] == "dash-1" + + list_resp = await client.get("/api/user-defined-pages", params={"enabledOnly": True}) + assert list_resp.status_code == 200 + items = list_resp.json() + assert len(items) == 1 + assert items[0]["title"] == "仪表盘" + assert items[0]["route"] == "/user-defined-pages/dash-1" + + +@pytest.mark.asyncio +async def test_save_source_triggers_build_and_event(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + await client.post("/api/user-defined-pages", json={"id": "live-page", "title": "实时页"}) + source = user_defined_pages_env.read_source_file("live-page", "src/Page.tsx") + + with patch("flocks.server.routes.user_defined_pages._builder.build") as build_mock: + build_mock.return_value = UserDefinedPageBuildMeta( + status="ready", + hash="abc123", + builtAt=1, + error=None, + ) + with patch("flocks.server.routes.user_defined_pages.publish_event", new_callable=AsyncMock) as publish_mock: + save_resp = await client.put( + "/api/user-defined-pages/live-page", + json={"sourcePath": "src/Page.tsx", "sourceContent": source}, + ) + + assert save_resp.status_code == 200, save_resp.text + body = save_resp.json() + assert body["build"]["status"] == "ready" + publish_mock.assert_any_await("user_defined_pages.updated", {"id": "live-page", "hash": "abc123"}) + + +@pytest.mark.asyncio +async def test_bundle_endpoint_available_after_create(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + await client.post("/api/user-defined-pages", json={"id": "empty-page", "title": "空页面"}) + bundle_resp = await client.get("/api/user-defined-pages/empty-page/bundle.js") + assert bundle_resp.status_code == 200 + assert "application/javascript" in bundle_resp.headers.get("content-type", "") + assert bundle_resp.text.strip() + + +@pytest.mark.asyncio +async def test_reject_invalid_page_id_on_create(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + resp = await client.post("/api/user-defined-pages", json={"id": "../bad", "title": "坏页面"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_admin_required_for_create(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + from fastapi import HTTPException, Request + + def _deny_admin(_request: Request): + raise HTTPException(status_code=403, detail="仅管理员可执行该操作") + + app.dependency_overrides[require_admin] = _deny_admin + try: + resp = await client.post("/api/user-defined-pages", json={"id": "denied-page", "title": "禁止"}) + assert resp.status_code == 403 + finally: + app.dependency_overrides.pop(require_admin, None) + + +@pytest.mark.asyncio +async def test_admin_required_for_build_and_api_reload(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + from fastapi import HTTPException, Request + + await client.post("/api/user-defined-pages", json={"id": "admin-guard-page", "title": "权限页"}) + user_defined_pages_env.save_source_file( + "admin-guard-page", + "api/routes.yaml", + "routes:\n - method: GET\n path: /x\n handler: handlers.x\n", + ) + user_defined_pages_env.save_source_file( + "admin-guard-page", + "api/handlers.py", + "def x(ctx, request):\n return {'ok': True}\n", + ) + + def _deny_admin(_request: Request): + raise HTTPException(status_code=403, detail="仅管理员可执行该操作") + + app.dependency_overrides[require_admin] = _deny_admin + try: + build_resp = await client.post("/api/user-defined-pages/admin-guard-page/build") + assert build_resp.status_code == 403 + + reload_resp = await client.post("/api/user-defined-pages/admin-guard-page/api/reload") + assert reload_resp.status_code == 403 + finally: + app.dependency_overrides.pop(require_admin, None) + + +@pytest.mark.asyncio +async def test_page_api_routes_reload_and_dispatch(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + await client.post("/api/user-defined-pages", json={"id": "api-page", "title": "接口页"}) + user_defined_pages_env.save_source_file( + "api-page", + "api/routes.yaml", + "routes:\n - method: GET\n path: /stats\n handler: handlers.get_stats\n", + ) + user_defined_pages_env.save_source_file( + "api-page", + "api/handlers.py", + "def get_stats(ctx, request):\n return {'ok': True}\n", + ) + + list_resp = await client.get("/api/user-defined-pages/api-page/api") + assert list_resp.status_code == 200 + assert list_resp.json()[0]["path"] == "/stats" + + reload_resp = await client.post("/api/user-defined-pages/api-page/api/reload") + assert reload_resp.status_code == 200 + assert reload_resp.json()["routes"][0]["handler"] == "handlers.get_stats" + + dispatch_resp = await client.get("/api/user-defined-pages/api-page/api/stats") + assert dispatch_resp.status_code == 200 + assert dispatch_resp.json()["ok"] is True + + +@pytest.mark.asyncio +async def test_export_and_import_user_defined_page(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + await client.post("/api/user-defined-pages", json={"id": "backup-page", "title": "备份页"}) + await client.put( + "/api/user-defined-pages/backup-page", + json={"sourcePath": "src/Page.tsx", "sourceContent": "export default function Page(){return
backup
;}"}, + ) + + export_resp = await client.get("/api/user-defined-pages/backup-page/export") + assert export_resp.status_code == 200 + assert export_resp.headers.get("content-type", "").startswith("application/zip") + + import_resp = await client.post( + "/api/user-defined-pages/import?overwrite=true", + files={"file": ("backup-page.zip", export_resp.content, "application/zip")}, + ) + assert import_resp.status_code == 200 + assert import_resp.json()["manifest"]["id"] == "backup-page" + + +@pytest.mark.asyncio +async def test_import_normalizes_manifest_identity(client: AsyncClient, user_defined_pages_env: UserDefinedPagesStore): + archive = _make_page_archive( + "fixed-page", + { + "id": "wrong-page", + "title": "导入页", + "route": "/user-defined-pages/wrong-page", + "icon": "LayoutDashboard", + "order": 10, + "enabled": True, + "placement": "home.after", + "entry": "src/index.tsx", + "updatedAt": 1, + }, + ) + + import_resp = await client.post( + "/api/user-defined-pages/import", + files={"file": ("fixed-page.zip", archive, "application/zip")}, + ) + + assert import_resp.status_code == 200, import_resp.text + body = import_resp.json() + assert body["manifest"]["id"] == "fixed-page" + assert body["manifest"]["route"] == "/user-defined-pages/fixed-page" + + list_resp = await client.get("/api/user-defined-pages") + assert list_resp.status_code == 200 + assert list_resp.json()[0]["id"] == "fixed-page" + assert list_resp.json()[0]["route"] == "/user-defined-pages/fixed-page" + + +@pytest.mark.asyncio +async def test_import_rejects_archives_with_too_many_files( + client: AsyncClient, + user_defined_pages_env: UserDefinedPagesStore, + monkeypatch, +): + monkeypatch.setattr(user_defined_pages_routes, "MAX_IMPORT_FILES", 1) + archive = _make_page_archive( + "too-many", + { + "id": "too-many", + "title": "过多文件", + "route": "/user-defined-pages/too-many", + "icon": "LayoutDashboard", + "order": 10, + "enabled": True, + "placement": "home.after", + "entry": "src/index.tsx", + "updatedAt": 1, + }, + ) + + import_resp = await client.post( + "/api/user-defined-pages/import", + files={"file": ("too-many.zip", archive, "application/zip")}, + ) + + assert import_resp.status_code == 400 + assert "too many files" in import_resp.text diff --git a/tests/server/routes/test_workflow_run_route.py b/tests/server/routes/test_workflow_run_route.py index 77061dc1f..af1b4aa7a 100644 --- a/tests/server/routes/test_workflow_run_route.py +++ b/tests/server/routes/test_workflow_run_route.py @@ -66,10 +66,28 @@ async def test_save_kafka_config_persists_consumer_settings( storage_write = AsyncMock(return_value=None) restart_workflow = AsyncMock(return_value={"state": "running", "error": None}) + persisted_triggers: list[list[str]] = [] - monkeypatch.setattr(workflow_module, "_read_workflow_from_fs", lambda _workflow_id: {"workflowJson": {}}) + monkeypatch.setattr( + workflow_module, + "_read_workflow_from_fs", + lambda _workflow_id: {"id": "wf-input", "workflowJson": {}}, + ) monkeypatch.setattr(workflow_module.Storage, "write", storage_write) monkeypatch.setattr(kafka_manager.default_manager, "restart_workflow", restart_workflow) + monkeypatch.setattr(workflow_module, "_get_workflow_trigger_defs", AsyncMock(return_value=[])) + + async def _fake_persist(workflow_id: str, workflow_data: dict, triggers: list) -> dict: + persisted_triggers.append([trigger.id for trigger in triggers]) + return { + **workflow_data, + "workflowJson": { + **workflow_data["workflowJson"], + "triggers": [trigger.model_dump(mode="json") for trigger in triggers], + }, + } + + monkeypatch.setattr(workflow_module, "_persist_workflow_triggers", _fake_persist) req = workflow_module.KafkaConfigRequest( enabled=True, @@ -101,4 +119,59 @@ async def test_save_kafka_config_persists_consumer_settings( assert "outputEnabled" not in saved_config assert "outputBroker" not in saved_config assert "outputTopic" not in saved_config + assert persisted_triggers == [["kafka-default"]] + restart_workflow.assert_awaited_once_with("wf-input") + + +@pytest.mark.asyncio +async def test_save_syslog_config_persists_listener_settings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from flocks.ingest.syslog import manager as syslog_manager + + storage_write = AsyncMock(return_value=None) + restart_workflow = AsyncMock(return_value={"state": "listening", "error": None}) + persisted_triggers: list[list[str]] = [] + + monkeypatch.setattr( + workflow_module, + "_read_workflow_from_fs", + lambda _workflow_id: {"id": "wf-input", "workflowJson": {}}, + ) + monkeypatch.setattr(workflow_module.Storage, "write", storage_write) + monkeypatch.setattr(syslog_manager.default_manager, "restart_workflow", restart_workflow) + monkeypatch.setattr(workflow_module, "_get_workflow_trigger_defs", AsyncMock(return_value=[])) + + async def _fake_persist(workflow_id: str, workflow_data: dict, triggers: list) -> dict: + persisted_triggers.append([trigger.id for trigger in triggers]) + return { + **workflow_data, + "workflowJson": { + **workflow_data["workflowJson"], + "triggers": [trigger.model_dump(mode="json") for trigger in triggers], + }, + } + + monkeypatch.setattr(workflow_module, "_persist_workflow_triggers", _fake_persist) + + req = workflow_module.SyslogConfigRequest( + enabled=True, + protocol="udp", + host="0.0.0.0", + port=5514, + format="auto", + inputKey="syslog_message", + ) + + response = await workflow_module.save_syslog_config("wf-input", req) + + assert response == {"ok": True, "listener": {"state": "listening", "error": None}} + storage_write.assert_awaited_once() + _, saved_config = storage_write.await_args.args + assert saved_config["enabled"] is True + assert saved_config["protocol"] == "udp" + assert saved_config["host"] == "0.0.0.0" + assert saved_config["port"] == 5514 + assert saved_config["inputKey"] == "syslog_message" + assert persisted_triggers == [["syslog-default"]] restart_workflow.assert_awaited_once_with("wf-input") diff --git a/tests/server/routes/test_workflow_trigger_routes.py b/tests/server/routes/test_workflow_trigger_routes.py new file mode 100644 index 000000000..845100e51 --- /dev/null +++ b/tests/server/routes/test_workflow_trigger_routes.py @@ -0,0 +1,484 @@ +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest +from httpx import AsyncClient + +from flocks.server.routes import workflow as workflow_routes + + +@pytest.mark.asyncio +async def test_list_workflow_triggers_returns_unified_status( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "schedule-default", + "type": "schedule", + "enabled": True, + "source": {"intervalSeconds": 60}, + } + ], + }, + } if workflow_id == "wf-1" else None, + ) + + async def _fake_statuses(_workflow_id: str, _workflow_json: dict[str, Any]) -> list[dict[str, Any]]: + return [ + { + "workflowId": "wf-1", + "triggerId": "schedule-default", + "triggerType": "schedule", + "state": "running", + } + ] + + monkeypatch.setattr( + workflow_routes, + "default_trigger_runtime", + SimpleNamespace(get_workflow_trigger_statuses=_fake_statuses), + ) + + response = await client.get("/api/workflow/wf-1/triggers") + + assert response.status_code == 200, response.text + body = response.json() + assert body[0]["trigger"]["id"] == "schedule-default" + assert body[0]["status"]["state"] == "running" + + +@pytest.mark.asyncio +async def test_list_workflow_triggers_respects_explicit_empty_trigger_list( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [], + }, + } if workflow_id == "wf-1" else None, + ) + + async def _fake_legacy_triggers(_workflow_id: str) -> list[Any]: + return [ + workflow_routes.TriggerDefinition.model_validate( + { + "id": "schedule-default", + "type": "schedule", + "enabled": True, + "source": {"intervalSeconds": 30}, + } + ) + ] + + async def _fake_statuses(_workflow_id: str, _workflow_json: dict[str, Any]) -> list[dict[str, Any]]: + return [] + + monkeypatch.setattr(workflow_routes, "_read_legacy_trigger_defs", _fake_legacy_triggers) + monkeypatch.setattr( + workflow_routes, + "default_trigger_runtime", + SimpleNamespace(get_workflow_trigger_statuses=_fake_statuses), + ) + + response = await client.get("/api/workflow/wf-1/triggers") + + assert response.status_code == 200, response.text + assert response.json() == [] + + +@pytest.mark.asyncio +async def test_preview_trigger_mapping_returns_mapped_inputs( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "hook-default", + "type": "custom_webhook", + "enabled": True, + "mapping": {"alert_data": "$.body.data[0]"}, + "filter": {"expr": "body.data[0].severity == 'high'"}, + } + ], + }, + }, + ) + + response = await client.post( + "/api/workflow/wf-1/triggers/hook-default/preview-mapping", + json={"body": {"data": [{"severity": "high"}]}}, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["matched"] is True + assert body["inputs"]["alert_data"] == {"severity": "high"} + assert body["inputs"]["_flocks"]["trigger"]["id"] == "hook-default" + + +@pytest.mark.asyncio +async def test_create_workflow_trigger_persists_and_restarts_runtime( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored_payloads: list[dict[str, Any]] = [] + + base_workflow = { + "id": "wf-1", + "name": "demo", + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + }, + } + + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: base_workflow if workflow_id == "wf-1" else None, + ) + + async def _fake_persist(workflow_id: str, workflow_data: dict[str, Any], triggers: list[Any]) -> dict[str, Any]: + stored_payloads.append( + { + "workflow_id": workflow_id, + "trigger_ids": [trigger.id for trigger in triggers], + } + ) + return { + **workflow_data, + "workflowJson": { + **workflow_data["workflowJson"], + "triggers": [trigger.model_dump(mode="json") for trigger in triggers], + }, + } + + runtime_calls: list[str] = [] + + async def _fake_restart(workflow_id: str, workflow_json: dict[str, Any]) -> dict[str, Any]: + runtime_calls.append(f"restart:{workflow_id}:{len(workflow_json.get('triggers', []))}") + return {} + + async def _fake_status(workflow_id: str, trigger: Any) -> dict[str, Any]: + return {"workflowId": workflow_id, "triggerId": trigger.id, "state": "ready"} + + monkeypatch.setattr(workflow_routes, "_persist_workflow_triggers", _fake_persist) + monkeypatch.setattr( + workflow_routes, + "default_trigger_runtime", + SimpleNamespace(restart_workflow=_fake_restart, get_trigger_status=_fake_status), + ) + + response = await client.post( + "/api/workflow/wf-1/triggers", + json={ + "id": "hook-default", + "type": "custom_webhook", + "enabled": True, + "source": {"path": "/alerts/demo", "method": "POST"}, + "mapping": {"payload": "$.body"}, + }, + ) + + assert response.status_code == 200, response.text + assert stored_payloads[0]["trigger_ids"] == ["hook-default"] + assert runtime_calls == ["restart:wf-1:1"] + assert response.json()["status"]["state"] == "ready" + + +@pytest.mark.asyncio +async def test_create_workflow_trigger_rejects_multiple_legacy_singletons( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "schedule-default", + "type": "schedule", + "enabled": True, + "source": {"intervalSeconds": 60}, + } + ], + }, + } if workflow_id == "wf-1" else None, + ) + + response = await client.post( + "/api/workflow/wf-1/triggers", + json={ + "id": "schedule-extra", + "type": "schedule", + "enabled": True, + "source": {"intervalSeconds": 300}, + }, + ) + + assert response.status_code == 409, response.text + assert "Only one schedule trigger" in response.json()["message"] + + +@pytest.mark.asyncio +async def test_delete_workflow_trigger_removes_definition_and_restarts_runtime( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored_payloads: list[dict[str, Any]] = [] + + base_workflow = { + "id": "wf-1", + "name": "demo", + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "hook-default", + "type": "custom_webhook", + "enabled": True, + "source": {"path": "/alerts/demo", "method": "POST"}, + "mapping": {"payload": "$.body"}, + } + ], + }, + } + + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: base_workflow if workflow_id == "wf-1" else None, + ) + + async def _fake_persist(workflow_id: str, workflow_data: dict[str, Any], triggers: list[Any]) -> dict[str, Any]: + stored_payloads.append( + { + "workflow_id": workflow_id, + "trigger_ids": [trigger.id for trigger in triggers], + } + ) + return { + **workflow_data, + "workflowJson": { + **workflow_data["workflowJson"], + "triggers": [trigger.model_dump(mode="json") for trigger in triggers], + }, + } + + runtime_calls: list[str] = [] + + async def _fake_restart(workflow_id: str, workflow_json: dict[str, Any]) -> dict[str, Any]: + runtime_calls.append(f"restart:{workflow_id}:{len(workflow_json.get('triggers', []))}") + return {} + + async def _fake_remove_legacy(*_args: Any, **_kwargs: Any) -> None: + return None + + monkeypatch.setattr(workflow_routes, "_persist_workflow_triggers", _fake_persist) + monkeypatch.setattr(workflow_routes, "_remove_legacy_trigger_state", _fake_remove_legacy) + monkeypatch.setattr( + workflow_routes, + "default_trigger_runtime", + SimpleNamespace(restart_workflow=_fake_restart), + ) + + response = await client.delete("/api/workflow/wf-1/triggers/hook-default") + + assert response.status_code == 200, response.text + assert stored_payloads[0]["trigger_ids"] == [] + assert runtime_calls == ["restart:wf-1:0"] + assert response.json() == {"ok": True, "triggerId": "hook-default"} + + +@pytest.mark.asyncio +async def test_webhook_route_authorizes_and_dispatches_trigger( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "hook-default", + "type": "custom_webhook", + "enabled": True, + "auth": {"type": "api_key", "apiKey": "demo-secret"}, + "mapping": {"payload": "$.body"}, + "source": {"path": "/webhook/workflows/wf-1/hook-default"}, + } + ], + }, + }, + ) + + async def _fake_dispatch_event(**kwargs: Any) -> dict[str, Any]: + event = kwargs["event"] + return { + "matched": True, + "executed": True, + "inputs": {"payload": event.body}, + "result": {"triggerId": kwargs["trigger"].id}, + } + + monkeypatch.setattr( + workflow_routes, + "default_trigger_runtime", + SimpleNamespace(dispatch_event=_fake_dispatch_event), + ) + + response = await client.post( + "/webhook/workflows/wf-1/hook-default", + headers={"x-api-key": "demo-secret"}, + json={"severity": "high"}, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["ok"] is True + assert body["executed"] is True + assert body["inputs"]["payload"] == {"severity": "high"} + assert isinstance(body["deliveryId"], str) + assert "result" not in body + + +@pytest.mark.asyncio +async def test_webhook_route_rejects_disabled_trigger( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "hook-default", + "type": "custom_webhook", + "enabled": False, + "auth": {"type": "api_key", "apiKey": "demo-secret"}, + } + ], + }, + }, + ) + + response = await client.post( + "/webhook/workflows/wf-1/hook-default", + headers={"x-api-key": "demo-secret"}, + json={"severity": "high"}, + ) + + assert response.status_code == 403, response.text + assert "disabled" in response.json()["message"] + + +@pytest.mark.asyncio +async def test_webhook_route_validates_hmac_signature( + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + workflow_routes, + "_read_workflow_from_fs", + lambda workflow_id: { + "id": workflow_id, + "workflowJson": { + "start": "n1", + "nodes": [{"id": "n1", "type": "python", "code": "result = {'ok': True}"}], + "edges": [], + "triggers": [ + { + "id": "hook-default", + "type": "custom_webhook", + "enabled": True, + "auth": { + "type": "hmac", + "secretRef": "secret://demo-hook", + "headerName": "x-signature", + }, + } + ], + }, + }, + ) + monkeypatch.setattr(workflow_routes, "_resolve_trigger_secret", lambda _ref: "demo-secret") + + async def _fake_dispatch_event(**_kwargs: Any) -> dict[str, Any]: + return {"matched": True, "executed": True, "inputs": {}} + + monkeypatch.setattr( + workflow_routes, + "default_trigger_runtime", + SimpleNamespace(dispatch_event=_fake_dispatch_event), + ) + + payload = b'{"severity":"high"}' + signature = workflow_routes.hmac.new( + b"demo-secret", + payload, + workflow_routes.hashlib.sha256, + ).hexdigest() + + ok_response = await client.post( + "/webhook/workflows/wf-1/hook-default", + headers={"x-signature": f"sha256={signature}", "content-type": "application/json"}, + content=payload, + ) + assert ok_response.status_code == 200, ok_response.text + + bad_response = await client.post( + "/webhook/workflows/wf-1/hook-default", + headers={"x-signature": "sha256=bad-signature", "content-type": "application/json"}, + content=payload, + ) + assert bad_response.status_code == 401, bad_response.text diff --git a/tests/server/test_auth_compat.py b/tests/server/test_auth_compat.py index a013e849f..d4014e2da 100644 --- a/tests/server/test_auth_compat.py +++ b/tests/server/test_auth_compat.py @@ -305,6 +305,10 @@ def test_channel_webhook_is_exempt_via_regex(self): assert auth_module.auth_middleware_exempt("/api/channel/wecom/webhook") is True assert auth_module.auth_middleware_exempt("/api/channel/feishu/webhook/") is True + def test_workflow_webhook_is_exempt_via_regex(self): + assert auth_module.auth_middleware_exempt("/webhook/workflows/wf-1/hook-default") is True + assert auth_module.auth_middleware_exempt("/webhook/workflows/wf-1/hook-default/") is True + def test_other_channel_subpaths_are_still_protected(self): # Only ``/webhook`` is public; ``/bind``, ``/restart``, ``/status`` # and friends still require auth. @@ -315,6 +319,7 @@ def test_other_channel_subpaths_are_still_protected(self): # Defense-in-depth: a malicious caller must not hide a protected path # behind a fake ``webhook`` segment. assert auth_module.auth_middleware_exempt("/api/channel/dingtalk/webhook/extra") is False + assert auth_module.auth_middleware_exempt("/webhook/workflows/wf-1/hook-default/extra") is False @pytest.mark.asyncio @@ -344,6 +349,26 @@ async def test_apply_auth_for_request_channel_webhook_passes_without_credentials auth_module.clear_auth_context(token) +@pytest.mark.asyncio +async def test_apply_auth_for_request_workflow_webhook_passes_without_credentials(monkeypatch): + monkeypatch.setattr( + auth_module, + "get_secret_manager", + lambda: _FakeSecrets({auth_module.API_TOKEN_SECRET_ID: "abc123"}), + ) + request = _make_request( + headers={"user-agent": "Alertmanager-Webhook"}, + client_host="203.0.113.20", + path="/webhook/workflows/wf-1/hook-default", + ) + blocked, token, user = await auth_module.apply_auth_for_request(request) + try: + assert blocked is None + assert user is None + finally: + auth_module.clear_auth_context(token) + + @pytest.mark.asyncio async def test_apply_auth_for_request_allows_password_reset_endpoints_when_required(monkeypatch): async def _has_users(): diff --git a/tests/server/test_input_dispatcher.py b/tests/server/test_input_dispatcher.py index 014272dba..a69bd2f80 100644 --- a/tests/server/test_input_dispatcher.py +++ b/tests/server/test_input_dispatcher.py @@ -111,14 +111,14 @@ async def test_llm_command_routes_raw_slash_text(self): event = UserInputEvent( source_type="webui", sessionID="ses_test", - text="/plan investigate routing", - parts=[{"type": "text", "text": "/plan investigate routing"}], + text="/bug investigate routing", + parts=[{"type": "text", "text": "/bug investigate routing"}], ) result = await dispatch_user_input(event, sink) assert result.action == "llm" - assert llm == [("/plan investigate routing", "/plan investigate routing")] + assert llm == [("/bug investigate routing", "/bug investigate routing")] assert not direct @pytest.mark.asyncio @@ -242,7 +242,7 @@ async def fake_provide(*, directory, init, fn): ), ) request = session_routes.PromptRequest( - parts=[{"type": "text", "text": "/plan investigate"}], + parts=[{"type": "text", "text": "/bug investigate"}], ) resp = await session_routes.send_session_message_async(session_id, request) @@ -250,7 +250,7 @@ async def fake_provide(*, directory, init, fn): await asyncio.sleep(0) dispatch_mock.assert_awaited_once() event = dispatch_mock.await_args.args[2] - assert event.text == "/plan investigate" + assert event.text == "/bug investigate" @pytest.mark.asyncio async def test_command_route_routes_through_dispatcher(self, monkeypatch): @@ -273,15 +273,15 @@ async def fake_provide(*, directory, init, fn): ) ), ) - request = session_routes.CommandRequest(command="plan", arguments="investigate") + request = session_routes.CommandRequest(command="bug", arguments="investigate") resp = await session_routes.send_session_command(session_id, request) assert resp["status"] == "accepted" await asyncio.sleep(0) dispatch_mock.assert_awaited_once() event = dispatch_mock.await_args.args[2] - assert event.text == "/plan investigate" - assert event.display_text == "/plan investigate" + assert event.text == "/bug investigate" + assert event.display_text == "/bug investigate" class TestPromptQueueRoutes: diff --git a/tests/server/test_log_routes.py b/tests/server/test_log_routes.py index 906aaae2b..455d70891 100644 --- a/tests/server/test_log_routes.py +++ b/tests/server/test_log_routes.py @@ -1,6 +1,10 @@ """Log route helpers.""" from pathlib import Path +from datetime import date + +import pytest +from fastapi import HTTPException from flocks.server.routes import logs as log_routes @@ -34,3 +38,83 @@ def test_read_log_file_reports_untruncated_small_file(tmp_path: Path) -> None: assert response.content == "one\ntwo" assert response.total_lines == 2 assert response.truncated is False + + +@pytest.mark.asyncio +async def test_list_logs_includes_root_and_date_log_files(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (tmp_path / "backend.log").write_text("backend\n", encoding="utf-8") + (tmp_path / "webui.log").write_text("webui\n", encoding="utf-8") + (day_dir / "flocks.log").write_text("main\n", encoding="utf-8") + (day_dir / "errors.log").write_text("errors\n", encoding="utf-8") + (tmp_path / "flocks.log.1").write_text("rotated\n", encoding="utf-8") + (tmp_path / "not-a-log.txt").write_text("ignore\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + response = await log_routes.list_logs() + + names = {item.name for item in response.files} + assert "backend.log" in names + assert "webui.log" in names + assert f"{today}/flocks.log" in names + assert f"{today}/errors.log" in names + assert "flocks.log.1" not in names + assert "not-a-log.txt" not in names + + +@pytest.mark.asyncio +async def test_latest_log_prefers_main_flocks_log(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (tmp_path / "backend.log").write_text("backend\n", encoding="utf-8") + (day_dir / "flocks.log").write_text("main\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + response = await log_routes.read_latest_log(tail=10) + + assert response.filename == f"{today}/flocks.log" + assert response.content == "main" + + +@pytest.mark.asyncio +async def test_read_log_allows_daily_log_files(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (day_dir / "flocks.log").write_text("main\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + response = await log_routes.read_log(f"{today}/flocks.log", tail=10) + + assert response.filename == f"{today}/flocks.log" + assert response.content == "main" + + +@pytest.mark.asyncio +async def test_read_log_rejects_rotated_suffix_files(tmp_path: Path, monkeypatch) -> None: + (tmp_path / "backend.log.1").write_text("rotated\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + with pytest.raises(HTTPException) as exc_info: + await log_routes.read_log("backend.log.1", tail=10) + assert exc_info.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_read_log_returns_same_nested_filename_as_list(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (day_dir / "errors.log").write_text("warn\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + listed = await log_routes.list_logs() + listed_name = next(item.name for item in listed.files if item.name.endswith("errors.log")) + response = await log_routes.read_log(listed_name, tail=10) + + assert listed_name == f"{today}/errors.log" + assert response.filename == listed_name + assert response.content == "warn" diff --git a/tests/session/test_orphan_tools.py b/tests/session/test_orphan_tools.py new file mode 100644 index 000000000..bee3a84a0 --- /dev/null +++ b/tests/session/test_orphan_tools.py @@ -0,0 +1,186 @@ +import pytest + +from flocks.session.message import ( + Message, + MessageRole, + ToolPart, + ToolStateCompleted, + ToolStateError, + ToolStateRunning, +) +from flocks.session.orphan_tools import ( + INTERRUPTED_TOOL_ERROR, + abort_all_orphan_running_parts, + abort_orphan_running_parts, + abort_orphan_running_parts_in_messages, +) +from flocks.session.core.status import SessionStatus, SessionStatusBusy +from flocks.session.session import Session + + +@pytest.mark.asyncio +async def test_abort_orphan_running_parts_marks_running_tools_error(): + session = await Session.create(project_id="proj_orphan", directory="/tmp") + msg = await Message.create(session.id, MessageRole.ASSISTANT, "") + part = ToolPart( + id="part_orphan_running", + sessionID=session.id, + messageID=msg.id, + callID="call_orphan_running", + tool="delegate_task", + state=ToolStateRunning( + input={"prompt": "keep going"}, + title="delegate_task", + metadata={"sessionId": "ses_child"}, + time={"start": 1234}, + ), + ) + + await Message.store_part(session.id, msg.id, part) + + repaired = await abort_orphan_running_parts(session.id) + parts = await Message.parts(msg.id, session.id) + repaired_part = next(p for p in parts if p.id == "part_orphan_running") + + assert repaired == 1 + assert isinstance(repaired_part.state, ToolStateError) + assert repaired_part.state.status == "error" + assert repaired_part.state.error == INTERRUPTED_TOOL_ERROR + assert repaired_part.state.input == {"prompt": "keep going"} + assert repaired_part.state.metadata == {"sessionId": "ses_child"} + assert repaired_part.state.time["start"] == 1234 + assert repaired_part.state.time["end"] >= 1234 + + +@pytest.mark.asyncio +async def test_abort_orphan_running_parts_leaves_terminal_tools_unchanged(): + session = await Session.create(project_id="proj_orphan_terminal", directory="/tmp") + msg = await Message.create(session.id, MessageRole.ASSISTANT, "") + completed = ToolPart( + id="part_orphan_completed", + sessionID=session.id, + messageID=msg.id, + callID="call_orphan_completed", + tool="bash", + state=ToolStateCompleted( + input={"cmd": "pwd"}, + output="/tmp", + title="bash", + metadata={}, + time={"start": 1000, "end": 2000}, + ), + ) + + await Message.store_part(session.id, msg.id, completed) + + repaired = await abort_orphan_running_parts(session.id) + parts = await Message.parts(msg.id, session.id) + completed_part = next(p for p in parts if p.id == "part_orphan_completed") + + assert repaired == 0 + assert completed_part.state.status == "completed" + assert completed_part.state.time == {"start": 1000, "end": 2000} + + +@pytest.mark.asyncio +async def test_abort_orphan_running_parts_in_messages_reuses_loaded_parts(): + session = await Session.create(project_id="proj_orphan_loaded", directory="/tmp") + msg = await Message.create(session.id, MessageRole.ASSISTANT, "") + running = ToolPart( + id="part_orphan_loaded_running", + sessionID=session.id, + messageID=msg.id, + callID="call_orphan_loaded_running", + tool="bash", + state=ToolStateRunning( + input={"cmd": "sleep 60"}, + metadata={"sessionId": "ses_child"}, + time={"start": 4321}, + ), + ) + completed = ToolPart( + id="part_orphan_loaded_completed", + sessionID=session.id, + messageID=msg.id, + callID="call_orphan_loaded_completed", + tool="bash", + state=ToolStateCompleted( + input={"cmd": "pwd"}, + output="/tmp", + title="bash", + metadata={}, + time={"start": 1000, "end": 2000}, + ), + ) + + await Message.store_part(session.id, msg.id, running) + await Message.store_part(session.id, msg.id, completed) + + messages = await Message.list_with_parts(session.id) + repaired = await abort_orphan_running_parts_in_messages(session.id, messages) + + assert repaired == 1 + repaired_running = next(p for p in messages[0].parts if p.id == "part_orphan_loaded_running") + untouched_completed = next(p for p in messages[0].parts if p.id == "part_orphan_loaded_completed") + assert isinstance(repaired_running.state, ToolStateError) + assert repaired_running.state.error == INTERRUPTED_TOOL_ERROR + assert repaired_running.state.metadata == {"sessionId": "ses_child"} + assert repaired_running.state.time["start"] == 4321 + assert repaired_running.state.time["end"] >= 4321 + assert untouched_completed.state.status == "completed" + + +@pytest.mark.asyncio +async def test_abort_all_orphan_running_parts_scans_persisted_sessions(): + session = await Session.create(project_id="proj_orphan_all", directory="/tmp") + msg = await Message.create(session.id, MessageRole.ASSISTANT, "") + part = ToolPart( + id="part_orphan_all", + sessionID=session.id, + messageID=msg.id, + callID="call_orphan_all", + tool="bash", + state=ToolStateRunning( + input={"cmd": "sleep 60"}, + time={"start": 5000}, + ), + ) + + await Message.store_part(session.id, msg.id, part) + + repaired = await abort_all_orphan_running_parts() + parts = await Message.parts(msg.id, session.id) + repaired_part = next(p for p in parts if p.id == "part_orphan_all") + + assert repaired == 1 + assert repaired_part.state.status == "error" + assert repaired_part.state.error == INTERRUPTED_TOOL_ERROR + + +@pytest.mark.asyncio +async def test_abort_all_orphan_running_parts_skips_busy_sessions(): + session = await Session.create(project_id="proj_orphan_busy", directory="/tmp") + msg = await Message.create(session.id, MessageRole.ASSISTANT, "") + part = ToolPart( + id="part_orphan_busy", + sessionID=session.id, + messageID=msg.id, + callID="call_orphan_busy", + tool="bash", + state=ToolStateRunning( + input={"cmd": "sleep 60"}, + time={"start": 5000}, + ), + ) + + await Message.store_part(session.id, msg.id, part) + SessionStatus.set(session.id, SessionStatusBusy()) + try: + repaired = await abort_all_orphan_running_parts() + finally: + SessionStatus.clear(session.id) + parts = await Message.parts(msg.id, session.id) + running_part = next(p for p in parts if p.id == "part_orphan_busy") + + assert repaired == 0 + assert running_part.state.status == "running" diff --git a/tests/session/test_prompt_tokens.py b/tests/session/test_prompt_tokens.py index 1604b2aea..66acdbc7e 100644 --- a/tests/session/test_prompt_tokens.py +++ b/tests/session/test_prompt_tokens.py @@ -14,10 +14,12 @@ import os import tempfile from pathlib import Path -from unittest.mock import MagicMock, patch +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch import pytest +from flocks.agent.agent import AgentInfo from flocks.session.prompt import ( PROMPT_DEFAULT, PromptTemplate, @@ -235,6 +237,69 @@ def test_omits_optional_lines_when_unset(self) -> None: assert "## Runtime Metadata" in block +class TestBuildSystemPrompts: + @pytest.mark.asyncio + async def test_builtin_system_subagent_child_uses_minimal_prompt(self): + agent = AgentInfo( + name="rex-junior", + mode="subagent", + tags=["system"], + prompt="You are Rex Junior.", + ) + with ( + patch("flocks.agent.registry.Agent.get", AsyncMock(return_value=agent)), + patch( + "flocks.session.session.Session.get_by_id", + AsyncMock(return_value=SimpleNamespace(parent_id="ses-parent")), + ), + ): + prompts = await SessionPrompt.build_system_prompts( + session_id="ses-child", + session_directory="/tmp/project", + agent_name="rex-junior", + agent_prompt="You are Rex Junior.", + provider_id="anthropic", + model_id="claude-sonnet", + tool_catalog_prompt_factory=lambda: "SHOULD_NOT_APPEAR", + ) + + assert len(prompts) == 2 + assert prompts[0] == "You are Rex Junior." + assert "## Environment" in prompts[1] + assert "Current working directory: /tmp/project" in prompts[1] + assert "Platform:" in prompts[1] + assert "Today's date:" in prompts[1] + assert "SHOULD_NOT_APPEAR" not in "\n".join(prompts) + assert PROMPT_DEFAULT.strip() not in "\n".join(prompts) + + @pytest.mark.asyncio + async def test_builtin_system_subagent_root_uses_full_prompt(self): + agent = AgentInfo( + name="rex-junior", + mode="subagent", + tags=["system"], + prompt="You are Rex Junior.", + ) + with ( + patch("flocks.agent.registry.Agent.get", AsyncMock(return_value=agent)), + patch( + "flocks.session.session.Session.get_by_id", + AsyncMock(return_value=SimpleNamespace(parent_id=None)), + ), + ): + prompts = await SessionPrompt.build_system_prompts( + session_id="ses-root", + session_directory="/tmp/project", + agent_name="rex-junior", + agent_prompt="You are Rex Junior.", + provider_id="anthropic", + model_id="claude-sonnet", + ) + + assert len(prompts) > 2 + assert any(PROMPT_DEFAULT.strip() in prompt for prompt in prompts) + + # --------------------------------------------------------------------------- # SystemPrompt.provider() — returns List[str] # --------------------------------------------------------------------------- @@ -282,6 +347,15 @@ def test_none_model_returns_list(self): except (AttributeError, TypeError): pytest.skip("provider(None) not supported by this implementation") + def test_provider_prompts_do_not_reference_retired_todo_tools(self): + prompt_dir = Path(__file__).resolve().parents[2] / "flocks" / "session" / "prompt" + retired = ("TodoWrite", "TodoRead", "todowrite", "todoread") + + for prompt_path in prompt_dir.glob("*.txt"): + content = prompt_path.read_text(encoding="utf-8") + for name in retired: + assert name not in content, f"{prompt_path.name} references retired tool {name}" + class TestPromptToolInstructions: def test_tool_instructions_are_platform_agnostic(self): diff --git a/tests/session/test_runner_step.py b/tests/session/test_runner_step.py index d32f4a6b3..a14d5c109 100644 --- a/tests/session/test_runner_step.py +++ b/tests/session/test_runner_step.py @@ -1967,6 +1967,118 @@ async def test_to_chat_messages_keeps_reasoning_only_assistant_message(monkeypat assert chat_messages[0].reasoning_source == "promoted_reasoning" +def test_get_queued_user_message_ids_only_marks_newly_queued_turns(): + runner = _make_runner("ses_runner_queued_users") + runner._step = 3 + + queued_user_ids = runner._get_queued_user_message_ids([ + SimpleNamespace(id="msg_100", role="assistant", finish="stop"), + SimpleNamespace(id="msg_200", role="user"), + SimpleNamespace(id="msg_300", role="user"), + SimpleNamespace(id="msg_400", role="user"), + ]) + + assert queued_user_ids == {"msg_300", "msg_400"} + + +@pytest.mark.asyncio +async def test_to_chat_messages_wraps_only_queued_user_messages(): + session = await Session.create( + project_id="test_runner_queued_wrap", + directory="/tmp/runner-queued-wrap", + ) + finished_assistant = await Message.create( + session_id=session.id, + role=MessageRole.ASSISTANT, + content="Initial answer", + ) + root_user = await Message.create( + session_id=session.id, + role=MessageRole.USER, + content="Continue fixing the bug", + ) + queued_user = await Message.create( + session_id=session.id, + role=MessageRole.USER, + content="What version is installed?", + ) + + runner = SessionRunner(session=session, static_cache={}) + runner._step = 3 + runner._queued_user_message_ids = {queued_user.id} + + chat_messages = await runner._to_chat_messages( + [ + SimpleNamespace( + id=finished_assistant.id, + role=MessageRole.ASSISTANT, + finish="stop", + error=None, + compacted=False, + ), + root_user, + queued_user, + ], + [], + ) + + user_messages = [message for message in chat_messages if message.role == "user"] + assert len(user_messages) == 2 + assert user_messages[0].content == "Continue fixing the bug" + assert isinstance(user_messages[1].content, str) + assert user_messages[1].content.startswith("") + assert "What version is installed?" in user_messages[1].content + + +@pytest.mark.asyncio +async def test_to_chat_messages_skips_reasoning_only_aborted_assistant(monkeypatch): + runner = _make_runner("ses_runner_aborted_reasoning_only") + runner.provider_id = "alibaba" + runner.model_id = "qwen3-max" + + monkeypatch.setattr( + runner_mod.Provider, + "resolve_model", + lambda provider_id, model_id: SimpleNamespace( + capabilities=SimpleNamespace( + interleaved={ + "field": "reasoning_content", + "echo": "tool_calls", + "cross_provider_policy": "promote", + } + ) + ), + ) + monkeypatch.setattr( + runner_mod.Message, + "parts", + AsyncMock(return_value=[ + ReasoningPart( + sessionID=runner.session.id, + messageID="msg_aborted_reasoning_only", + text="This should not leak into replay history.", + time=PartTime(start=1), + ) + ]), + ) + monkeypatch.setattr(runner_mod.Message, "get_parts_revision", lambda *_args: 1) + + chat_messages = await runner._to_chat_messages( + [ + SimpleNamespace( + id="msg_aborted_reasoning_only", + role=MessageRole.ASSISTANT, + finish="error", + error={"name": "MessageAbortedError"}, + compacted=False, + ) + ], + [], + ) + + assert chat_messages == [] + + def test_provider_capability_key_includes_interleaved_policy(monkeypatch): runner = _make_runner("ses_runner_interleaved_capability_key") runner.provider_id = "deepseek" @@ -2041,6 +2153,128 @@ async def fake_create(*args, **kwargs): assert captured_kwargs["provider_id"] == runner.provider_id +@pytest.mark.asyncio +async def test_process_step_invalidates_chat_cache_for_queued_messages(monkeypatch): + runner = _make_runner("ses_runner_queued_cache_invalidation") + runner._step = 3 + runner._static_cache["chat_context_cache"] = { + "latest": { + "system_cache_key": "cached", + "message_signatures": [], + "chat_messages": [{"role": "user", "content": "stale"}], + } + } + runner.callbacks = RunnerCallbacks(on_error=AsyncMock()) + + root_user = SimpleNamespace(id="msg_200", role="user") + last_user = UserMessageInfo( + id="msg_300", + sessionID=runner.session.id, + role="user", + time={"created": 1_000}, + agent="rex", + model={"providerID": "anthropic", "modelID": "claude-sonnet"}, + ) + messages = [ + SimpleNamespace(id="msg_100", role="assistant", finish="stop"), + root_user, + last_user, + ] + + agent = SimpleNamespace(name="rex", steps=None, mode="primary", prompt="", tools=[]) + provider = MagicMock() + provider.is_configured.return_value = True + assistant_msg = SimpleNamespace(id="msg_assistant_queued_cache") + + async def fake_to_chat_messages(_messages, _system_prompts): # noqa: ANN001 + assert "chat_context_cache" not in runner._static_cache + assert runner._queued_user_message_ids == {last_user.id} + return [SimpleNamespace(role="user", content="queued")] + + monkeypatch.setattr(runner_mod.Agent, "get", AsyncMock(return_value=agent)) + monkeypatch.setattr(runner_mod.Provider, "get", lambda provider_id: provider) + monkeypatch.setattr(runner_mod.Provider, "apply_config", AsyncMock(return_value=None)) + monkeypatch.setattr(runner_mod.SessionPrompt, "build_system_prompts", AsyncMock(return_value=[])) + monkeypatch.setattr(runner, "_build_callable_tool_schema", AsyncMock(return_value=[])) + monkeypatch.setattr(runner, "_to_chat_messages", fake_to_chat_messages) + monkeypatch.setattr(runner_mod.Message, "get_text_content", AsyncMock(return_value="queued")) + monkeypatch.setattr(runner_mod.Message, "create", AsyncMock(return_value=assistant_msg)) + monkeypatch.setattr(runner_mod.Message, "update", AsyncMock(return_value=None)) + monkeypatch.setattr( + runner, + "_call_llm", + AsyncMock(return_value=StepResult(action="stop", content="done")), + ) + + result = await runner._process_step(messages, last_user) + + assert result.action == "stop" + assert result.content == "done" + + +@pytest.mark.asyncio +async def test_process_step_marks_aborted_llm_message_as_error(monkeypatch): + runner = _make_runner("ses_runner_aborted_result") + runner.callbacks = RunnerCallbacks(on_error=AsyncMock()) + + last_user = UserMessageInfo( + id="msg_user_aborted_result", + sessionID=runner.session.id, + role="user", + time={"created": 1_000}, + agent="rex", + model={"providerID": "anthropic", "modelID": "claude-sonnet"}, + ) + agent = SimpleNamespace(name="rex", steps=None, mode="primary", prompt="", tools=[]) + provider = MagicMock() + provider.is_configured.return_value = True + assistant_msg = SimpleNamespace(id="msg_assistant_aborted_result") + update_mock = AsyncMock(return_value=None) + record_usage_mock = AsyncMock(return_value=None) + usage = {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13} + + async def fake_call_llm(*_args, **_kwargs): + runner._llm_call_aborted = True + return StepResult( + action="continue", + content="partial answer", + tool_calls=[ToolCall(id="call_1", name="read", arguments={})], + usage=usage, + ) + + monkeypatch.setattr(runner_mod.Agent, "get", AsyncMock(return_value=agent)) + monkeypatch.setattr(runner_mod.Provider, "get", lambda provider_id: provider) + monkeypatch.setattr(runner_mod.Provider, "apply_config", AsyncMock(return_value=None)) + monkeypatch.setattr(runner_mod.SessionPrompt, "build_system_prompts", AsyncMock(return_value=[])) + monkeypatch.setattr(runner, "_build_callable_tool_schema", AsyncMock(return_value=[])) + monkeypatch.setattr( + runner, + "_to_chat_messages", + AsyncMock(return_value=[SimpleNamespace(role="user", content="hi")]), + ) + monkeypatch.setattr(runner_mod.Message, "get_text_content", AsyncMock(return_value="hi")) + monkeypatch.setattr(runner_mod.Message, "create", AsyncMock(return_value=assistant_msg)) + monkeypatch.setattr(runner_mod.Message, "update", update_mock) + monkeypatch.setattr(runner, "_record_usage_if_available", record_usage_mock) + monkeypatch.setattr(runner, "_call_llm", fake_call_llm) + + result = await runner._process_step([last_user], last_user) + + assert result.action == "stop" + assert result.content == "partial answer" + assert any( + call.args[1] == assistant_msg.id + and call.kwargs.get("finish") == "error" + and call.kwargs.get("error", {}).get("name") == "MessageAbortedError" + for call in update_mock.await_args_list + ) + assert not any( + call.args[1] == assistant_msg.id and call.kwargs.get("finish") in {"stop", "tool-calls"} + for call in update_mock.await_args_list + ) + record_usage_mock.assert_awaited_once_with(usage, message_id=assistant_msg.id) + + @pytest.mark.asyncio async def test_call_llm_skips_observability_when_langfuse_inactive(monkeypatch): runner = _make_runner("ses_runner_langfuse_inactive") diff --git a/tests/session/test_session_abort_inject.py b/tests/session/test_session_abort_inject.py index cdd9f5b7c..ab3e06b9c 100644 --- a/tests/session/test_session_abort_inject.py +++ b/tests/session/test_session_abort_inject.py @@ -117,6 +117,50 @@ def test_runner_without_external_event(self): runner.abort() assert runner.is_aborted is True + @pytest.mark.asyncio + async def test_session_loop_run_publishes_busy_and_idle_status_events(self): + session_info = _make_session_info("status_event_session") + event_callback = AsyncMock() + callbacks = LoopCallbacks(event_publish_callback=event_callback) + + with patch( + "flocks.session.session_loop.Session.get_by_id", + AsyncMock(return_value=session_info), + ), patch( + "flocks.session.session_loop.Message.list", + AsyncMock(return_value=[]), + ), patch( + "flocks.session.orphan_tools.abort_orphan_running_parts", + AsyncMock(return_value=0), + ), patch( + "flocks.session.session_loop.SessionLoop._run_loop", + AsyncMock(return_value=LoopResult(action="stop")), + ), patch( + "flocks.session.session_loop.Session.touch", + AsyncMock(), + ), patch( + "flocks.bus.bus.Bus.publish", + AsyncMock(), + ): + result = await SessionLoop.run( + session_id=session_info.id, + provider_id="test-provider", + model_id="test-model", + agent_name="rex", + callbacks=callbacks, + ) + + assert result.action == "stop" + status_events = [ + call.args + for call in event_callback.await_args_list + if call.args and call.args[0] == "session.status" + ] + assert status_events == [ + ("session.status", {"sessionID": session_info.id, "status": {"type": "busy"}}), + ("session.status", {"sessionID": session_info.id, "status": {"type": "idle"}}), + ] + # --------------------------------------------------------------------------- # SessionLoop abort tests @@ -575,5 +619,3 @@ def test_step_counter_default(self): agent_name="test", ) assert ctx.step == 0 - - diff --git a/tests/session/test_stream_processor.py b/tests/session/test_stream_processor.py index 40f5896bd..1f4475e32 100644 --- a/tests/session/test_stream_processor.py +++ b/tests/session/test_stream_processor.py @@ -58,11 +58,13 @@ def _make_processor( text_callback=None, reasoning_callback=None, event_callback=None, + abort_event=None, ): return StreamProcessor( session_id=session_id, assistant_message=_make_assistant_msg(session_id), agent=_make_agent(), + abort_event=abort_event, text_delta_callback=text_callback, reasoning_delta_callback=reasoning_callback, event_publish_callback=event_callback, @@ -422,6 +424,123 @@ async def test_tool_call_executes_tool(self): assert "tc_exec" in proc.tool_calls + @pytest.mark.asyncio + async def test_tool_call_passes_session_abort_event_to_tool_context(self): + abort_event = asyncio.Event() + proc = _make_processor(abort_event=abort_event) + seen_abort = {} + + async def _fake_execute(*, tool_name, ctx, **kwargs): + seen_abort["event"] = ctx.abort + return ToolResult(success=True, output="ok", title=tool_name, metadata={}) + + with ( + patch("flocks.session.streaming.stream_processor.Message.store_part", new=AsyncMock()), + patch("flocks.session.streaming.stream_processor.Message.update_part", new=AsyncMock()), + patch( + "flocks.session.streaming.stream_processor.ToolRegistry.execute", + new=AsyncMock(side_effect=_fake_execute), + ), + ): + await proc.process_event(ToolInputStartEvent(id="tc_abort", tool_name="run_workflow")) + await proc.process_event( + ToolCallEvent(tool_call_id="tc_abort", tool_name="run_workflow", input={"workflow": "wf.json"}) + ) + + assert seen_abort["event"] is abort_event + + @pytest.mark.asyncio + async def test_cancelled_tool_blocks_late_running_metadata_updates(self): + event_callback = AsyncMock() + proc = _make_processor(event_callback=event_callback) + saved_cb = {} + + async def _cancelled_execute(*, tool_name, ctx, **kwargs): + saved_cb["cb"] = ctx._metadata_callback + raise asyncio.CancelledError() + + with ( + patch("flocks.session.streaming.stream_processor.Message.store_part", new=AsyncMock()), + patch("flocks.session.streaming.stream_processor.Message.update_part", new=AsyncMock()), + patch( + "flocks.session.streaming.stream_processor.ToolRegistry.execute", + new=AsyncMock(side_effect=_cancelled_execute), + ), + ): + await proc.process_event(ToolInputStartEvent(id="tc_cancel", tool_name="run_workflow")) + with pytest.raises(asyncio.CancelledError): + await proc.process_event( + ToolCallEvent(tool_call_id="tc_cancel", tool_name="run_workflow", input={"workflow": "wf.json"}) + ) + + baseline_calls = len(event_callback.await_args_list) + saved_cb["cb"]({ + "title": "Running workflow: late-update", + "metadata": {"phase": "running", "step_index": 99}, + }) + await asyncio.sleep(0) + assert len(event_callback.await_args_list) == baseline_calls + + @pytest.mark.asyncio + async def test_completed_tool_cancels_pending_running_metadata_tasks(self): + proc = _make_processor(event_callback=AsyncMock()) + created_tasks = [] + + class _FakeTask: + def __init__(self, coro): + self._callbacks = [] + self.cancelled = False + coro.close() + + def add_done_callback(self, callback): + self._callbacks.append(callback) + + def cancel(self): + if self.cancelled: + return + self.cancelled = True + for callback in list(self._callbacks): + callback(self) + + def result(self): + raise asyncio.CancelledError() + + def _fake_create_task(coro): + task = _FakeTask(coro) + created_tasks.append(task) + return task + + async def _successful_execute(*, tool_name, ctx, **kwargs): + ctx.metadata({ + "title": "Running workflow: inflight-update", + "metadata": {"phase": "running", "step_index": 1}, + }) + return ToolResult(success=True, output="ok", title=tool_name, metadata={}) + + with ( + patch("flocks.session.streaming.stream_processor.Message.store_part", new=AsyncMock()), + patch("flocks.session.streaming.stream_processor.Message.update_part", new=AsyncMock()), + patch( + "flocks.session.streaming.stream_processor.ToolRegistry.execute", + new=_successful_execute, + ), + patch( + "flocks.session.streaming.stream_processor.asyncio.create_task", + side_effect=_fake_create_task, + ), + ): + await proc.process_event(ToolInputStartEvent(id="tc_inflight", tool_name="run_workflow")) + await proc.process_event( + ToolCallEvent( + tool_call_id="tc_inflight", + tool_name="run_workflow", + input={"workflow": "wf.json"}, + ) + ) + + assert len(created_tasks) == 2 + assert all(task.cancelled for task in created_tasks) + @pytest.mark.asyncio async def test_tool_call_skips_tool_span_without_langfuse_generation(self): proc = _make_processor() diff --git a/tests/session/test_stream_processor_parallel_tools.py b/tests/session/test_stream_processor_parallel_tools.py new file mode 100644 index 000000000..13485987f --- /dev/null +++ b/tests/session/test_stream_processor_parallel_tools.py @@ -0,0 +1,66 @@ +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from flocks.session.streaming.stream_events import ToolCallEvent +from flocks.session.streaming.stream_processor import StreamProcessor +from flocks.tool.registry import ToolResult + + +@pytest.mark.asyncio +async def test_foreground_subagent_tool_calls_start_in_parallel(): + processor = StreamProcessor( + session_id="ses-parent", + assistant_message=SimpleNamespace(id="msg-assistant"), + agent=SimpleNamespace(name="rex"), + ) + release = asyncio.Event() + both_started = asyncio.Event() + started: list[str] = [] + + async def fake_execute(tool_name, ctx, **_kwargs): + started.append(ctx.call_id) + if len(started) == 2: + both_started.set() + await release.wait() + return ToolResult(success=True, output=f"{tool_name} done") + + with ( + patch("flocks.session.streaming.stream_processor.Message.store_part", AsyncMock()), + patch("flocks.session.streaming.stream_processor.Message.parts", AsyncMock(return_value=[])), + patch("flocks.session.streaming.stream_processor.ToolRegistry.execute", fake_execute), + patch.object( + processor, + "_resolve_sandbox_meta", + AsyncMock(return_value={"blocked": False, "error": None, "extra": {}}), + ), + ): + await processor.process_event(ToolCallEvent( + tool_call_id="call-1", + tool_name="delegate_task", + input={ + "description": "inspect one", + "prompt": "Inspect one", + "subagent_type": "explore", + }, + )) + await processor.process_event(ToolCallEvent( + tool_call_id="call-2", + tool_name="delegate_task", + input={ + "description": "inspect two", + "prompt": "Inspect two", + "subagent_type": "explore", + }, + )) + + await asyncio.wait_for(both_started.wait(), timeout=0.5) + assert set(started) == {"call-1", "call-2"} + + release.set() + await processor.drain_parallel_tool_calls() + + assert processor.tool_calls["call-1"].status == "completed" + assert processor.tool_calls["call-2"].status == "completed" diff --git a/tests/skill/test_installer.py b/tests/skill/test_installer.py index 4dbf8eb8b..d8f7a87e5 100644 --- a/tests/skill/test_installer.py +++ b/tests/skill/test_installer.py @@ -93,11 +93,37 @@ def test_github_url_with_subpath(self): assert r["kind"] == "github" assert "owner/repo" in r["value"] + def test_github_blob_url_with_skill_directory_subpath(self): + r = _resolve_source( + "https://github.com/mattpocock/skills/blob/main/skills/engineering/diagnose" + ) + assert r["kind"] == "github" + assert r["value"] == "mattpocock/skills/skills/engineering/diagnose" + + def test_github_blob_url_to_skill_md_uses_parent_directory(self): + r = _resolve_source( + "https://github.com/mattpocock/skills/blob/main/skills/engineering/diagnose/SKILL.md" + ) + assert r["kind"] == "github" + assert r["value"] == "mattpocock/skills/skills/engineering/diagnose" + + def test_github_scheme_blob_path_with_subpath(self): + r = _resolve_source( + "github:mattpocock/skills/blob/main/skills/engineering/diagnose" + ) + assert r["kind"] == "github" + assert r["value"] == "mattpocock/skills/skills/engineering/diagnose" + def test_https_url(self): r = _resolve_source("https://example.com/SKILL.md") assert r["kind"] == "url" assert r["value"] == "https://example.com/SKILL.md" + def test_skills_sh_url(self): + r = _resolve_source("https://www.skills.sh/owner/repo/demo") + assert r["kind"] == "skills_sh" + assert r["value"] == "owner/repo/demo" + def test_local_absolute(self): r = _resolve_source("/home/user/skills/my-skill") assert r["kind"] == "local" @@ -253,10 +279,44 @@ def test_save_no_name_uses_hint(self, tmp_skills_dir: Path): class TestInstallFromSource: @pytest.mark.asyncio - async def test_safeskill_not_available(self, tmp_skills_dir): - result = await SkillInstaller.install_from_source("safeskill:test") + async def test_skills_sh_cli_staging_imports_agent_skill(self, tmp_skills_dir): + class Proc: + returncode = 0 + + async def communicate(self): + return b"installed", b"" + + async def fake_create_subprocess_exec(*_cmd, **kwargs): + staged_skill = Path(kwargs["cwd"]) / ".agents" / "skills" / "demo" + staged_skill.mkdir(parents=True) + (staged_skill / "SKILL.md").write_text( + "---\nname: demo\ndescription: Demo\n---\n", + encoding="utf-8", + ) + return Proc() + + with ( + patch("flocks.skill.installer.shutil.which", return_value="/usr/bin/npx"), + patch("flocks.skill.installer._user_skills_root", return_value=tmp_skills_dir), + patch( + "flocks.skill.installer.asyncio.create_subprocess_exec", + fake_create_subprocess_exec, + ), + ): + result = await SkillInstaller.install_from_source( + "https://www.skills.sh/owner/repo/demo" + ) + + assert result.success is True + assert result.skill_name == "demo" + assert (tmp_skills_dir / "demo" / "SKILL.md").exists() + + @pytest.mark.asyncio + async def test_safeskill_requires_npx(self, tmp_skills_dir): + with patch("flocks.skill.installer.shutil.which", return_value=None): + result = await SkillInstaller.install_from_source("safeskill:test") assert result.success is False - assert "SafeSkill" in (result.error or "") + assert "npx is required" in (result.error or "") @pytest.mark.asyncio async def test_local_file(self, tmp_path: Path, tmp_skills_dir: Path): @@ -280,18 +340,24 @@ async def test_local_file_not_found(self): async def test_url_success(self, tmp_skills_dir: Path): mock_content = "---\nname: url-skill\ndescription: From URL\n---\n" - mock_resp = AsyncMock() - mock_resp.status_code = 200 - mock_resp.text = mock_content + class Resp: + status_code = 200 + text = mock_content + headers = {} + + class Client: + async def __aenter__(self): + return self - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) - mock_client.get = AsyncMock(return_value=mock_resp) + async def __aexit__(self, *_args): + return None + + async def get(self, _url: str): + return Resp() with ( patch("flocks.skill.installer._user_skills_root", return_value=tmp_skills_dir), - patch("httpx.AsyncClient", return_value=mock_client), + patch("httpx.AsyncClient", return_value=Client()), ): result = await SkillInstaller.install_from_source( "https://example.com/SKILL.md" @@ -302,23 +368,73 @@ async def test_url_success(self, tmp_skills_dir: Path): @pytest.mark.asyncio async def test_url_http_error(self, tmp_skills_dir: Path): - mock_resp = AsyncMock() - mock_resp.status_code = 404 + class Resp: + status_code = 404 + + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) - mock_client.get = AsyncMock(return_value=mock_resp) + async def get(self, _url: str): + return Resp() with ( patch("flocks.skill.installer._user_skills_root", return_value=tmp_skills_dir), - patch("httpx.AsyncClient", return_value=mock_client), + patch("httpx.AsyncClient", return_value=Client()), ): result = await SkillInstaller.install_from_source("https://example.com/SKILL.md") assert result.success is False assert "404" in (result.error or "") + @pytest.mark.asyncio + async def test_github_api_403_falls_back_to_raw_skill_md(self, tmp_skills_dir: Path): + skill_content = ( + "---\n" + "name: web-design-guidelines\n" + "description: Web design review\n" + "---\n" + "# Web Interface Guidelines\n" + ) + + class Resp: + def __init__(self, status_code: int, text: str = ""): + self.status_code = status_code + self.text = text + + def json(self): + return [] + + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def get(self, url: str): + if url.endswith("/main/skills/web-design-guidelines/SKILL.md"): + return Resp(200, skill_content) + if "api.github.com" in url: + return Resp(403, "rate limited") + return Resp(404, "not found") + + with ( + patch("flocks.skill.installer._user_skills_root", return_value=tmp_skills_dir), + patch("httpx.AsyncClient", return_value=Client()), + ): + result = await SkillInstaller.install_from_source( + "github:vercel-labs/agent-skills/web-design-guidelines" + ) + + assert result.success is True + assert result.skill_name == "web-design-guidelines" + assert "raw GitHub SKILL.md fallback" in result.message + assert (tmp_skills_dir / "web-design-guidelines" / "SKILL.md").exists() + # --------------------------------------------------------------------------- # SkillInstaller._build_install_command diff --git a/tests/task/test_task.py b/tests/task/test_task.py index 341be42d4..fb4cbbfee 100644 --- a/tests/task/test_task.py +++ b/tests/task/test_task.py @@ -14,6 +14,7 @@ import flocks.task.plugin_sync as plugin_sync_module from flocks.server.routes import question as question_routes from flocks.config.config import Config +from flocks.session.message import ToolPart, ToolStateCompleted from flocks.storage.storage import Storage from flocks.task.background import BackgroundManager, BackgroundTask, LaunchInput from flocks.task.executor import TaskExecutor @@ -552,6 +553,108 @@ async def test_background_task_uses_unpinned_model_as_runtime_override( assert task.status == "completed" +@pytest.mark.asyncio +async def test_background_task_completion_injects_parent_context( + monkeypatch: pytest.MonkeyPatch, +): + create_message = AsyncMock() + update_part = AsyncMock() + parent_loop_run = AsyncMock(return_value=SimpleNamespace(action="stop")) + parent_part = ToolPart( + id="part-parent-tool", + sessionID="ses-parent", + messageID="msg-parent", + type="tool", + callID="call-parent", + tool="delegate_task", + state=ToolStateCompleted( + status="completed", + input={ + "description": "inspect cache", + "prompt": "Inspect cache", + "subagent_type": "explore", + "run_in_background": True, + }, + output="Background task launched", + title="inspect cache", + metadata={ + "sessionId": "ses-child", + "taskId": "bg_parent_inject", + "status": "running", + "background": True, + }, + time={"start": 1, "end": 2}, + ), + ) + monkeypatch.setattr(background_module.Message, "create", create_message) + monkeypatch.setattr(background_module.Message, "parts", AsyncMock(return_value=[parent_part])) + monkeypatch.setattr(background_module.Message, "update_part", update_part) + monkeypatch.setattr(background_module.SessionLoop, "run", parent_loop_run) + + manager = BackgroundManager() + task = BackgroundTask( + id="bg_parent_inject", + status="completed", + description="inspect cache", + prompt="", + agent="explore", + parent_session_id="ses-parent", + parent_message_id="msg-parent", + parent_call_id="call-parent", + parent_agent="rex", + parent_model={"providerID": "anthropic", "modelID": "claude-sonnet"}, + session_id="ses-child", + output="cache keys are built in three files", + ) + + await manager._inject_parent_completion(task) + + create_message.assert_awaited_once() + kwargs = create_message.await_args.kwargs + assert kwargs["session_id"] == "ses-parent" + assert kwargs["synthetic"] is True + assert kwargs["part_metadata"]["kind"] == "background_task_result" + assert '' in kwargs["content"] + assert "cache keys are built in three files" in kwargs["content"] + update_part.assert_awaited_once() + update_kwargs = update_part.await_args.kwargs + assert update_kwargs["part_id"] == "part-parent-tool" + assert update_kwargs["state"].status == "completed" + assert update_kwargs["state"].metadata["status"] == "completed" + assert update_kwargs["state"].output == "cache keys are built in three files" + await asyncio.sleep(0) + parent_loop_run.assert_awaited_once() + assert parent_loop_run.await_args.kwargs["session_id"] == "ses-parent" + assert parent_loop_run.await_args.kwargs["agent_name"] == "rex" + + +@pytest.mark.asyncio +async def test_background_task_completion_does_not_resume_running_parent( + monkeypatch: pytest.MonkeyPatch, +): + parent_loop_run = AsyncMock(return_value=SimpleNamespace(action="stop")) + monkeypatch.setattr(background_module.SessionLoop, "run", parent_loop_run) + monkeypatch.setattr(background_module.SessionLoop, "is_running", lambda _session_id: True) + + manager = BackgroundManager() + task = BackgroundTask( + id="bg_parent_running", + status="completed", + description="inspect cache", + prompt="", + agent="explore", + parent_session_id="ses-parent", + parent_agent="rex", + session_id="ses-child", + output="done", + ) + + manager._schedule_parent_resume(task) + await asyncio.sleep(0) + + parent_loop_run.assert_not_awaited() + + @pytest.mark.asyncio async def test_trigger_workflow_resolves_workflow_id_from_filesystem( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/tool/test_agent_toolset.py b/tests/tool/test_agent_toolset.py index c1c18312a..4e0cb7a47 100644 --- a/tests/tool/test_agent_toolset.py +++ b/tests/tool/test_agent_toolset.py @@ -92,8 +92,6 @@ def test_resolve_agent_initial_tools_keeps_empty_non_rex_tools_empty() -> None: def test_builtin_agent_yaml_tool_names_match_current_registry_surface() -> None: available_tool_names = [ "apply_patch", - "background_cancel", - "background_output", "bash", "channel_message", "delegate_task", @@ -102,16 +100,14 @@ def test_builtin_agent_yaml_tool_names_match_current_registry_surface() -> None: "grep", "lsp", "memory_search", - "plan_exit", "question", "read", "run_workflow", "run_workflow_node", - "session_list", + "session_manage", "skill_load", "task", - "todoread", - "todowrite", + "todo", "tool_search", "webfetch", "websearch", diff --git a/tests/tool/test_builtin_management_tools.py b/tests/tool/test_builtin_management_tools.py index eb0a2406a..85c9fc240 100644 --- a/tests/tool/test_builtin_management_tools.py +++ b/tests/tool/test_builtin_management_tools.py @@ -28,3 +28,12 @@ def test_lsp_remains_non_native_by_default() -> None: assert tool is not None assert tool.info.native is False + + +def test_model_config_tools_remain_non_native_by_default() -> None: + ToolRegistry.init() + + for name in ("list_providers", "add_provider", "add_model"): + tool = ToolRegistry.get(name) + assert tool is not None + assert tool.info.native is False diff --git a/tests/tool/test_credential_context_config_override.py b/tests/tool/test_credential_context_config_override.py new file mode 100644 index 000000000..064c099b2 --- /dev/null +++ b/tests/tool/test_credential_context_config_override.py @@ -0,0 +1,138 @@ +"""Unit tests for get_config_override dual-key matching. + +Regression suite for the bug where handlers whose ``SERVICE_ID`` constant uses +the full versioned storage_key (e.g. ``"sangfor_af_v8_0_48"``) instead of the +bare service_id (e.g. ``"sangfor_af"``) would always receive ``None`` from +``get_config_override``, causing a silent fallback to the global default config +(wrong IP / wrong credentials). + +The tests set ContextVars directly — no DB, no ToolRegistry, no network. +""" +from __future__ import annotations + +import pytest + +from flocks.tool.credential_context import ( + _config_override, + _config_override_service, + _config_override_storage_key, + get_config_override, +) + +_SAMPLE_CONFIG = {"base_url": "https://10.201.255.17", "enabled": True} + + +@pytest.fixture(autouse=True) +def _reset_context_vars(): + """Reset all three ContextVars before AND after every test. + + ContextVars persist for the lifetime of an execution context (the test + thread), so without explicit cleanup a test that sets a var can leak state + into the next test — even if the next test calls _set_context(), it might + leave the var in an unexpected state if it only sets some of the vars. + """ + _config_override.set(None) + _config_override_service.set(None) + _config_override_storage_key.set(None) + yield + _config_override.set(None) + _config_override_service.set(None) + _config_override_storage_key.set(None) + + +def _set_context( + *, + config: dict, + service_id: str | None, + storage_key: str | None, +): + """Set the three ContextVars that activate_device_credentials populates.""" + _config_override.set(config) + _config_override_service.set(service_id) + _config_override_storage_key.set(storage_key) + + +# --------------------------------------------------------------------------- +# Core matching scenarios +# --------------------------------------------------------------------------- + +def test_matches_bare_service_id(): + """Handler uses bare service_id — original behaviour must still work.""" + _set_context( + config=_SAMPLE_CONFIG, + service_id="sangfor_af", + storage_key="sangfor_af_v8_0_48", + ) + result = get_config_override("sangfor_af") + assert result is _SAMPLE_CONFIG + + +def test_matches_versioned_storage_key(): + """Handler uses versioned SERVICE_ID — the new behaviour introduced by this fix.""" + _set_context( + config=_SAMPLE_CONFIG, + service_id="sangfor_af", + storage_key="sangfor_af_v8_0_48", + ) + result = get_config_override("sangfor_af_v8_0_48") + assert result is _SAMPLE_CONFIG + + +def test_no_match_returns_none(): + """A completely unrelated service_id must never receive another device's config.""" + _set_context( + config=_SAMPLE_CONFIG, + service_id="sangfor_af", + storage_key="sangfor_af_v8_0_48", + ) + assert get_config_override("tdp_api") is None + assert get_config_override("sangfor_af_v8_0_85") is None # different version + + +def test_no_override_active_returns_none(): + """When no device credential context is active, always return None.""" + _config_override.set(None) + _config_override_service.set(None) + _config_override_storage_key.set(None) + + assert get_config_override("sangfor_af") is None + assert get_config_override("sangfor_af_v8_0_48") is None + + +# --------------------------------------------------------------------------- +# Guard: storage_key=None should not accidentally match an empty string lookup +# --------------------------------------------------------------------------- + +def test_none_storage_key_does_not_match_empty_string(): + """storage_key=None must not match service_id='' (falsy equality trap).""" + _set_context( + config=_SAMPLE_CONFIG, + service_id="sangfor_af", + storage_key=None, + ) + assert get_config_override("") is None + + +def test_none_service_id_does_not_match_empty_string(): + """service_id=None must not match service_id='' (falsy equality trap).""" + _set_context( + config=_SAMPLE_CONFIG, + service_id=None, + storage_key="sangfor_af_v8_0_48", + ) + assert get_config_override("") is None + + +# --------------------------------------------------------------------------- +# Identical service_id and storage_key (edge case) +# --------------------------------------------------------------------------- + +def test_identical_service_and_storage_key(): + """When both keys are the same (no version suffix), matching still works.""" + _set_context( + config=_SAMPLE_CONFIG, + service_id="tdp_api", + storage_key="tdp_api", + ) + assert get_config_override("tdp_api") is _SAMPLE_CONFIG + assert get_config_override("other") is None diff --git a/tests/tool/test_delegate_task_batch_compat.py b/tests/tool/test_delegate_task_batch_compat.py deleted file mode 100644 index 74da3f2f5..000000000 --- a/tests/tool/test_delegate_task_batch_compat.py +++ /dev/null @@ -1,119 +0,0 @@ -from types import SimpleNamespace -from unittest.mock import AsyncMock, patch - -import pytest - -from flocks.tool.registry import ToolContext, ToolRegistry - - -def _make_ctx() -> ToolContext: - return ToolContext(session_id="test-session", message_id="test-message", agent="rex") - - -class TestDelegateTaskTolerance: - def test_delegate_task_schema_allows_omitting_optional_fields(self): - schema = ToolRegistry.get_schema("delegate_task") - assert schema is not None - assert "prompt" in schema.required - assert "load_skills" not in schema.required - assert "description" not in schema.required - - @pytest.mark.asyncio - async def test_delegate_task_derives_description_and_ignores_blank_skills(self): - manager = SimpleNamespace( - launch=AsyncMock(return_value=SimpleNamespace( - id="task-1", - description="Investigate threatbook.cn assets", - agent="asset-survey", - status="running", - session_id="ses-child", - )) - ) - with patch("flocks.tool.agent.delegate_task._find_completed_delegate", AsyncMock(return_value=None)), patch("flocks.tool.agent.delegate_task.Config.get", AsyncMock(return_value=SimpleNamespace(categories=None))), patch("flocks.tool.agent.delegate_task.is_delegatable", return_value=True), patch("flocks.tool.agent.delegate_task.get_background_manager", return_value=manager), patch("flocks.tool.agent.delegate_task.Skill.get", AsyncMock()) as skill_get: - result = await ToolRegistry.execute( - "delegate_task", - ctx=_make_ctx(), - subagent_type="asset-survey", - prompt="Investigate threatbook.cn assets", - run_in_background=True, - load_skills=["", " "], - ) - - assert result.success is True - assert result.title == "Investigate threatbook.cn assets" - assert result.metadata["sessionId"] == "ses-child" - skill_get.assert_not_awaited() - manager.launch.assert_awaited_once() - launch_input = manager.launch.await_args.args[0] - assert launch_input.description == "Investigate threatbook.cn assets" - - @pytest.mark.asyncio - async def test_delegate_task_category_model_uses_runtime_override_without_pinning(self): - manager = SimpleNamespace( - launch=AsyncMock(return_value=SimpleNamespace( - id="task-2", - description="quick task", - agent="rex-junior", - status="running", - session_id="ses-quick", - )) - ) - cfg = SimpleNamespace(categories={ - "quick": { - "model": "anthropic/claude-haiku-4-5", - "prompt_append": None, - } - }) - - with patch("flocks.tool.agent.delegate_task._find_completed_delegate", AsyncMock(return_value=None)), \ - patch("flocks.tool.agent.delegate_task.Config.get", AsyncMock(return_value=cfg)), \ - patch("flocks.tool.agent.delegate_task._validate_category_model", return_value={ - "providerID": "anthropic", - "modelID": "claude-haiku-4-5", - }), \ - patch("flocks.tool.agent.delegate_task.get_background_manager", return_value=manager): - result = await ToolRegistry.execute( - "delegate_task", - ctx=_make_ctx(), - category="quick", - prompt="Summarize the diff", - description="quick task", - run_in_background=True, - ) - - assert result.success is True - manager.launch.assert_awaited_once() - launch_input = manager.launch.await_args.args[0] - assert launch_input.model == { - "providerID": "anthropic", - "modelID": "claude-haiku-4-5", - } - assert launch_input.model_pinned is False - - @pytest.mark.asyncio - async def test_delegate_task_sync_continue_fails_when_last_message_missing(self): - session = SimpleNamespace( - id="ses-child", - agent="asset-survey", - ) - - with patch("flocks.tool.agent.delegate_task.Config.get", AsyncMock(return_value=SimpleNamespace(categories=None))), \ - patch("flocks.tool.agent.delegate_task.Session.get_by_id", AsyncMock(return_value=session)), \ - patch("flocks.tool.agent.delegate_task.Message.create", AsyncMock()), \ - patch("flocks.tool.agent.delegate_task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( - action="stop", - error=None, - last_message=None, - ))): - result = await ToolRegistry.execute( - "delegate_task", - ctx=_make_ctx(), - session_id="ses-child", - prompt="Continue investigating", - ) - - assert result.success is True - assert result.metadata["sessionId"] == "ses-child" - assert result.metadata["emptyOutput"] is True - assert "without producing a final assistant message" in (result.output or "") - diff --git a/tests/tool/test_delegate_task_compat.py b/tests/tool/test_delegate_task_compat.py new file mode 100644 index 000000000..f97355f1c --- /dev/null +++ b/tests/tool/test_delegate_task_compat.py @@ -0,0 +1,177 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from flocks.tool.registry import ToolContext, ToolRegistry + + +def _make_ctx() -> ToolContext: + return ToolContext(session_id="test-session", message_id="test-message", agent="rex") + + +class TestDelegateTaskTolerance: + def test_delegate_task_schema_allows_omitting_optional_fields(self): + schema = ToolRegistry.get_schema("delegate_task") + assert schema is not None + assert "prompt" in schema.required + assert "load_skills" not in schema.required + assert "description" not in schema.required + assert "run_in_background" not in schema.properties + # Legacy batch shape is gone: tasks=[...] is no longer a public option. + assert "tasks" not in schema.properties + + @pytest.mark.asyncio + async def test_delegate_task_derives_description_and_ignores_blank_skills(self): + parent_session = SimpleNamespace( + id="test-session", + project_id="proj", + directory="/tmp/project", + provider=None, + model=None, + ) + child_session = SimpleNamespace(id="ses-child") + with ( + patch("flocks.tool.agent.delegate_task._find_completed_delegate", AsyncMock(return_value=None)), + patch("flocks.tool.agent.delegate_task.Config.get", AsyncMock(return_value=SimpleNamespace(categories=None))), + patch("flocks.tool.agent.delegate_task.is_delegatable", return_value=True), + patch("flocks.tool.agent.delegate_task.Skill.get", AsyncMock()) as skill_get, + patch("flocks.tool.agent.delegate_task.Session.get_by_id", AsyncMock(return_value=parent_session)), + patch("flocks.tool.agent.delegate_task.Session.create", AsyncMock(return_value=child_session)), + patch("flocks.tool.agent.delegate_task.Message.create", AsyncMock()), + patch("flocks.tool.agent.delegate_task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( + action="stop", + error=None, + last_message=None, + ))), + ): + result = await ToolRegistry.execute( + "delegate_task", + ctx=_make_ctx(), + subagent_type="asset-survey", + prompt="Investigate threatbook.cn assets", + load_skills=["", " "], + ) + + assert result.success is True + assert result.title == "Investigate threatbook.cn assets" + assert result.metadata["sessionId"] == "ses-child" + skill_get.assert_not_awaited() + + @pytest.mark.asyncio + async def test_delegate_task_category_model_uses_runtime_override_without_pinning(self): + parent_session = SimpleNamespace( + id="test-session", + project_id="proj", + directory="/tmp/project", + provider=None, + model=None, + ) + child_session = SimpleNamespace(id="ses-quick") + cfg = SimpleNamespace(categories={ + "quick": { + "model": "anthropic/claude-haiku-4-5", + "prompt_append": None, + } + }) + + with patch("flocks.tool.agent.delegate_task._find_completed_delegate", AsyncMock(return_value=None)), \ + patch("flocks.tool.agent.delegate_task.Config.get", AsyncMock(return_value=cfg)), \ + patch("flocks.tool.agent.delegate_task._validate_category_model", return_value={ + "providerID": "anthropic", + "modelID": "claude-haiku-4-5", + }), \ + patch("flocks.tool.agent.delegate_task.Session.get_by_id", AsyncMock(return_value=parent_session)), \ + patch("flocks.tool.agent.delegate_task.Session.create", AsyncMock(return_value=child_session)) as create_session, \ + patch("flocks.tool.agent.delegate_task.Message.create", AsyncMock()), \ + patch("flocks.tool.agent.delegate_task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( + action="stop", + error=None, + last_message=None, + ))) as loop_run: + result = await ToolRegistry.execute( + "delegate_task", + ctx=_make_ctx(), + category="quick", + prompt="Summarize the diff", + description="quick task", + ) + + assert result.success is True + assert create_session.await_args.kwargs["provider"] == "anthropic" + assert create_session.await_args.kwargs["model"] == "claude-haiku-4-5" + assert create_session.await_args.kwargs["model_pinned"] is False + assert loop_run.await_args.kwargs["provider_id"] == "anthropic" + assert loop_run.await_args.kwargs["model_id"] == "claude-haiku-4-5" + + @pytest.mark.asyncio + async def test_delegate_task_rejects_background_execution(self): + # run_in_background is not in the public schema, so the registry + # rejects it with an "unknown parameters" error before the function + # body runs. This guards the schema-level ban. + result = await ToolRegistry.execute( + "delegate_task", + ctx=_make_ctx(), + subagent_type="asset-survey", + prompt="Investigate threatbook.cn assets", + run_in_background=True, + ) + + assert result.success is False + assert "unknown parameters: run_in_background" in (result.error or "") + + @pytest.mark.asyncio + async def test_delegate_task_function_body_guard_rejects_background(self): + # Direct (in-process) callers that bypass the registry still hit the + # function-body guard. This is the second line of defense. + from flocks.tool.agent.delegate_task import delegate_task_tool + + result = await delegate_task_tool( + _make_ctx(), + subagent_type="asset-survey", + prompt="Investigate threatbook.cn assets", + run_in_background=True, + ) + + assert result.success is False + assert "Background subagent execution is disabled" in (result.error or "") + + @pytest.mark.asyncio + async def test_delegate_task_rejects_legacy_batch_tasks_param(self): + # tasks=[...] has been removed from the schema; passing it now + # surfaces a schema-level "unknown parameters" error. + result = await ToolRegistry.execute( + "delegate_task", + ctx=_make_ctx(), + tasks=[{"prompt": "x", "subagent_type": "explore"}], + ) + + assert result.success is False + assert "unknown parameters: tasks" in (result.error or "") + + @pytest.mark.asyncio + async def test_delegate_task_sync_continue_fails_when_last_message_missing(self): + session = SimpleNamespace( + id="ses-child", + agent="asset-survey", + ) + + with patch("flocks.tool.agent.delegate_task.Config.get", AsyncMock(return_value=SimpleNamespace(categories=None))), \ + patch("flocks.tool.agent.delegate_task.Session.get_by_id", AsyncMock(return_value=session)), \ + patch("flocks.tool.agent.delegate_task.Message.create", AsyncMock()), \ + patch("flocks.tool.agent.delegate_task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( + action="stop", + error=None, + last_message=None, + ))): + result = await ToolRegistry.execute( + "delegate_task", + ctx=_make_ctx(), + session_id="ses-child", + prompt="Continue investigating", + ) + + assert result.success is True + assert result.metadata["sessionId"] == "ses-child" + assert result.metadata["emptyOutput"] is True + assert "without producing a final assistant message" in (result.output or "") diff --git a/tests/tool/test_device_plugin_index.py b/tests/tool/test_device_plugin_index.py new file mode 100644 index 000000000..05f68632d --- /dev/null +++ b/tests/tool/test_device_plugin_index.py @@ -0,0 +1,431 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import yaml +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from flocks.tool.device.models import CustomDeviceTemplateCreate + + +def _write_provider(root: Path, data: dict) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "_provider.yaml").write_text( + yaml.safe_dump(data, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + + +def _write_tool(root: Path, name: str = "device_ping") -> None: + (root / f"{name}.yaml").write_text( + yaml.safe_dump( + { + "name": name, + "description": "Ping device", + "provider": "demo_api", + "handler": {"type": "http", "path": "/ping"}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + +def _reset_env(monkeypatch, tmp_path): + from flocks.config.config import Config + from flocks.config import api_versioning + from flocks.storage.storage import Storage + + home = tmp_path / "home" + data = tmp_path / "data" + project = tmp_path / "project" + home.mkdir() + data.mkdir() + project.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("FLOCKS_DATA_DIR", str(data)) + monkeypatch.chdir(project) + Config._global_config = None + Config._cached_config = None + Storage._db_path = None + Storage._initialized = False + api_versioning._reset_descriptor_cache() + return home, data, project + + +def test_device_plugin_index_filters_and_shapes_templates(monkeypatch, tmp_path): + from flocks.tool.device import plugin_index + + _reset_env(monkeypatch, tmp_path) + root = tmp_path / "bundled" / "demo_device" + _write_provider( + root, + { + "name": "Demo Device", + "service_id": "demo_api", + "version": "1.2.3", + "integration_type": "device", + "vendor": "demo", + "credential_fields": [ + {"key": "base_url", "label": "Base URL", "storage": "config"}, + ], + }, + ) + _write_tool(root) + + api_root = tmp_path / "bundled" / "not_device" + _write_provider( + api_root, + { + "name": "API Only", + "service_id": "api_only", + "integration_type": "api", + }, + ) + + entries = [ + SimpleNamespace( + id="demo_device", + name="Demo Device", + version="1.2.3", + installedVersion=None, + description="Demo", + descriptionCn="演示", + state="available", + source="bundled", + installPath=None, + ), + SimpleNamespace( + id="not_device", + name="API Only", + version="1.0.0", + installedVersion=None, + description="Nope", + descriptionCn=None, + state="available", + source="bundled", + installPath=None, + ), + ] + monkeypatch.setattr(plugin_index.hub_catalog, "list_catalog", lambda plugin_type=None: entries) + monkeypatch.setattr( + plugin_index.hub_catalog, + "system_plugin_root", + lambda plugin_type, plugin_id: root if plugin_id == "demo_device" else api_root, + ) + monkeypatch.setattr(plugin_index.ToolRegistry, "init", classmethod(lambda cls: None)) + monkeypatch.setattr(plugin_index.ToolRegistry, "list_tools", classmethod(lambda cls: [])) + + templates = plugin_index.list_device_templates(refresh=True) + + assert [template.plugin_id for template in templates] == ["demo_device"] + template = templates[0] + assert template.storage_key == "demo_api_v1_2_3" + assert template.service_id == "demo_api" + assert template.vendor == "demo" + assert template.installed is False + assert template.state == "available" + assert template.source == "bundled" + assert template.tool_count == 1 + assert template.credential_schema[0]["key"] == "base_url" + + +def test_device_plugin_index_normalizes_plugin_id_name(monkeypatch, tmp_path): + from flocks.tool.device import plugin_index + + _reset_env(monkeypatch, tmp_path) + root = tmp_path / "bundled" / "onesig_v2_5_3_D20250710" + _write_provider( + root, + { + "name": "onesig_v2_5_3_D20250710", + "service_id": "onesig_v2_5_3_D20250710_api", + "version": "2.5.3 D20250710", + "integration_type": "device", + "vendor": "threatbook", + "credential_fields": [], + }, + ) + _write_tool(root, "onesig_v2_5_3_D20250710_login") + + entries = [ + SimpleNamespace( + id="onesig_v2_5_3_D20250710", + name="onesig_v2_5_3_D20250710", + version="2.5.3 D20250710", + installedVersion=None, + description="OneSIG legacy", + descriptionCn="OneSIG 老版本", + state="available", + source="bundled", + installPath=None, + ), + ] + monkeypatch.setattr(plugin_index.hub_catalog, "list_catalog", lambda plugin_type=None: entries) + monkeypatch.setattr(plugin_index.hub_catalog, "system_plugin_root", lambda plugin_type, plugin_id: root) + monkeypatch.setattr(plugin_index.ToolRegistry, "init", classmethod(lambda cls: None)) + monkeypatch.setattr(plugin_index.ToolRegistry, "list_tools", classmethod(lambda cls: [])) + + templates = plugin_index.list_device_templates(refresh=True) + + assert len(templates) == 1 + assert templates[0].name == "onesig" + assert templates[0].storage_key == "onesig_v2_5_3_D20250710_api_v2_5_3_D20250710" + assert templates[0].version == "2.5.3 D20250710" + + +def test_device_template_refresh_reloads_plugin_tools(monkeypatch, tmp_path): + from flocks.tool.device import plugin_index + + _reset_env(monkeypatch, tmp_path) + calls: list[str] = [] + + monkeypatch.setattr(plugin_index.hub_catalog, "list_catalog", lambda plugin_type=None: []) + monkeypatch.setattr( + plugin_index.ToolRegistry, + "refresh_plugin_tools", + classmethod(lambda cls: calls.append("refresh") or []), + ) + + assert plugin_index.list_device_templates(refresh=True) == [] + assert calls == ["refresh"] + + +def test_create_custom_device_template_writes_user_plugin(monkeypatch, tmp_path): + from flocks.tool.device import plugin_index + + home, data, _project = _reset_env(monkeypatch, tmp_path) + monkeypatch.setattr(plugin_index.hub_catalog, "list_catalog", lambda plugin_type=None: []) + monkeypatch.setattr(plugin_index.hub_catalog, "system_plugin_root", lambda plugin_type, plugin_id: None) + monkeypatch.setattr(plugin_index.ToolRegistry, "refresh_plugin_tools", classmethod(lambda cls: [])) + monkeypatch.setattr(plugin_index.ToolRegistry, "init", classmethod(lambda cls: None)) + monkeypatch.setattr(plugin_index.ToolRegistry, "list_tools", classmethod(lambda cls: [])) + + template = plugin_index.create_custom_device_template( + CustomDeviceTemplateCreate( + plugin_id="custom_demo", + name="Custom Demo", + vendor="demo", + service_id="custom_demo_api", + version="0.1.0", + description="Custom device", + credential_fields=[ + {"key": "base_url", "label": "Base URL", "storage": "config"}, + ], + tools=[ + { + "name": "custom_demo_ping", + "description": "Ping", + "handler": {"type": "http", "path": "/ping"}, + }, + ], + ) + ) + + plugin_dir = home / ".flocks" / "plugins" / "tools" / "device" / "custom_demo" + provider = yaml.safe_load((plugin_dir / "_provider.yaml").read_text(encoding="utf-8")) + tool = yaml.safe_load((plugin_dir / "custom_demo_ping.yaml").read_text(encoding="utf-8")) + records = json.loads((data / "hub" / "installed.json").read_text(encoding="utf-8")) + + assert template.plugin_id == "custom_demo" + assert template.storage_key == "custom_demo_api_v0_1_0" + assert provider["integration_type"] == "device" + assert provider["service_id"] == "custom_demo_api" + assert tool["provider"] == "custom_demo_api" + assert records["plugins"]["device:custom_demo"]["type"] == "device" + + +def test_device_template_route_is_not_shadowed_by_device_id(monkeypatch): + from flocks.server.routes import device as device_routes + + monkeypatch.setattr( + device_routes, + "list_device_templates", + lambda refresh=False: [ + { + "plugin_id": "demo", + "storage_key": "demo_api_v1", + "service_id": "demo_api", + "name": "Demo", + "version": "1", + "credential_schema": [], + "tool_count": 0, + "installed": True, + "state": "installed", + "source": "project", + } + ], + ) + + app = FastAPI() + app.include_router(device_routes.router, prefix="/api/devices") + client = TestClient(app) + + response = client.get("/api/devices/templates") + + assert response.status_code == 200 + assert response.json()[0]["plugin_id"] == "demo" + + +def test_device_template_route_forwards_refresh(monkeypatch): + from flocks.server.routes import device as device_routes + + calls: list[bool] = [] + + def fake_list_device_templates(*, refresh: bool = False): + calls.append(refresh) + return [ + { + "plugin_id": "demo", + "storage_key": "demo_api_v1", + "service_id": "demo_api", + "name": "Demo", + "version": "1", + "credential_schema": [], + "tool_count": 0, + "installed": True, + "state": "installed", + "source": "project", + } + ] + + monkeypatch.setattr(device_routes, "list_device_templates", fake_list_device_templates) + + app = FastAPI() + app.include_router(device_routes.router, prefix="/api/devices") + client = TestClient(app) + + response = client.get("/api/devices/templates?refresh=true") + + assert response.status_code == 200 + assert calls == [True] + + +@pytest.mark.asyncio +async def test_device_list_auto_creates_user_device_plugin_instance(monkeypatch, tmp_path): + from flocks.server.routes import device as device_routes + from flocks.storage.storage import Storage + from flocks.tool.device.store import list_devices + from flocks.tool.device import plugin_index + + home, _data, _project = _reset_env(monkeypatch, tmp_path) + await Storage.init() + + root = home / ".flocks" / "plugins" / "tools" / "device" / "custom_demo" + _write_provider( + root, + { + "name": "Custom Demo", + "service_id": "custom_demo_api", + "version": "0.1.0", + "integration_type": "device", + "vendor": "custom_vendor", + "credential_fields": [], + }, + ) + _write_tool(root, "custom_demo_ping") + + monkeypatch.setattr(plugin_index.hub_catalog, "list_catalog", lambda plugin_type=None: []) + monkeypatch.setattr(plugin_index.hub_catalog, "system_plugin_root", lambda plugin_type, plugin_id: None) + monkeypatch.setattr(plugin_index.ToolRegistry, "init", classmethod(lambda cls: None)) + monkeypatch.setattr(plugin_index.ToolRegistry, "list_tools", classmethod(lambda cls: [])) + + app = FastAPI() + app.include_router(device_routes.router, prefix="/api/devices") + client = TestClient(app) + + response = client.get("/api/devices?refresh=true") + repeated = client.get("/api/devices?refresh=true") + devices = await list_devices() + + assert response.status_code == 200 + assert repeated.status_code == 200 + assert len(devices) == 1 + assert devices[0].name == "Custom Demo" + assert devices[0].storage_key == "custom_demo_api_v0_1_0" + assert devices[0].service_id == "custom_demo_api" + assert devices[0].enabled is True + + delete_response = client.delete(f"/api/devices/{devices[0].id}") + after_delete = client.get("/api/devices?refresh=true") + devices_after_delete = await list_devices() + + assert delete_response.status_code == 204 + assert after_delete.status_code == 200 + assert after_delete.json() == [] + assert devices_after_delete == [] + + manual_create = client.post( + "/api/devices", + json={ + "name": "Custom Demo Manual", + "storage_key": "custom_demo_api_v0_1_0", + "service_id": "custom_demo_api", + "fields": {}, + }, + ) + after_manual_create = client.get("/api/devices?refresh=true") + + assert manual_create.status_code == 201 + assert len(after_manual_create.json()) == 1 + assert after_manual_create.json()[0]["name"] == "Custom Demo Manual" + + +@pytest.mark.asyncio +async def test_auto_creating_user_device_instances_is_serialized(monkeypatch, tmp_path): + from flocks.storage.storage import Storage + from flocks.tool.device import intake, plugin_index + from flocks.tool.device.store import list_devices + + home, _data, _project = _reset_env(monkeypatch, tmp_path) + await Storage.init() + + root = home / ".flocks" / "plugins" / "tools" / "device" / "custom_demo" + _write_provider( + root, + { + "name": "Custom Demo", + "service_id": "custom_demo_api", + "version": "0.1.0", + "integration_type": "device", + "vendor": "custom_vendor", + "credential_fields": [], + }, + ) + _write_tool(root, "custom_demo_ping") + + async def fake_sync_service_tool_state(service_id: str) -> None: + return None + + original_insert_device = intake.insert_device + insert_attempts = 0 + + async def slow_insert_device(**kwargs): + nonlocal insert_attempts + insert_attempts += 1 + await asyncio.sleep(0) + await original_insert_device(**kwargs) + + monkeypatch.setattr(plugin_index.hub_catalog, "list_catalog", lambda plugin_type=None: []) + monkeypatch.setattr(plugin_index.hub_catalog, "system_plugin_root", lambda plugin_type, plugin_id: None) + monkeypatch.setattr(plugin_index.ToolRegistry, "refresh_plugin_tools", classmethod(lambda cls: [])) + monkeypatch.setattr(plugin_index.ToolRegistry, "init", classmethod(lambda cls: None)) + monkeypatch.setattr(plugin_index.ToolRegistry, "list_tools", classmethod(lambda cls: [])) + monkeypatch.setattr(intake, "insert_device", slow_insert_device) + monkeypatch.setattr(intake, "sync_service_tool_state", fake_sync_service_tool_state) + + created_counts = await asyncio.gather( + intake.ensure_user_device_instances(refresh_templates=True), + intake.ensure_user_device_instances(refresh_templates=True), + ) + devices = await list_devices() + + assert sorted(created_counts) == [0, 1] + assert insert_attempts == 1 + assert len(devices) == 1 + assert devices[0].storage_key == "custom_demo_api_v0_1_0" diff --git a/tests/tool/test_device_secrets.py b/tests/tool/test_device_secrets.py new file mode 100644 index 000000000..770b30501 --- /dev/null +++ b/tests/tool/test_device_secrets.py @@ -0,0 +1,25 @@ +from unittest.mock import MagicMock, patch + +from flocks.tool.device.secrets import persist_fields + + +def test_persist_fields_strips_tdp_config_api_base_url(): + with patch("flocks.security.get_secret_manager", return_value=MagicMock()): + fields = persist_fields( + "device-1", + "tdp_api_v3_3_10", + {"base_url": "https://tdp.local/config/api"}, + ) + + assert fields["base_url"] == "https://tdp.local" + + +def test_persist_fields_keeps_non_tdp_base_url_paths(): + with patch("flocks.security.get_secret_manager", return_value=MagicMock()): + fields = persist_fields( + "device-1", + "proxy_device_v1", + {"base_url": "https://proxy.local/config/api"}, + ) + + assert fields["base_url"] == "https://proxy.local/config/api" diff --git a/tests/tool/test_flocks_skills.py b/tests/tool/test_flocks_skills.py index 6902bdc72..fe6c36cdd 100644 --- a/tests/tool/test_flocks_skills.py +++ b/tests/tool/test_flocks_skills.py @@ -93,13 +93,13 @@ async def test_missing_flocks_executable(): from flocks.tool.skill.flocks_skills import flocks_skills with patch("flocks.tool.skill.flocks_skills._flocks_executable", return_value=None): - result = await flocks_skills(ctx, subcommand="list") + result = await flocks_skills(ctx, subcommand="status") assert result.success is False assert "not found" in (result.error or "").lower() @pytest.mark.asyncio -async def test_list_success(): +async def test_status_success(): from flocks.tool.skill.flocks_skills import flocks_skills ctx = make_ctx() @@ -108,13 +108,13 @@ async def test_list_success(): patch("flocks.tool.skill.flocks_skills._flocks_executable", return_value="/usr/bin/flocks"), patch("flocks.tool.skill.flocks_skills.asyncio.create_subprocess_exec", return_value=proc) as mock_exec, ): - result = await flocks_skills(ctx, subcommand="list") + result = await flocks_skills(ctx, subcommand="status") assert result.success is True assert "find-ioc" in (result.output or "") cmd_args = mock_exec.call_args[0] assert "skills" in cmd_args - assert "list" in cmd_args + assert "status" in cmd_args ctx.ask.assert_not_called() @@ -138,6 +138,24 @@ async def test_find_passes_args(): ctx.ask.assert_not_called() +@pytest.mark.asyncio +async def test_remove_appends_yes_for_non_interactive_tool_calls(): + from flocks.tool.skill.flocks_skills import flocks_skills + + ctx = make_ctx() + proc = make_proc(stdout=b"removed\n", returncode=0) + with ( + patch("flocks.tool.skill.flocks_skills._flocks_executable", return_value="/usr/bin/flocks"), + patch("flocks.tool.skill.flocks_skills.asyncio.create_subprocess_exec", return_value=proc) as mock_exec, + ): + result = await flocks_skills(ctx, subcommand="remove", args="old-skill") + + assert result.success is True + cmd_args = mock_exec.call_args[0] + assert cmd_args == ("/usr/bin/flocks", "skills", "remove", "old-skill", "--yes") + ctx.ask.assert_called_once() + + @pytest.mark.asyncio async def test_nonzero_exit_returns_failure(): from flocks.tool.skill.flocks_skills import flocks_skills @@ -225,7 +243,7 @@ async def test_long_output_is_truncated(): patch("flocks.tool.skill.flocks_skills._flocks_executable", return_value="/usr/bin/flocks"), patch("flocks.tool.skill.flocks_skills.asyncio.create_subprocess_exec", return_value=proc), ): - result = await flocks_skills(ctx, subcommand="list") + result = await flocks_skills(ctx, subcommand="status") assert result.success is True assert len(result.output or "") <= _MAX_OUTPUT + 100 # allow for truncation notice diff --git a/tests/tool/test_omo_parity.py b/tests/tool/test_omo_parity.py index 308c8e922..9aa0b746f 100644 --- a/tests/tool/test_omo_parity.py +++ b/tests/tool/test_omo_parity.py @@ -8,9 +8,9 @@ class TestOmoToolParity: - """Minimal parity checks for background tools.""" + """Minimal parity checks for background tool removal.""" - def test_background_tools_registered(self): + def test_background_tools_not_exposed_to_models(self): tools = ToolRegistry.all_tool_ids() - assert "background_output" in tools - assert "background_cancel" in tools + assert "background_output" not in tools + assert "background_cancel" not in tools diff --git a/tests/tool/test_session_manage_tool.py b/tests/tool/test_session_manage_tool.py new file mode 100644 index 000000000..cae00eb23 --- /dev/null +++ b/tests/tool/test_session_manage_tool.py @@ -0,0 +1,87 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from flocks.tool.registry import ToolContext, ToolRegistry, ToolResult +import flocks.tool.system.session_manage # noqa: F401 - ensure tool registration +from flocks.tool.system.session_manage import session_manage + + +def make_ctx() -> ToolContext: + ctx = MagicMock(spec=ToolContext) + ctx.ask = AsyncMock(return_value=None) + return ctx + + +def test_session_manage_is_single_registered_session_tool(): + names = {tool.name for tool in ToolRegistry.list_tools()} + + assert "session_manage" in names + assert "session_list" not in names + assert "session_get" not in names + assert "session_create" not in names + assert "session_update" not in names + assert "session_delete" not in names + assert "session_archive" not in names + + +def test_session_manage_schema_uses_action_dispatch(): + tool = next(tool for tool in ToolRegistry.list_tools() if tool.name == "session_manage") + schema = tool.get_schema() + + assert schema.properties["action"]["enum"] == [ + "list", + "get", + "create", + "update", + "delete", + "archive", + ] + assert schema.required == ["action"] + assert "session_id" in schema.properties + + +@pytest.mark.asyncio +async def test_session_manage_dispatches_list_action(): + ctx = make_ctx() + expected = ToolResult(success=True, output={"sessions": []}) + + with patch( + "flocks.tool.system.session_manage._session_list_impl", + AsyncMock(return_value=expected), + ) as list_impl: + result = await session_manage(ctx, action="list", status="active", limit=3) + + assert result is expected + list_impl.assert_awaited_once() + _, kwargs = list_impl.await_args + assert kwargs["status"] == "active" + assert kwargs["limit"] == 3 + ctx.ask.assert_not_called() + + +@pytest.mark.asyncio +async def test_session_manage_requires_session_id_for_get(): + result = await session_manage(make_ctx(), action="get") + + assert result.success is False + assert "session_id" in (result.error or "") + + +@pytest.mark.asyncio +async def test_session_manage_delete_requests_confirmation(): + ctx = make_ctx() + expected = ToolResult(success=True, output="deleted") + + with patch( + "flocks.tool.system.session_manage._session_delete_impl", + AsyncMock(return_value=expected), + ) as delete_impl: + result = await session_manage(ctx, action="delete", session_id="ses_123") + + assert result is expected + ctx.ask.assert_awaited_once() + ask_kwargs = ctx.ask.await_args.kwargs + assert ask_kwargs["permission"] == "session_manage" + assert ask_kwargs["metadata"] == {"action": "delete", "session_id": "ses_123"} + delete_impl.assert_awaited_once() diff --git a/tests/tool/test_slash_command_extended.py b/tests/tool/test_slash_command_extended.py index d85d724f2..865172d24 100644 --- a/tests/tool/test_slash_command_extended.py +++ b/tests/tool/test_slash_command_extended.py @@ -415,6 +415,6 @@ async def test_help_shows_full_webui_help(self): assert "Direct commands:" in result.output assert "Other commands (handled through the normal assistant/session flow):" in result.output assert "/clear" in result.output - assert "/plan" in result.output + assert "/bug" in result.output assert "/compact" in result.output assert "/model" not in result.output diff --git a/tests/tool/test_task_model_pinning.py b/tests/tool/test_task_model_pinning.py index a2c37b696..4f75e10ab 100644 --- a/tests/tool/test_task_model_pinning.py +++ b/tests/tool/test_task_model_pinning.py @@ -1,231 +1,61 @@ -from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest -from flocks.tool.registry import ToolContext -from flocks.tool.agent.task import _resolve_child_model, task_tool +from flocks.tool.agent.task import task_tool +from flocks.tool.registry import ToolContext, ToolRegistry, ToolResult def _make_ctx() -> ToolContext: return ToolContext(session_id="test-session", message_id="test-message", agent="rex") -class TestTaskModelPinning: - @pytest.mark.asyncio - async def test_resolve_child_model_ignores_unpinned_parent_session_model(self): - parent_session = SimpleNamespace( - provider="anthropic", - model="stale-model", - model_pinned=False, - ) - - with patch("flocks.storage.storage.Storage.read", AsyncMock(return_value={})), \ - patch("flocks.agent.registry.Agent.get", AsyncMock(return_value=None)), \ - patch("flocks.config.config.Config.resolve_default_llm", AsyncMock(return_value={ - "provider_id": "openai", - "model_id": "gpt-5", - })): - provider, model, source = await _resolve_child_model("explore", parent_session) - - assert provider == "openai" - assert model == "gpt-5" - assert source == "config" +class TestTaskCompatibilityAlias: + def test_task_schema_does_not_expose_background_execution(self): + schema = ToolRegistry.get_schema("task") + assert schema is not None + assert "run_in_background" not in schema.properties + # Legacy batch shape is gone. + assert "tasks" not in schema.properties @pytest.mark.asyncio - async def test_task_tool_explicit_model_override_pins_child_session(self): - manager = SimpleNamespace( - launch=AsyncMock(return_value=SimpleNamespace( - id="bg-task", - description="delegate explore", - agent="explore", - status="running", - session_id="ses-child", - )) - ) - parent_session = SimpleNamespace( - id="ses-parent", - project_id="proj", - directory="/tmp/project", - provider=None, - model=None, - model_pinned=False, + async def test_task_tool_rejects_background_execution_when_called_directly(self): + result = await task_tool( + _make_ctx(), + description="delegate explore", + prompt="Inspect the repository", + subagent_type="explore", + run_in_background=True, ) - with patch("flocks.tool.agent.task.is_delegatable", return_value=True), \ - patch("flocks.tool.agent.task.Session.get_by_id", AsyncMock(return_value=parent_session)), \ - patch("flocks.tool.agent.task.get_background_manager", return_value=manager): - result = await task_tool( - _make_ctx(), - description="delegate explore", - prompt="Inspect the repository", - subagent_type="explore", - run_in_background=True, - model="openai/gpt-5", - ) - - assert result.success is True - manager.launch.assert_awaited_once() - launch_input = manager.launch.await_args.args[0] - assert launch_input.model == { - "providerID": "openai", - "modelID": "gpt-5", - } - assert launch_input.model_pinned is True + assert result.success is False + assert "Background subagent execution is disabled" in (result.error or "") @pytest.mark.asyncio - async def test_task_tool_default_resolution_does_not_pin_child_session(self): - manager = SimpleNamespace( - launch=AsyncMock(return_value=SimpleNamespace( - id="bg-task", - description="delegate explore", - agent="explore", - status="running", - session_id="ses-child", - )) - ) - parent_session = SimpleNamespace( - id="ses-parent", - project_id="proj", - directory="/tmp/project", - provider="anthropic", - model="stale-model", - model_pinned=False, + async def test_task_tool_forwards_single_call_to_delegate_task(self): + delegate_result = ToolResult( + success=True, + output="ok", + metadata={"sessionId": "ses-child"}, ) - with patch("flocks.tool.agent.task.is_delegatable", return_value=True), \ - patch("flocks.tool.agent.task.Session.get_by_id", AsyncMock(return_value=parent_session)), \ - patch("flocks.tool.agent.task.get_background_manager", return_value=manager), \ - patch("flocks.storage.storage.Storage.read", AsyncMock(return_value={})), \ - patch("flocks.agent.registry.Agent.get", AsyncMock(return_value=None)), \ - patch("flocks.config.config.Config.resolve_default_llm", AsyncMock(return_value={ - "provider_id": "anthropic", - "model_id": "claude-sonnet-4-6", - })): + with patch( + "flocks.tool.agent.task.delegate_task_tool", + AsyncMock(return_value=delegate_result), + ) as delegate: result = await task_tool( _make_ctx(), description="delegate explore", prompt="Inspect the repository", subagent_type="explore", - run_in_background=True, - ) - - assert result.success is True - manager.launch.assert_awaited_once() - launch_input = manager.launch.await_args.args[0] - assert launch_input.model is None - assert launch_input.model_pinned is False - - @pytest.mark.asyncio - async def test_task_tool_sync_continue_returns_session_loop_error(self): - parent_session = SimpleNamespace( - id="ses-parent", - project_id="proj", - directory="/tmp/project", - provider=None, - model=None, - model_pinned=False, - ) - child_session = SimpleNamespace( - id="ses-child", - agent="explore", - ) - - with patch("flocks.tool.agent.task.is_delegatable", return_value=True), \ - patch("flocks.tool.agent.task.Session.get_by_id", AsyncMock(side_effect=[parent_session, child_session])), \ - patch("flocks.tool.agent.task.Message.create", AsyncMock()), \ - patch("flocks.tool.agent.task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( - action="error", - error="subagent crashed", - last_message=None, - ))): - result = await task_tool( - _make_ctx(), - description="continue explore", - prompt="Continue the task", - subagent_type="explore", - session_id="ses-child", - ) - - assert result.success is False - assert result.metadata["sessionId"] == "ses-child" - assert "subagent crashed" in (result.error or "") - - @pytest.mark.asyncio - async def test_task_tool_sync_continue_fails_when_last_message_missing(self): - parent_session = SimpleNamespace( - id="ses-parent", - project_id="proj", - directory="/tmp/project", - provider=None, - model=None, - model_pinned=False, - ) - child_session = SimpleNamespace( - id="ses-child", - agent="explore", - ) - - with patch("flocks.tool.agent.task.is_delegatable", return_value=True), \ - patch("flocks.tool.agent.task.Session.get_by_id", AsyncMock(side_effect=[parent_session, child_session])), \ - patch("flocks.tool.agent.task.Message.create", AsyncMock()), \ - patch("flocks.tool.agent.task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( - action="stop", - error=None, - last_message=None, - ))): - result = await task_tool( - _make_ctx(), - description="continue explore", - prompt="Continue the task", - subagent_type="explore", - session_id="ses-child", - ) - - assert result.success is True - assert result.metadata["sessionId"] == "ses-child" - assert result.metadata["emptyOutput"] is True - assert "without producing a final assistant message" in (result.output or "") - - @pytest.mark.asyncio - async def test_task_tool_sync_continue_fails_when_last_message_has_no_text(self): - parent_session = SimpleNamespace( - id="ses-parent", - project_id="proj", - directory="/tmp/project", - provider=None, - model=None, - model_pinned=False, - ) - child_session = SimpleNamespace( - id="ses-child", - agent="explore", - ) - last_message = SimpleNamespace( - id="msg-last", - sessionID="ses-child", - finish="stop", - error=None, - ) - - with patch("flocks.tool.agent.task.is_delegatable", return_value=True), \ - patch("flocks.tool.agent.task.Session.get_by_id", AsyncMock(side_effect=[parent_session, child_session])), \ - patch("flocks.tool.agent.task.Message.create", AsyncMock()), \ - patch("flocks.tool.agent.task.SessionLoop.run", AsyncMock(return_value=SimpleNamespace( - action="stop", - error=None, - last_message=last_message, - ))), \ - patch("flocks.tool.subagent_result.Message.get_text_content", AsyncMock(return_value="")): - result = await task_tool( - _make_ctx(), - description="continue explore", - prompt="Continue the task", - subagent_type="explore", - session_id="ses-child", + model="openai/gpt-5", ) - assert result.success is True - assert result.metadata["sessionId"] == "ses-child" - assert result.metadata["emptyOutput"] is True - assert "without text output" in (result.output or "") + assert result is delegate_result + delegate.assert_awaited_once() + kwargs = delegate.await_args.kwargs + assert kwargs["description"] == "delegate explore" + assert kwargs["prompt"] == "Inspect the repository" + assert kwargs["subagent_type"] == "explore" + assert kwargs["run_in_background"] is False + assert kwargs["model"] == "openai/gpt-5" diff --git a/tests/tool/test_tdp_api_tools.py b/tests/tool/test_tdp_api_tools.py index 717b41e0f..ba7073b5f 100644 --- a/tests/tool/test_tdp_api_tools.py +++ b/tests/tool/test_tdp_api_tools.py @@ -7,7 +7,7 @@ from flocks.tool.registry import ToolContext from flocks.tool.tool_loader import yaml_to_tool -BASE = Path.cwd() / ".flocks" / "plugins" / "tools" / "api" / "tdp_v3_3_10" +BASE = Path.cwd() / ".flocks" / "plugins" / "tools" / "device" / "tdp_v3_3_10" def _load_tool(yaml_name: str): @@ -99,6 +99,34 @@ async def test_tdp_dashboard_status_uses_combined_credentials_and_signs_request( assert request_kwargs["params"]["sign"] +@pytest.mark.asyncio +async def test_tdp_dashboard_status_strips_config_api_from_base_url(): + tool = _load_tool("tdp_dashboard_status.yaml") + fake_session = _FakeSession([_FakeResponse(json_payload={"response_code": 0, "data": {"agent_count": 6}})]) + + with ( + patch( + "flocks.config.config_writer.ConfigWriter.get_api_service_raw", + return_value={ + "apiKey": "{secret:tdp_api_key}", + "secret": "{secret:tdp_secret}", + "base_url": "https://tdp.local/config/api", + }, + ), + patch( + "flocks.security.get_secret_manager", + return_value=MagicMock( + get=MagicMock(side_effect=lambda key: {"tdp_api_key": "demo-api", "tdp_secret": "demo-secret"}.get(key)) + ), + ), + patch("aiohttp.ClientSession", return_value=fake_session), + ): + result = await tool.handler(ToolContext(session_id="test", message_id="test")) + + assert result.success is True + assert fake_session.calls[0][1] == "https://tdp.local/api/v1/dashboard/status" + + @pytest.mark.asyncio async def test_tdp_dashboard_status_can_switch_to_dashboard_block_action(): tool = _load_tool("tdp_dashboard_status.yaml") @@ -384,10 +412,10 @@ async def test_tdp_log_search_search_uses_default_sql_and_size(): assert request_kwargs["json"]["size"] == 10 -def test_tdp_log_search_schema_marks_sql_as_required(): +def test_tdp_log_search_schema_keeps_sql_optional_with_default(): tool = _load_tool("tdp_log_search.yaml") - assert "sql" in tool.info.get_schema().required + assert "sql" not in tool.info.get_schema().required assert tool.info.get_schema().properties["sql"]["default"] == "threat.level = 'attack'" diff --git a/tests/tool/test_tool_catalog.py b/tests/tool/test_tool_catalog.py index 15f65a215..0ded9c667 100644 --- a/tests/tool/test_tool_catalog.py +++ b/tests/tool/test_tool_catalog.py @@ -59,8 +59,7 @@ def test_catalog_uses_real_builtin_tool_names_for_metadata_keys() -> None: for name in [ "doc_parser", "lsp", - "todoread", - "todowrite", + "todo", "memory_search", "memory_get", "memory_write", diff --git a/tests/tool/test_tools.py b/tests/tool/test_tools.py index 0d5b03f72..e63e97cf7 100644 --- a/tests/tool/test_tools.py +++ b/tests/tool/test_tools.py @@ -1,11 +1,11 @@ """ -Test Tool System - Comprehensive test suite for all 25 tools +Test Tool System - Comprehensive test suite for tools Tests cover: - Tool registration and discovery - P0 Core tools (6): read, write, edit, bash, grep, glob -- P1 tools (6): webfetch, todoread, todowrite, question, plan_enter, plan_exit -- P2 tools (5): task, lsp, skill, background_output, background_cancel +- P1 tools (4): webfetch, todo, question, websearch +- P2 tools: task, lsp, skill - P3 tools (2): websearch, apply_patch - Permission system integration - Error handling @@ -35,7 +35,6 @@ ParameterType, ) from flocks.tool.code import bash as bash_module -import flocks.tool.task.plan as plan_module import flocks.tool.system.question as question_module @@ -143,18 +142,17 @@ def test_registry_initialization(self): """Test that registry initializes with built-in tools""" # Registry should be initialized when flocks.tool is imported tools = ToolRegistry.all_tool_ids() - assert len(tools) >= 23, f"Expected at least 23 tools, got {len(tools)}: {tools}" + assert len(tools) >= 21, f"Expected at least 21 tools, got {len(tools)}: {tools}" def test_expected_tools_registered(self): """Test all expected tools are registered""" expected_tools = [ # P0 Core tools (6) "read", "write", "edit", "bash", "grep", "glob", - # P1 tools (6) - "webfetch", "todoread", "todowrite", "question", "plan_enter", "plan_exit", - # P2 tools (5) + # P1 tools + "webfetch", "todo", "question", + # P2 tools "task", "lsp", "skill_load", - "background_output", "background_cancel", # P3 tools (2) "websearch", "apply_patch", ] @@ -239,7 +237,7 @@ async def test_read_nonexistent_file(self, tool_context, temp_dir): ) assert not result.success - assert "could not find" in result.error.lower() + assert "not found" in result.error.lower() @pytest.mark.asyncio async def test_read_permission_requested(self, tool_context_with_permission, test_files): @@ -591,7 +589,7 @@ class TestTodoTools: """Test the todo tools""" @pytest.mark.asyncio - async def test_todowrite_create_todos(self, tool_context): + async def test_todo_write_create_todos(self, tool_context): """Test creating todos""" todos = [ {"id": "1", "content": "First task", "status": "pending"}, @@ -599,9 +597,10 @@ async def test_todowrite_create_todos(self, tool_context): ] result = await ToolRegistry.execute( - "todowrite", + "todo", ctx=tool_context, - todos=todos + action="write", + todos=todos, ) assert result.success @@ -611,33 +610,36 @@ async def test_todowrite_create_todos(self, tool_context): assert payload["newTodos"][1]["activeForm"] == "Working on second task" assert payload["verificationNudgeNeeded"] is False - def test_todowrite_schema_requires_structured_items(self): + def test_todo_schema_requires_structured_items(self): """Tool schema should expose object items, not string arrays.""" - schema = ToolRegistry.get_schema("todowrite") + schema = ToolRegistry.get_schema("todo") assert schema is not None + assert schema.properties["action"]["enum"] == ["read", "write"] assert schema.properties["todos"]["type"] == "array" assert schema.properties["todos"]["items"]["type"] == "object" assert schema.properties["todos"]["items"]["required"] == ["id", "content", "status"] assert "activeForm" in schema.properties["todos"]["items"]["properties"] @pytest.mark.asyncio - async def test_todoread_get_todos(self, tool_context): + async def test_todo_read_get_todos(self, tool_context): """Test reading todos""" # First write some todos todos = [ {"id": "1", "content": "Test task", "status": "pending"}, ] await ToolRegistry.execute( - "todowrite", + "todo", ctx=tool_context, - todos=todos + action="write", + todos=todos, ) # Then read them result = await ToolRegistry.execute( - "todoread", - ctx=tool_context + "todo", + ctx=tool_context, + action="read", ) assert result.success @@ -645,11 +647,12 @@ async def test_todoread_get_todos(self, tool_context): assert payload[0]["content"] == "Test task" @pytest.mark.asyncio - async def test_todowrite_rejects_string_arrays(self, tool_context): + async def test_todo_write_rejects_string_arrays(self, tool_context): """Invalid todo payloads should fail loudly instead of returning [].""" result = await ToolRegistry.execute( - "todowrite", + "todo", ctx=tool_context, + action="write", todos=[ "1. First task", "2. Second task", @@ -660,7 +663,7 @@ async def test_todowrite_rejects_string_arrays(self, tool_context): assert "todos[0] must be an object" in (result.error or "") @pytest.mark.asyncio - async def test_todowrite_persists_to_session_todo_store(self, clean_todo_storage): + async def test_todo_write_persists_to_session_todo_store(self, clean_todo_storage): """todo tools should use the shared session todo store.""" from flocks.session.features.todo import Todo @@ -674,7 +677,7 @@ async def test_todowrite_persists_to_session_todo_store(self, clean_todo_storage {"id": "persist", "content": "Persist todo", "status": "pending"}, ] - result = await ToolRegistry.execute("todowrite", ctx=ctx, todos=todos) + result = await ToolRegistry.execute("todo", ctx=ctx, action="write", todos=todos) assert result.success stored = await Todo.get(session_id) @@ -683,7 +686,7 @@ async def test_todowrite_persists_to_session_todo_store(self, clean_todo_storage assert stored[0].content == "Persist todo" @pytest.mark.asyncio - async def test_todowrite_clears_storage_when_all_todos_are_terminal(self, clean_todo_storage): + async def test_todo_write_clears_storage_when_all_todos_are_terminal(self, clean_todo_storage): """Completed/cancelled-only todo lists should be cleared from persistence.""" from flocks.session.features.todo import Todo @@ -695,14 +698,16 @@ async def test_todowrite_clears_storage_when_all_todos_are_terminal(self, clean_ ) await ToolRegistry.execute( - "todowrite", + "todo", ctx=ctx, + action="write", todos=[{"id": "1", "content": "Still open", "status": "in_progress"}], ) result = await ToolRegistry.execute( - "todowrite", + "todo", ctx=ctx, + action="write", todos=[ {"id": "1", "content": "Done task", "status": "completed"}, {"id": "2", "content": "Cancelled task", "status": "cancelled"}, @@ -715,11 +720,12 @@ async def test_todowrite_clears_storage_when_all_todos_are_terminal(self, clean_ assert await Todo.get(session_id) == [] @pytest.mark.asyncio - async def test_todowrite_sets_verification_nudge_for_completed_batches(self, tool_context): + async def test_todo_write_sets_verification_nudge_for_completed_batches(self, tool_context): """Large completed batches without verification work should return a nudge.""" result = await ToolRegistry.execute( - "todowrite", + "todo", ctx=tool_context, + action="write", todos=[ {"id": "1", "content": "Implement feature", "status": "completed"}, {"id": "2", "content": "Fix bug", "status": "completed"}, @@ -742,98 +748,6 @@ async def test_question_tool_exists(self): assert tool.info.name == "question" -class TestPlanTools: - """Test the plan tools""" - - @pytest.mark.asyncio - async def test_plan_enter_exists(self): - """Test that plan_enter tool is registered""" - tool = ToolRegistry.get("plan_enter") - assert tool is not None - - @pytest.mark.asyncio - async def test_plan_exit_exists(self): - """Test that plan_exit tool is registered""" - tool = ToolRegistry.get("plan_exit") - assert tool is not None - - @pytest.mark.asyncio - async def test_plan_enter_sets_call_id_for_question_handler(self, monkeypatch): - """plan_enter should attach the current tool call id to question context.""" - seen: Dict[str, Any] = {} - switch_calls: List[Dict[str, str]] = [] - - async def fake_handler(session_id: str, questions: List[Dict[str, Any]]) -> List[List[str]]: - seen["session_id"] = session_id - seen["message_id"] = question_module.get_current_message_id() - seen["call_id"] = question_module.get_current_call_id() - seen["questions"] = questions - return [["Yes"]] - - async def fake_switch(session_id: str, from_agent: str, to_agent: str, message: str) -> None: - switch_calls.append({ - "session_id": session_id, - "from_agent": from_agent, - "to_agent": to_agent, - "message": message, - }) - - monkeypatch.setattr(question_module, "_question_handler", fake_handler) - monkeypatch.setattr(plan_module, "_agent_switch_callback", fake_switch) - - result = await ToolRegistry.execute( - "plan_enter", - ctx=ToolContext( - session_id="test-session-001", - message_id="test-message-001", - agent="test", - call_id="call-plan-enter-001", - ), - ) - - assert result.success - assert seen["session_id"] == "test-session-001" - assert seen["message_id"] == "test-message-001" - assert seen["call_id"] == "call-plan-enter-001" - assert len(switch_calls) == 1 - assert switch_calls[0]["to_agent"] == "plan" - - @pytest.mark.asyncio - async def test_plan_exit_skips_question_confirmation(self, monkeypatch): - """plan_exit should switch modes directly without opening a question.""" - switch_calls: List[Dict[str, str]] = [] - - async def fail_handler(session_id: str, questions: List[Dict[str, Any]]) -> List[List[str]]: - pytest.fail("plan_exit should not invoke the question handler") - - async def fake_switch(session_id: str, from_agent: str, to_agent: str, message: str) -> None: - switch_calls.append({ - "session_id": session_id, - "from_agent": from_agent, - "to_agent": to_agent, - "message": message, - }) - - monkeypatch.setattr(question_module, "_question_handler", fail_handler) - monkeypatch.setattr(plan_module, "_agent_switch_callback", fake_switch) - - result = await ToolRegistry.execute( - "plan_exit", - ctx=ToolContext( - session_id="test-session-002", - message_id="test-message-002", - agent="plan", - call_id="call-plan-exit-001", - ), - ) - - assert result.success - assert "switched back to rex agent" in result.output - assert len(switch_calls) == 1 - assert switch_calls[0]["to_agent"] == "rex" - assert "complete" in switch_calls[0]["message"] - - class TestWebFetchTool: """Test the webfetch tool""" @@ -1136,7 +1050,7 @@ async def test_nonexistent_tool(self, tool_context): ) assert not result.success - assert "could not find" in result.error.lower() + assert "not found" in result.error.lower() @pytest.mark.asyncio async def test_builtin_tool_rejects_unknown_parameter(self, tool_context, temp_dir): @@ -1698,16 +1612,16 @@ async def test_todo_status_transitions(self, tool_context): todos = [ {"id": "1", "content": "Task 1", "status": "pending"}, ] - await ToolRegistry.execute("todowrite", ctx=tool_context, todos=todos) + await ToolRegistry.execute("todo", ctx=tool_context, action="write", todos=todos) # Update status to in_progress todos[0]["status"] = "in_progress" - result = await ToolRegistry.execute("todowrite", ctx=tool_context, todos=todos) + result = await ToolRegistry.execute("todo", ctx=tool_context, action="write", todos=todos) assert result.success # Update status to completed todos[0]["status"] = "completed" - result = await ToolRegistry.execute("todowrite", ctx=tool_context, todos=todos) + result = await ToolRegistry.execute("todo", ctx=tool_context, action="write", todos=todos) assert result.success @pytest.mark.asyncio @@ -1720,11 +1634,11 @@ async def test_todo_multiple_items(self, tool_context): {"id": "4", "content": "Task 4", "status": "pending"}, ] - result = await ToolRegistry.execute("todowrite", ctx=tool_context, todos=todos) + result = await ToolRegistry.execute("todo", ctx=tool_context, action="write", todos=todos) assert result.success # Read and verify - read_result = await ToolRegistry.execute("todoread", ctx=tool_context) + read_result = await ToolRegistry.execute("todo", ctx=tool_context, action="read") assert read_result.success assert "Task 1" in read_result.output assert "Task 4" in read_result.output diff --git a/tests/tool/test_watcher_atomic_save.py b/tests/tool/test_watcher_atomic_save.py index e88c6a053..913b6c84a 100644 --- a/tests/tool/test_watcher_atomic_save.py +++ b/tests/tool/test_watcher_atomic_save.py @@ -13,7 +13,7 @@ from types import SimpleNamespace -from flocks.tool.registry import _tool_event_should_reload +from flocks.tool.registry import ToolFileWatcher, _tool_event_should_reload from flocks.agent.registry import _agent_event_should_reload from flocks.skill.skill import _skill_event_should_reload @@ -60,6 +60,10 @@ def test_tool_watcher_accepts_direct_modify_on_yaml() -> None: assert _tool_event_should_reload(evt) is True +def test_tool_watcher_includes_device_plugin_directory() -> None: + assert "device" in ToolFileWatcher._WATCH_SUBDIRS + + # --------------------------------------------------------------------------- # Agent watcher predicate # --------------------------------------------------------------------------- diff --git a/tests/updater/test_updater.py b/tests/updater/test_updater.py index f926ed419..51668f86e 100644 --- a/tests/updater/test_updater.py +++ b/tests/updater/test_updater.py @@ -2,6 +2,7 @@ import shutil import subprocess import tarfile +import tomllib from os import utime from pathlib import Path from types import SimpleNamespace @@ -426,6 +427,36 @@ def test_build_uv_sync_env_returns_none_on_windows( assert updater._build_uv_sync_env() is None +def test_build_dependency_sync_command_skips_project_install_on_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(updater.sys, "platform", "win32") + + assert updater._build_dependency_sync_command("uv", uv_default_index="https://mirror.example/simple") == [ + "uv", + "sync", + "--no-install-project", + "--default-index", + "https://mirror.example/simple", + ] + + +def test_build_dependency_sync_command_keeps_project_install_on_non_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(updater.sys, "platform", "linux") + + assert updater._build_dependency_sync_command("uv") == ["uv", "sync"] + + +def test_wheel_build_config_does_not_force_include_flockshub() -> None: + pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml" + pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8")) + wheel_config = pyproject["tool"]["hatch"]["build"]["targets"]["wheel"] + + assert ".flocks/flockshub" not in wheel_config.get("force-include", {}) + + def test_build_frontend_subprocess_env_prepends_bundled_node_on_windows( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -539,6 +570,8 @@ async def test_download_archive_uses_curl_user_agent_for_gitee_web_archive( captured: dict[str, object] = {} class _FakeStreamResponse: + status_code = 200 + async def __aenter__(self): return self @@ -593,6 +626,8 @@ async def test_download_archive_keeps_auth_header_for_non_gitee_sources( captured: dict[str, object] = {} class _FakeStreamResponse: + status_code = 200 + async def __aenter__(self): return self diff --git a/tests/user_defined_pages/test_api_runtime.py b/tests/user_defined_pages/test_api_runtime.py new file mode 100644 index 000000000..2b5c9beb6 --- /dev/null +++ b/tests/user_defined_pages/test_api_runtime.py @@ -0,0 +1,188 @@ +import pytest +from fastapi import FastAPI, Request +from httpx import AsyncClient, ASGITransport + +from flocks.user_defined_pages.api_runtime import UserDefinedPageApiRuntime +from flocks.user_defined_pages.store import UserDefinedPagesStore + + +@pytest.fixture +def runtime_store(tmp_path, monkeypatch): + root = tmp_path / "user_defined_pages" + monkeypatch.setenv("FLOCKS_USER_DEFINED_PAGES_ROOT", str(root)) + store = UserDefinedPagesStore() + store.create_page(page_id="runtime-page", title="运行时页面") + return store + + +@pytest.fixture +def runtime_app(): + return FastAPI() + + +@pytest.mark.asyncio +async def test_api_runtime_dispatch_sync_and_async(runtime_store: UserDefinedPagesStore, runtime_app: FastAPI): + runtime_store.save_source_file( + "runtime-page", + "api/routes.yaml", + ( + "routes:\n" + " - method: GET\n" + " path: /stats\n" + " handler: handlers.get_stats\n" + " - method: POST\n" + " path: /ack\n" + " handler: handlers.ack\n" + ), + ) + runtime_store.save_source_file( + "runtime-page", + "api/handlers.py", + ( + "def get_stats(ctx, request):\n" + " return {'ok': True, 'pageId': ctx.page_id}\n\n" + "async def ack(ctx, request):\n" + " body = await request.json()\n" + " return {'acked': body.get('id')}\n" + ), + ) + runtime = UserDefinedPageApiRuntime(runtime_store) + + @runtime_app.get("/api/user-defined-pages/{page_id}/api/{api_path:path}") + async def _get_dispatch(page_id: str, api_path: str, request: Request): + return await runtime.dispatch(page_id, api_path, request, {"role": "admin"}) + + @runtime_app.post("/api/user-defined-pages/{page_id}/api/{api_path:path}") + async def _post_dispatch(page_id: str, api_path: str, request: Request): + return await runtime.dispatch(page_id, api_path, request, {"role": "admin"}) + + async with AsyncClient(transport=ASGITransport(app=runtime_app), base_url="http://test") as client: + resp_get = await client.get("/api/user-defined-pages/runtime-page/api/stats") + assert resp_get.status_code == 200 + assert resp_get.json()["pageId"] == "runtime-page" + + resp_post = await client.post("/api/user-defined-pages/runtime-page/api/ack", json={"id": "a-1"}) + assert resp_post.status_code == 200 + assert resp_post.json() == {"acked": "a-1"} + + +@pytest.mark.asyncio +async def test_api_runtime_timeout_and_reload(runtime_store: UserDefinedPagesStore, runtime_app: FastAPI): + runtime_store.save_source_file( + "runtime-page", + "api/routes.yaml", + ( + "routes:\n" + " - method: GET\n" + " path: /slow\n" + " handler: handlers.slow\n" + " timeoutMs: 5\n" + ), + ) + runtime_store.save_source_file( + "runtime-page", + "api/handlers.py", + ( + "import asyncio\n" + "async def slow(ctx, request):\n" + " await asyncio.sleep(0.05)\n" + " return {'ok': True}\n" + ), + ) + runtime = UserDefinedPageApiRuntime(runtime_store) + + @runtime_app.get("/api/user-defined-pages/{page_id}/api/{api_path:path}") + async def _dispatch(page_id: str, api_path: str, request: Request): + return await runtime.dispatch(page_id, api_path, request, {"role": "admin"}) + + async with AsyncClient(transport=ASGITransport(app=runtime_app), base_url="http://test") as client: + timeout_resp = await client.get("/api/user-defined-pages/runtime-page/api/slow") + assert timeout_resp.status_code == 504 + + runtime_store.save_source_file( + "runtime-page", + "api/routes.yaml", + ( + "routes:\n" + " - method: GET\n" + " path: /slow\n" + " handler: handlers.fast\n" + ), + ) + runtime_store.save_source_file( + "runtime-page", + "api/handlers.py", + "def fast(ctx, request):\n return {'ok': True}\n", + ) + routes = await runtime.reload_page("runtime-page") + assert routes[0]["handler"] == "handlers.fast" + + async with AsyncClient(transport=ASGITransport(app=runtime_app), base_url="http://test") as client: + ok_resp = await client.get("/api/user-defined-pages/runtime-page/api/slow") + assert ok_resp.status_code == 200 + assert ok_resp.json() == {"ok": True} + + +@pytest.mark.asyncio +async def test_api_runtime_rejects_oversized_request_body(runtime_store: UserDefinedPagesStore, runtime_app: FastAPI): + runtime_store.save_source_file( + "runtime-page", + "api/routes.yaml", + ( + "routes:\n" + " - method: POST\n" + " path: /echo\n" + " handler: handlers.echo\n" + ), + ) + runtime_store.save_source_file( + "runtime-page", + "api/handlers.py", + ( + "async def echo(ctx, request):\n" + " body = await request.body()\n" + " return {'size': len(body)}\n" + ), + ) + runtime = UserDefinedPageApiRuntime(runtime_store) + + @runtime_app.post("/api/user-defined-pages/{page_id}/api/{api_path:path}") + async def _dispatch(page_id: str, api_path: str, request: Request): + return await runtime.dispatch(page_id, api_path, request, {"role": "admin"}) + + payload = "x" * 1_000_001 + async with AsyncClient(transport=ASGITransport(app=runtime_app), base_url="http://test") as client: + resp = await client.post("/api/user-defined-pages/runtime-page/api/echo", content=payload) + assert resp.status_code == 413 + + +@pytest.mark.asyncio +async def test_api_runtime_blocks_non_local_imports(runtime_store: UserDefinedPagesStore, runtime_app: FastAPI): + runtime_store.save_source_file( + "runtime-page", + "api/routes.yaml", + ( + "routes:\n" + " - method: GET\n" + " path: /unsafe\n" + " handler: handlers.unsafe\n" + ), + ) + runtime_store.save_source_file( + "runtime-page", + "api/handlers.py", + ( + "from flocks.server import app\n" + "def unsafe(ctx, request):\n" + " return {'ok': True}\n" + ), + ) + runtime = UserDefinedPageApiRuntime(runtime_store) + + @runtime_app.get("/api/user-defined-pages/{page_id}/api/{api_path:path}") + async def _dispatch(page_id: str, api_path: str, request: Request): + return await runtime.dispatch(page_id, api_path, request, {"role": "admin"}) + + async with AsyncClient(transport=ASGITransport(app=runtime_app), base_url="http://test") as client: + resp = await client.get("/api/user-defined-pages/runtime-page/api/unsafe") + assert resp.status_code == 500 diff --git a/tests/user_defined_pages/test_bootstrap.py b/tests/user_defined_pages/test_bootstrap.py new file mode 100644 index 000000000..8aecc6e85 --- /dev/null +++ b/tests/user_defined_pages/test_bootstrap.py @@ -0,0 +1,39 @@ +import pytest + +from flocks.user_defined_pages.bootstrap import reconcile_user_defined_pages +from flocks.user_defined_pages.store import UserDefinedPagesStore + + +class _BuilderStub: + def __init__(self): + self.calls: list[str] = [] + + def build(self, page_id: str): + self.calls.append(page_id) + return type("Meta", (), {"status": "ready", "error": None}) + + +class _RuntimeStub: + def __init__(self): + self.calls: list[str] = [] + + async def reload_page(self, page_id: str): + self.calls.append(page_id) + return [] + + +@pytest.mark.asyncio +async def test_reconcile_rebuilds_missing_bundle_and_preloads_api(tmp_path, monkeypatch): + root = tmp_path / "user_defined_pages" + monkeypatch.setenv("FLOCKS_USER_DEFINED_PAGES_ROOT", str(root)) + store = UserDefinedPagesStore() + store.create_page(page_id="boot-page", title="启动页") + store.save_source_file("boot-page", "api/routes.yaml", "routes: []\n") + store.save_source_file("boot-page", "api/handlers.py", "def noop(ctx, request):\n return {}\n") + + builder = _BuilderStub() + runtime = _RuntimeStub() + await reconcile_user_defined_pages(store=store, builder=builder, runtime=runtime) + + assert builder.calls == ["boot-page"] + assert runtime.calls == ["boot-page"] diff --git a/tests/user_defined_pages/test_builder.py b/tests/user_defined_pages/test_builder.py new file mode 100644 index 000000000..e4af8c8f2 --- /dev/null +++ b/tests/user_defined_pages/test_builder.py @@ -0,0 +1,32 @@ +import pytest + +from flocks.user_defined_pages.builder import UserDefinedPagesBuilder, resolve_esbuild_bin +from flocks.user_defined_pages.store import UserDefinedPagesStore + + +@pytest.fixture +def built_store(tmp_path, monkeypatch): + root = tmp_path / "user_defined_pages" + monkeypatch.setenv("FLOCKS_USER_DEFINED_PAGES_ROOT", str(root)) + store = UserDefinedPagesStore() + store.create_page(page_id="build-page", title="构建页") + return store + + +@pytest.mark.skipif(resolve_esbuild_bin() is None, reason="esbuild is not installed") +def test_builder_produces_ready_bundle(built_store: UserDefinedPagesStore): + builder = UserDefinedPagesBuilder(built_store) + meta = builder.build("build-page") + assert meta.status == "ready" + assert meta.hash + assert built_store.bundle_path("build-page").is_file() + + +def test_builder_rejects_entry_outside_page_dir(built_store: UserDefinedPagesStore): + built_store.create_page(page_id="build-page-neighbor", title="相邻页") + built_store.save_manifest("build-page", {"entry": "../build-page-neighbor/src/index.tsx"}) + + builder = UserDefinedPagesBuilder(built_store) + + with pytest.raises(ValueError, match="invalid entry path"): + builder.build("build-page") diff --git a/tests/user_defined_pages/test_store.py b/tests/user_defined_pages/test_store.py new file mode 100644 index 000000000..4952598d9 --- /dev/null +++ b/tests/user_defined_pages/test_store.py @@ -0,0 +1,74 @@ +import json + +import pytest + +from flocks.user_defined_pages.store import UserDefinedPagesStore + + +@pytest.fixture +def store(tmp_path, monkeypatch): + root = tmp_path / "user_defined_pages" + monkeypatch.setenv("FLOCKS_USER_DEFINED_PAGES_ROOT", str(root)) + return UserDefinedPagesStore() + + +def test_create_page_scaffold(store: UserDefinedPagesStore): + detail = store.create_page(page_id="my-dashboard", title="我的大屏") + assert detail.manifest.id == "my-dashboard" + assert detail.manifest.route == "/user-defined-pages/my-dashboard" + assert (store.page_dir("my-dashboard") / "src" / "Page.tsx").is_file() + assert (store.page_dir("my-dashboard") / "manifest.json").is_file() + + +def test_list_pages_enabled_only(store: UserDefinedPagesStore): + store.create_page(page_id="enabled-page", title="启用页") + disabled = store.create_page(page_id="disabled-page", title="禁用页") + store.save_manifest("disabled-page", {**disabled.manifest.model_dump(), "enabled": False}) + + all_pages = store.list_pages(enabled_only=False) + enabled_pages = store.list_pages(enabled_only=True) + + assert {page.id for page in all_pages} == {"enabled-page", "disabled-page"} + assert [page.id for page in enabled_pages] == ["enabled-page"] + + +def test_reject_path_traversal_on_write(store: UserDefinedPagesStore): + store.create_page(page_id="safe-page", title="安全页") + with pytest.raises(ValueError, match="writes are not allowed"): + store.save_source_file("safe-page", "../escape.tsx", "bad") + + +def test_allow_page_api_source_files(store: UserDefinedPagesStore): + store.create_page(page_id="api-page", title="API 页") + store.save_source_file("api-page", "api/routes.yaml", "routes: []\n") + store.save_source_file("api-page", "api/handlers.py", "def ping(ctx, request):\n return {'ok': True}\n") + assert store.read_source_file("api-page", "api/routes.yaml").startswith("routes:") + detail = store.get_page("api-page") + assert "api/routes.yaml" in detail.sourceFiles + assert "api/handlers.py" in detail.sourceFiles + + +def test_reject_unsupported_api_extension(store: UserDefinedPagesStore): + store.create_page(page_id="api-ext-page", title="API 后缀页") + with pytest.raises(ValueError, match="unsupported source file type"): + store.save_source_file("api-ext-page", "api/secret.txt", "nope") + + +def test_reject_invalid_page_id(store: UserDefinedPagesStore): + with pytest.raises(ValueError, match="invalid page id"): + store.validate_page_id("../bad") + + +def test_asset_path_stays_inside_assets_dir(store: UserDefinedPagesStore): + store.create_page(page_id="asset-page", title="资源页") + with pytest.raises(ValueError, match="path traversal is not allowed"): + store.asset_path("asset-page", "../manifest.json") + + +def test_manifest_roundtrip(store: UserDefinedPagesStore): + store.create_page(page_id="roundtrip", title="原始标题") + manifest = store.save_manifest("roundtrip", {"title": "新标题", "order": 10}) + assert manifest.title == "新标题" + assert manifest.order == 10 + raw = json.loads((store.page_dir("roundtrip") / "manifest.json").read_text(encoding="utf-8")) + assert raw["route"] == "/user-defined-pages/roundtrip" diff --git a/tests/user_defined_pages/test_watcher.py b/tests/user_defined_pages/test_watcher.py new file mode 100644 index 000000000..535ef23b2 --- /dev/null +++ b/tests/user_defined_pages/test_watcher.py @@ -0,0 +1,36 @@ +from flocks.user_defined_pages import watcher as watcher_module +from flocks.user_defined_pages.watcher import UserDefinedPagesWatcher, _PendingAction + + +class _RuntimeStub: + async def reload_page(self, _page_id: str): + return [{"method": "GET", "path": "/stats", "handler": "handlers.stats"}] + + +class _BuilderStub: + def build(self, _page_id: str): + raise AssertionError("build should not be called for api-only change") + + +def test_watcher_api_change_uses_main_loop_bridge(monkeypatch): + emitted: list[tuple[str, dict]] = [] + bridge_calls: list[str] = [] + + def _bridge(coro, *, timeout_seconds=5.0): + bridge_calls.append("called") + coro.close() + return [{"method": "GET", "path": "/stats", "handler": "handlers.stats"}] + + def _emit(event_type: str, properties: dict): + emitted.append((event_type, properties)) + + monkeypatch.setattr(watcher_module, "_run_on_main_loop_sync", _bridge) + monkeypatch.setattr(watcher_module, "_publish_event_sync", _emit) + + watcher = UserDefinedPagesWatcher(builder=_BuilderStub(), api_runtime=_RuntimeStub()) + watcher._pending_pages["demo-page"] = _PendingAction(api_changed=True) + watcher._run_pending_builds() + + assert bridge_calls == ["called"] + assert emitted[0][0] == "user_defined_pages.api_changed" + assert emitted[0][1]["id"] == "demo-page" diff --git a/tests/utils/test_append_upgrade_log.py b/tests/utils/test_append_upgrade_log.py index 666253689..94ffceca5 100644 --- a/tests/utils/test_append_upgrade_log.py +++ b/tests/utils/test_append_upgrade_log.py @@ -2,6 +2,7 @@ import re from pathlib import Path +from datetime import date from flocks.utils.log import append_upgrade_text_log @@ -10,8 +11,9 @@ def test_append_upgrade_text_log_writes_timestamped_lines(monkeypatch, tmp_path: monkeypatch.setenv("FLOCKS_LOG_DIR", str(tmp_path)) append_upgrade_text_log("first line") append_upgrade_text_log("a\nb") - text = (tmp_path / "update.log").read_text(encoding="utf-8") + text = (tmp_path / date.today().isoformat() / "errors.log").read_text(encoding="utf-8") lines = text.strip().splitlines() + assert not (tmp_path / "update.log").exists() assert len(lines) == 3 assert re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| first line$", lines[0]) assert re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| a$", lines[1]) diff --git a/tests/utils/test_log_compatibility.py b/tests/utils/test_log_compatibility.py index bcaabd2ac..2e9ca76a7 100644 --- a/tests/utils/test_log_compatibility.py +++ b/tests/utils/test_log_compatibility.py @@ -6,7 +6,9 @@ import time import tempfile from pathlib import Path -from flocks.utils.log import Log, Logger, LogLevel, _RotatingTextWriter, rotate_log_file +from datetime import datetime, timedelta +import flocks.utils.log as log_module +from flocks.utils.log import Log, Logger, LogLevel class TestLoggerCompatibility: @@ -245,38 +247,6 @@ def test_large_object_values_are_truncated(self, monkeypatch): finally: Log._writer = old_stderr - def test_rotate_log_file_keeps_bounded_backups(self, tmp_path: Path): - """Test oversized runtime logs rotate before another process appends.""" - log_path = tmp_path / "backend.log" - log_path.write_text("x" * 20, encoding="utf-8") - - rotate_log_file(log_path, max_bytes=10, backup_count=2) - - assert not log_path.exists() - assert (tmp_path / "backend.log.1").read_text(encoding="utf-8") == "x" * 20 - - log_path.write_text("y" * 20, encoding="utf-8") - rotate_log_file(log_path, max_bytes=10, backup_count=2) - - assert (tmp_path / "backend.log.1").read_text(encoding="utf-8") == "y" * 20 - assert (tmp_path / "backend.log.2").read_text(encoding="utf-8") == "x" * 20 - - def test_rotating_text_writer_rotates_during_log_writes(self, tmp_path: Path): - """Test Log's writer rotates when the next line would exceed the limit.""" - log_path = tmp_path / "session.log" - writer = _RotatingTextWriter(log_path, max_bytes=12, backup_count=1) - - try: - writer.write("first\n") - writer.flush() - writer.write("second\n") - writer.flush() - finally: - writer.close() - - assert log_path.read_text(encoding="utf-8") == "second\n" - assert (tmp_path / "session.log.1").read_text(encoding="utf-8") == "first\n" - def test_time_diff_calculation(self): """Test time difference calculation between logs""" logger = Log.create(service="test") @@ -342,14 +312,84 @@ async def test_init_creates_log_file(self): log_dir = Path(tmpdir) / ".flocks" / "logs" assert log_dir.exists() - dev_log = log_dir / "dev.log" - assert dev_log.exists() + today_dir = log_dir / datetime.now().date().isoformat() + assert (today_dir / "flocks.log").exists() + assert (today_dir / "errors.log").exists() + assert not (log_dir / "dev.log").exists() finally: # Restore os.environ["HOME"] = str(old_home) if Log._writer: Log._writer.close() Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None + + async def test_init_uses_stable_main_log_file(self): + """Test production logging appends to the daily flocks.log.""" + with tempfile.TemporaryDirectory() as tmpdir: + old_home = Path.home() + import os + os.environ["HOME"] = tmpdir + + try: + await Log.init(print=False, dev=False, level=LogLevel.INFO) + Log.Default.info("first") + first_file = Path(Log.file()) + await Log.init(print=False, dev=False, level=LogLevel.INFO) + Log.Default.info("second") + + log_dir = Path(tmpdir) / ".flocks" / "logs" + today_dir = log_dir / datetime.now().date().isoformat() + assert first_file == today_dir / "flocks.log" + assert Path(Log.file()) == first_file + assert not list(log_dir.glob("????-??-??T??????.log")) + assert not list(log_dir.glob("*.log.1")) + content = first_file.read_text(encoding="utf-8") + assert "first" in content + assert "second" in content + finally: + os.environ["HOME"] = str(old_home) + if Log._writer: + Log._writer.close() + Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None + + async def test_warn_and_error_are_copied_to_errors_log(self): + """Test warning and error lines are available in errors.log for quick triage.""" + with tempfile.TemporaryDirectory() as tmpdir: + old_home = Path.home() + import os + os.environ["HOME"] = tmpdir + + try: + await Log.init(print=False, dev=False, level=LogLevel.INFO) + logger = Log.create(service="error-copy") + logger.info("info") + logger.warn("warn") + logger.error("error") + + log_dir = Path(tmpdir) / ".flocks" / "logs" + today_dir = log_dir / datetime.now().date().isoformat() + main_content = (today_dir / "flocks.log").read_text(encoding="utf-8") + error_content = (today_dir / "errors.log").read_text(encoding="utf-8") + assert "info" in main_content + assert "warn" in main_content + assert "error" in main_content + assert "info" not in error_content + assert "warn" in error_content + assert "error" in error_content + finally: + os.environ["HOME"] = str(old_home) + if Log._writer: + Log._writer.close() + Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None async def test_init_print_mode(self): """Test that init with print=True uses stderr""" @@ -357,6 +397,7 @@ async def test_init_print_mode(self): # Should not create a file writer assert Log._writer is None + assert Log._error_writer is None async def test_log_file_path(self): """Test log file path method""" @@ -364,18 +405,82 @@ async def test_log_file_path(self): assert file_path.endswith("flocks.log") or "flocks" in file_path async def test_cleanup_removes_rotated_siblings_for_old_timestamp_logs(self, tmp_path: Path): - """Test cleanup deletes rotated backups for base files outside retention.""" - for day in range(11): - base = tmp_path / f"2026-05-{day + 1:02d}T010203.log" - base.write_text("base", encoding="utf-8") - (tmp_path / f"{base.name}.1").write_text("rotated", encoding="utf-8") + """Test cleanup deletes legacy timestamp logs by age, not by file count.""" + old_stamp = (datetime.now() - timedelta(days=31)).strftime("%Y-%m-%dT%H%M%S") + recent_stamp = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%dT%H%M%S") + old_base = tmp_path / f"{old_stamp}.log" + recent_base = tmp_path / f"{recent_stamp}.log" + old_base.write_text("old", encoding="utf-8") + (tmp_path / f"{old_base.name}.1").write_text("old rotated", encoding="utf-8") + recent_base.write_text("recent", encoding="utf-8") + (tmp_path / f"{recent_base.name}.1").write_text("recent rotated", encoding="utf-8") - await Log._cleanup(tmp_path) + await Log._cleanup(tmp_path, retention_days=30) - assert not (tmp_path / "2026-05-01T010203.log").exists() - assert not (tmp_path / "2026-05-01T010203.log.1").exists() - assert (tmp_path / "2026-05-02T010203.log").exists() - assert (tmp_path / "2026-05-02T010203.log.1").exists() + assert not old_base.exists() + assert not (tmp_path / f"{old_base.name}.1").exists() + assert recent_base.exists() + assert (tmp_path / f"{recent_base.name}.1").exists() + + async def test_cleanup_removes_old_date_directories(self, tmp_path: Path): + old_day = (datetime.now() - timedelta(days=31)).date().isoformat() + recent_day = (datetime.now() - timedelta(days=1)).date().isoformat() + old_dir = tmp_path / old_day + recent_dir = tmp_path / recent_day + old_dir.mkdir() + recent_dir.mkdir() + (old_dir / "flocks.log").write_text("old", encoding="utf-8") + (recent_dir / "flocks.log").write_text("recent", encoding="utf-8") + + await Log._cleanup(tmp_path, retention_days=30) + + assert not old_dir.exists() + assert recent_dir.exists() + + async def test_day_rollover_switches_writer_and_runs_cleanup(self, tmp_path: Path, monkeypatch): + """Test long-running processes move to the new daily log and clean old days.""" + old_day = (datetime.now() - timedelta(days=31)).date().isoformat() + first_day = (datetime.now() - timedelta(days=1)).date() + second_day = datetime.now().date() + old_dir = tmp_path / old_day + old_dir.mkdir() + (old_dir / "flocks.log").write_text("old", encoding="utf-8") + + class FakeDate: + current = first_day + + @classmethod + def today(cls): + return cls.current + + monkeypatch.setenv("FLOCKS_LOG_DIR", str(tmp_path)) + monkeypatch.setattr(log_module, "date", FakeDate) + + try: + await Log.init(print=False, dev=False, level=LogLevel.INFO) + Log.Default.info("first day") + first_file = Path(Log.file()) + + FakeDate.current = second_day + Log.Default.warn("second day") + second_file = Path(Log.file()) + + assert first_file == tmp_path / first_day.isoformat() / "flocks.log" + assert second_file == tmp_path / second_day.isoformat() / "flocks.log" + assert "first day" in first_file.read_text(encoding="utf-8") + assert "second day" in second_file.read_text(encoding="utf-8") + assert "second day" in (tmp_path / second_day.isoformat() / "errors.log").read_text(encoding="utf-8") + assert not old_dir.exists() + finally: + if Log._writer: + Log._writer.close() + Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None + Log._log_file = None + Log._log_dir_path = None + Log._log_date = None if __name__ == "__main__": diff --git a/tests/workflow/test_logging_config.py b/tests/workflow/test_logging_config.py index 656c90a8d..ed9a5878c 100644 --- a/tests/workflow/test_logging_config.py +++ b/tests/workflow/test_logging_config.py @@ -1,24 +1,50 @@ import logging -from logging.handlers import RotatingFileHandler +from io import StringIO from pathlib import Path from flocks.workflow.logging_config import setup_workflow_logging +from flocks.workflow.runner import run_workflow -def test_workflow_file_logging_uses_rotating_handler(monkeypatch, tmp_path: Path) -> None: +def test_workflow_file_logging_is_disabled(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv("FLOCKS_LOG_DIR", str(tmp_path)) - monkeypatch.setenv("FLOCKS_LOG_MAX_BYTES", "1234") - monkeypatch.setenv("FLOCKS_LOG_BACKUP_COUNT", "2") setup_workflow_logging(stream=None) logger = logging.getLogger("flocks.workflow") - handlers = [handler for handler in logger.handlers if isinstance(handler, RotatingFileHandler)] try: - assert len(handlers) == 1 - assert handlers[0].baseFilename == str(tmp_path / "workflow.log") - assert handlers[0].maxBytes == 1234 - assert handlers[0].backupCount == 2 + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], logging.StreamHandler) + assert not (tmp_path / "workflow.log").exists() + finally: + logger.handlers.clear() + + +def test_run_workflow_default_logging_suppresses_routine_execution_noise() -> None: + stream = StringIO() + setup_workflow_logging(stream=stream) + + logger = logging.getLogger("flocks.workflow") + try: + result = run_workflow( + workflow={ + "start": "collect_messages", + "nodes": [ + { + "id": "collect_messages", + "type": "python", + "code": "outputs['ok'] = True", + }, + ], + "edges": [], + }, + ensure_requirements=False, + ) + + assert result.status == "SUCCEEDED" + logs = stream.getvalue() + assert "开始执行 workflow" not in logs + assert "workflow 信息" not in logs finally: logger.handlers.clear() diff --git a/tests/workflow/test_run_workflow_history.py b/tests/workflow/test_run_workflow_history.py index c5861f964..2caa1b25d 100644 --- a/tests/workflow/test_run_workflow_history.py +++ b/tests/workflow/test_run_workflow_history.py @@ -1,4 +1,4 @@ -"""Test run_workflow tool returns execution history in results.""" +"""Test run_workflow tool keeps execution history in metadata.""" import json import pytest @@ -8,21 +8,16 @@ from flocks.tool.registry import ToolContext -class MockToolContext: - """Mock ToolContext for testing.""" - - async def ask(self, **kwargs): - """Mock permission request - always allow.""" - pass - - def metadata(self, data=None, **kwargs): - """Mock metadata update.""" - pass +class MockToolContext(ToolContext): + """ToolContext with stable IDs for workflow tests.""" + + def __init__(self) -> None: + super().__init__(session_id="test-session", message_id="test-message") @pytest.mark.asyncio async def test_workflow_history_in_output(): - """Test that workflow execution history is included in tool output.""" + """History stays in metadata while the default output stays concise.""" # Create a simple test workflow workflow = { @@ -104,18 +99,17 @@ async def test_workflow_history_in_output(): assert "outputs" in result.metadata assert result.metadata["outputs"]["final"] == 35 - # Verify output text contains history information - assert "Execution History" in result.output - assert "step1" in result.output - assert "step2" in result.output - assert "step3" in result.output - assert "Inputs:" in result.output - assert "Outputs:" in result.output + # Verify output text no longer expands the full execution history + assert "Status: SUCCEEDED" in result.output + assert "Final Outputs:" in result.output + assert "Execution History" not in result.output + assert "Inputs:" not in result.output + assert "Stdout:" not in result.output @pytest.mark.asyncio async def test_workflow_history_with_error(): - """Test that workflow history is included even when execution fails.""" + """History is preserved in metadata even when execution fails.""" workflow = { "name": "test_error_workflow", @@ -172,14 +166,16 @@ async def test_workflow_history_with_error(): assert "Intentional error" in step2["error"] assert "traceback" in step2 - # Output should contain error information + # Output should contain only the top-level failure summary assert "Error:" in result.output - assert "step2" in result.output + assert "Execution History" not in result.output + assert "Inputs:" not in result.output + assert "Stdout:" not in result.output @pytest.mark.asyncio async def test_workflow_history_with_stdout(): - """Test that stdout from nodes is captured in history.""" + """Stdout remains in metadata history even if hidden from tool output.""" workflow = { "name": "test_stdout_workflow", @@ -214,9 +210,9 @@ async def test_workflow_history_with_stdout(): assert "stdout" in step1 assert "Hello from step1" in step1["stdout"] - # Output should show stdout - assert "Stdout:" in result.output - assert "Hello from step1" in result.output + # Output should stay concise and omit per-step stdout details + assert "Stdout:" not in result.output + assert "Hello from step1" not in result.output if __name__ == "__main__": diff --git a/tests/workflow/test_tool_run_workflow.py b/tests/workflow/test_tool_run_workflow.py index 0764e99bf..afd0fa491 100644 --- a/tests/workflow/test_tool_run_workflow.py +++ b/tests/workflow/test_tool_run_workflow.py @@ -614,6 +614,35 @@ async def test_run_workflow_with_trace(self, tool_context_with_permission, simpl assert call_kwargs.get("trace") is True assert call_kwargs.get("use_llm") is True + @pytest.mark.anyio + async def test_run_workflow_passes_cancel_callback(self, tool_context_with_permission, simple_workflow): + """Session abort should be forwarded to workflow runtime cancellation.""" + fake = FakeRunWorkflowResult(**{ + "status": "SUCCEEDED", + "run_id": "run-cancel", + "steps": 1, + "last_node_id": "node-1", + "outputs": {}, + "history": [], + "error": None, + }) + mock_run = Mock(name="run_workflow", return_value=fake) + with patch.object(run_workflow_module, "_get_workflow_runtime", return_value=_runtime_tuple(run_fn=mock_run)): + result = await ToolRegistry.execute( + "run_workflow", + ctx=tool_context_with_permission, + workflow=simple_workflow, + inputs={}, + ) + + assert result.success is True + call_kwargs = mock_run.call_args[1] + cancel = call_kwargs.get("cancel") + assert callable(cancel) + assert cancel() is False + tool_context_with_permission.abort.set() + assert cancel() is True + @pytest.mark.anyio async def test_run_workflow_disable_llm(self, tool_context_with_permission, simple_workflow): """Test workflow execution with use_llm disabled""" @@ -726,8 +755,9 @@ async def test_run_workflow_result_formatting(self, tool_context_with_permission assert "Run ID: run-format" in output assert "Steps executed: 3" in output assert "Last node: node-3" in output - assert "Outputs:" in output - assert "Execution History" in output + assert "Final Outputs:" in output + assert "Execution History" not in output + assert result.metadata["history"] == fake.history @pytest.mark.anyio async def test_run_workflow_result_with_error(self, tool_context_with_permission, simple_workflow): diff --git a/tests/workflow/test_trigger_dispatcher.py b/tests/workflow/test_trigger_dispatcher.py new file mode 100644 index 000000000..276a90ba4 --- /dev/null +++ b/tests/workflow/test_trigger_dispatcher.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import pytest + +from flocks.workflow.triggers.dispatcher import ( + EventDispatcher, + build_trigger_event, + evaluate_trigger_filter, + lookup_mapping_path, + preview_trigger_mapping, +) +from flocks.workflow.triggers.models import TriggerDefinition + + +def test_lookup_mapping_path_supports_nested_access() -> None: + payload = { + "body": { + "data": [ + {"severity": "high", "source": {"ip": "1.1.1.1"}}, + ] + } + } + + assert lookup_mapping_path(payload, "$.body.data[0].severity") == "high" + assert lookup_mapping_path(payload, "$.body.data[0].source.ip") == "1.1.1.1" + assert lookup_mapping_path(payload, "$.body.data[1]") is None + + +def test_preview_trigger_mapping_builds_flocks_envelope() -> None: + trigger = TriggerDefinition.model_validate( + { + "id": "custom-webhook", + "type": "custom_webhook", + "mapping": { + "alert_data": "$.body.data[0]", + }, + "inputs": {"static_value": 7}, + } + ) + event = build_trigger_event( + workflow_id="wf-1", + trigger=trigger, + body={"data": [{"severity": "high"}]}, + ) + + mapped = preview_trigger_mapping(trigger, event) + + assert mapped["static_value"] == 7 + assert mapped["alert_data"] == {"severity": "high"} + assert mapped["_flocks"]["trigger"]["id"] == "custom-webhook" + assert mapped["_flocks"]["trigger"]["type"] == "custom_webhook" + + +def test_trigger_filter_expression_matches_expected_payload() -> None: + trigger = TriggerDefinition.model_validate( + { + "id": "high-only", + "type": "custom_webhook", + "filter": {"expr": "body.data[0].severity in ['high', 'critical']"}, + } + ) + event = build_trigger_event( + workflow_id="wf-1", + trigger=trigger, + body={"data": [{"severity": "high"}]}, + ) + + matched, error = evaluate_trigger_filter(trigger, event) + + assert matched is True + assert error is None + + +@pytest.mark.asyncio +async def test_event_dispatcher_skips_execution_when_filter_does_not_match() -> None: + dispatcher = EventDispatcher() + trigger = TriggerDefinition.model_validate( + { + "id": "critical-only", + "type": "custom_webhook", + "filter": {"expr": "body.severity == 'critical'"}, + "mapping": {"severity": "$.body.severity"}, + } + ) + event = build_trigger_event( + workflow_id="wf-1", + trigger=trigger, + body={"severity": "low"}, + ) + + async def _executor(_inputs: dict[str, object]) -> dict[str, bool]: + raise AssertionError("executor must not run when the filter misses") + + result = await dispatcher.dispatch(trigger=trigger, event=event, executor=_executor) + + assert result["matched"] is False + assert result["executed"] is False + assert result["inputs"]["severity"] == "low" diff --git a/tests/workflow/test_trigger_runtime.py b/tests/workflow/test_trigger_runtime.py new file mode 100644 index 000000000..e230a29b1 --- /dev/null +++ b/tests/workflow/test_trigger_runtime.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from flocks.workflow.triggers import runtime as runtime_module + + +@pytest.mark.asyncio +async def test_sync_legacy_configs_disables_explicit_empty_trigger_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + writes: list[tuple[str, dict]] = [] + + async def _fake_write(key: str, value: dict) -> None: + writes.append((key, value)) + + monkeypatch.setattr(runtime_module.Storage, "write", _fake_write) + + runtime = runtime_module.TriggerRuntime() + triggers = await runtime._sync_legacy_configs_from_workflow( # noqa: SLF001 + "wf-empty", + {"start": "n1", "nodes": [], "edges": [], "triggers": []}, + ) + + assert triggers == [] + assert { + key for key, _value in writes + } == { + "workflow_poller_config/wf-empty", + "workflow_syslog_config/wf-empty", + "workflow_kafka_config/wf-empty", + } + assert all(value["enabled"] is False for _key, value in writes) + + +@pytest.mark.asyncio +async def test_custom_adapter_restarts_when_definition_changes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + started_modes: list[str] = [] + stopped_modes: list[str] = [] + + class _FakeAdapter: + def __init__(self, definition: dict) -> None: + self._definition = definition + + def start(self, definition: dict, emit) -> None: # noqa: ANN001 + del emit + started_modes.append(str((definition.get("source") or {}).get("mode"))) + + def stop(self) -> None: + stopped_modes.append(str((self._definition.get("source") or {}).get("mode"))) + + monkeypatch.setattr( + runtime_module, + "list_trigger_plugins", + lambda: [{"id": "demo-adapter", "handlerPath": "/tmp/demo-handler.py"}], + ) + monkeypatch.setattr( + runtime_module, + "load_trigger_plugin_module", + lambda _plugin_spec: SimpleNamespace( + create_trigger_adapter=lambda definition: _FakeAdapter(definition) + ), + ) + + runtime = runtime_module.TriggerRuntime() + initial_workflow = { + "triggers": [ + { + "id": "custom-trigger", + "type": "custom_adapter", + "enabled": True, + "source": {"adapterId": "demo-adapter", "mode": "initial"}, + } + ] + } + updated_workflow = { + "triggers": [ + { + "id": "custom-trigger", + "type": "custom_adapter", + "enabled": True, + "source": {"adapterId": "demo-adapter", "mode": "updated"}, + } + ] + } + + await runtime._start_custom_adapters_for_workflow("wf-custom", initial_workflow) # noqa: SLF001 + await asyncio.sleep(0) + + await runtime._start_custom_adapters_for_workflow("wf-custom", updated_workflow) # noqa: SLF001 + await asyncio.sleep(0) + + assert started_modes == ["initial", "updated"] + assert stopped_modes == ["initial"] + + await runtime.stop_all() diff --git a/tests/workflow/test_trigger_schedule_cron.py b/tests/workflow/test_trigger_schedule_cron.py new file mode 100644 index 000000000..4f0710a62 --- /dev/null +++ b/tests/workflow/test_trigger_schedule_cron.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from flocks.workflow.poller_manager import WorkflowPollerManager + + +def test_poller_config_supports_cron_expression() -> None: + manager = WorkflowPollerManager() + + config = manager._normalize_config( # noqa: SLF001 - focused unit test + "wf-1", + { + "enabled": True, + "cronExpression": "*/5 * * * *", + "timeoutSeconds": 120, + }, + ) + + assert config["enabled"] is True + assert config["cronExpression"] == "*/5 * * * *" + assert config["intervalSeconds"] == 30 + + +def test_poller_next_run_uses_cron_when_present() -> None: + manager = WorkflowPollerManager() + + next_run_at = manager._compute_next_run_at_ms( # noqa: SLF001 - focused unit test + { + "intervalSeconds": 30, + "cronExpression": "*/5 * * * *", + }, + base_ts_s=0, + ) + + assert next_run_at == 300000 diff --git a/tests/workflow/test_workflow_cancellation.py b/tests/workflow/test_workflow_cancellation.py index 008f555d9..a7bfd5621 100644 --- a/tests/workflow/test_workflow_cancellation.py +++ b/tests/workflow/test_workflow_cancellation.py @@ -179,3 +179,94 @@ async def _sleep_tool(ctx) -> ToolResult: assert result.status == "CANCELLED" assert time.perf_counter() - started < 1.0 + + +def test_run_workflow_cancels_python_node_llm_ask(monkeypatch) -> None: + """Python node llm.ask() should use the same cooperative cancellation path.""" + from flocks.provider import provider as provider_mod + from flocks.workflow import llm as workflow_llm_mod + + class _FakeResponse: + def __init__(self, content: str): + self.content = content + + class _FakeModel: + def __init__(self, model_id: str): + self.id = model_id + + class _SlowProvider: + id = "demo" + + def configure(self, _cfg): + return None + + def is_configured(self): + return True + + async def chat(self, model_id: str, messages, **kwargs): + del model_id, messages, kwargs + await asyncio.sleep(5) + return _FakeResponse("late") + + provider = _SlowProvider() + + monkeypatch.setattr(provider_mod.Provider, "_ensure_initialized", lambda: None) + + async def _noop_apply_config(*_args, **_kwargs): + return None + + monkeypatch.setattr(provider_mod.Provider, "apply_config", _noop_apply_config) + monkeypatch.setattr(provider_mod.Provider, "get", lambda pid: provider if pid == "demo" else None) + monkeypatch.setattr( + provider_mod.Provider, + "list_models", + lambda provider_id=None: [_FakeModel("m")] if provider_id == "demo" else [], + ) + + async def _noop_config_get(): + class _Cfg: + model = None + + def model_dump(self, **kwargs): + del kwargs + return {} + + return _Cfg() + + async def _resolve_default_llm(): + return {"provider_id": "demo", "model_id": "m"} + + monkeypatch.setattr(workflow_llm_mod.Config, "get", _noop_config_get) + monkeypatch.setattr(workflow_llm_mod.Config, "resolve_default_llm", _resolve_default_llm) + + workflow = { + "name": "cancel_python_llm_ask", + "start": "slow", + "nodes": [ + { + "id": "slow", + "type": "python", + "code": "outputs['answer'] = llm.ask('hello')", + } + ], + "edges": [], + } + + cancel_event = threading.Event() + timer = threading.Timer(0.05, cancel_event.set) + started = time.perf_counter() + timer.start() + try: + result = run_workflow( + workflow=workflow, + inputs={}, + ensure_requirements=False, + node_timeout_s=10, + cancel=cancel_event.is_set, + ) + finally: + timer.cancel() + + assert result.status == "CANCELLED" + assert result.outputs == {} + assert time.perf_counter() - started < 1.0 diff --git a/tests/workflow/test_workflow_llm.py b/tests/workflow/test_workflow_llm.py index 27c921485..1a845fc4c 100644 --- a/tests/workflow/test_workflow_llm.py +++ b/tests/workflow/test_workflow_llm.py @@ -1,7 +1,10 @@ import asyncio +import threading +import time import pytest +from flocks.workflow.errors import RunCancelledError from flocks.workflow.llm import LLMClient from flocks.workflow.engine import WorkflowEngine from flocks.workflow.models import Workflow @@ -25,13 +28,38 @@ def __init__( *, configured: bool = True, models: list[str] | None = None, + _shared_state: dict[str, object] | None = None, ): self.id = provider_id self._behavior = behavior self._configured = configured self._models = models or [] - self.calls = 0 - self.last_config = None + self._shared_state = _shared_state or {"calls": 0, "last_config": None} + + @property + def calls(self) -> int: + return int(self._shared_state["calls"]) + + @calls.setter + def calls(self, value: int) -> None: + self._shared_state["calls"] = value + + @property + def last_config(self): + return self._shared_state["last_config"] + + @last_config.setter + def last_config(self, value) -> None: + self._shared_state["last_config"] = value + + def __copy__(self): + return _FakeProvider( + self.id, + self._behavior, + configured=self._configured, + models=list(self._models), + _shared_state=self._shared_state, + ) def configure(self, cfg): # pragma: no cover self.last_config = cfg @@ -56,6 +84,9 @@ async def chat(self, model_id: str, messages, **kwargs): if current == "timeout": await asyncio.sleep(0.05) return _FakeResponse("late") + if current == "slow": + await asyncio.sleep(5) + return _FakeResponse("late") return _FakeResponse(f"{self.id}:{model_id}") @@ -221,6 +252,28 @@ def test_llm_timeout_retries_then_raises(monkeypatch): assert provider.calls == 3 +def test_llm_ask_honors_cancel_checker(monkeypatch): + provider = _FakeProvider("demo", "slow", models=["m"]) + _patch_provider(monkeypatch, {"demo": provider}) + + cancel_event = threading.Event() + timer = threading.Timer(0.05, cancel_event.set) + started = time.perf_counter() + timer.start() + try: + client = LLMClient( + provider_id="demo", + model="m", + cancel_checker=cancel_event.is_set, + ) + with pytest.raises(RunCancelledError, match="Run cancelled"): + client.ask("hello") + finally: + timer.cancel() + + assert time.perf_counter() - started < 1.0 + + def test_llm_raises_clear_error_when_default_is_unavailable(monkeypatch): provider = _FakeProvider("fallback", "ok", configured=False, models=["fallback-model"]) _patch_provider(monkeypatch, {"fallback": provider}) diff --git a/tests/workspace/test_workspace_manager.py b/tests/workspace/test_workspace_manager.py index cbcee1e2a..87677bff9 100644 --- a/tests/workspace/test_workspace_manager.py +++ b/tests/workspace/test_workspace_manager.py @@ -151,6 +151,7 @@ class TestIsTextFile: ("notes.txt", True), ("server.log", True), ("config.json", True), + ("events.jsonl", True), ("settings.yaml", True), ("settings.yml", True), ("pyproject.toml", True), diff --git a/tests/workspace/test_workspace_routes.py b/tests/workspace/test_workspace_routes.py index 69b404d4b..5192ec606 100644 --- a/tests/workspace/test_workspace_routes.py +++ b/tests/workspace/test_workspace_routes.py @@ -344,6 +344,29 @@ def test_read_text_file(self, workspace_client): assert data["path"] == "outputs/note.md" assert data["content"] == "# Hello\nWorld" + def test_read_jsonl_file(self, workspace_client): + ws = _ws(workspace_client) + (ws / "outputs" / "events.jsonl").write_text('{"id": 1}\n{"id": 2}\n') + r = _client(workspace_client).get("/api/workspace/file?path=outputs/events.jsonl") + assert r.status_code == 200 + data = r.json() + assert data["path"] == "outputs/events.jsonl" + assert data["content"] == '{"id": 1}\n{"id": 2}\n' + assert data["truncated"] is False + + def test_read_large_text_file_returns_truncated_preview(self, workspace_client, monkeypatch): + monkeypatch.setenv("FLOCKS_WORKSPACE_MAX_READ_BYTES", "10") + ws = _ws(workspace_client) + (ws / "outputs" / "large.jsonl").write_text("0123456789ABCDEF", encoding="utf-8") + r = _client(workspace_client).get("/api/workspace/file?path=outputs/large.jsonl") + assert r.status_code == 200 + data = r.json() + assert data["path"] == "outputs/large.jsonl" + assert data["content"] == "0123456789" + assert data["truncated"] is True + assert data["size"] == 16 + assert data["preview_limit_bytes"] == 10 + def test_read_nonexistent_returns_404(self, workspace_client): r = _client(workspace_client).get("/api/workspace/file?path=ghost.txt") assert r.status_code == 404 @@ -545,6 +568,19 @@ def test_read_memory_file(self, workspace_client): data = r.json() assert data["path"] == "MEMORY.md" assert "Key facts" in data["content"] + assert data["truncated"] is False + + def test_read_large_memory_file_returns_truncated_preview(self, workspace_client, monkeypatch): + monkeypatch.setenv("FLOCKS_WORKSPACE_MAX_READ_BYTES", "8") + mem = _mem(workspace_client) + (mem / "large.md").write_text("abcdefghijk", encoding="utf-8") + r = _client(workspace_client).get("/api/workspace/memory/file?path=large.md") + assert r.status_code == 200 + data = r.json() + assert data["content"] == "abcdefgh" + assert data["truncated"] is True + assert data["size"] == 11 + assert data["preview_limit_bytes"] == 8 def test_read_memory_nonexistent_returns_404(self, workspace_client): r = _client(workspace_client).get("/api/workspace/memory/file?path=ghost.md") diff --git a/tui/flocks/acp/agent.ts b/tui/flocks/acp/agent.ts index ac41ddda5..8440bbcae 100644 --- a/tui/flocks/acp/agent.ts +++ b/tui/flocks/acp/agent.ts @@ -32,15 +32,15 @@ import { LoadAPIKeyError } from "ai" import type { Event, FlocksClient, SessionMessageResponse } from "@flocks-ai/sdk/v2" import { applyPatch } from "diff" -const LegacyTodoWriteOutput = z.array(Todo.Info) +const LegacyTodoEntriesOutput = z.array(Todo.Info) -function parseTodoWriteEntries(rawOutput: string, rawMetadata: unknown): Todo.Info[] | undefined { +function parseTodoEntries(rawOutput: string, rawMetadata: unknown): Todo.Info[] | undefined { try { const parsed = JSON.parse(rawOutput) const structured = Todo.WriteOutput.safeParse(parsed) if (structured.success) return structured.data.newTodos - const legacy = LegacyTodoWriteOutput.safeParse(parsed) + const legacy = LegacyTodoEntriesOutput.safeParse(parsed) if (legacy.success) return legacy.data } catch { // Fall through to metadata-based parsing below. @@ -52,10 +52,10 @@ function parseTodoWriteEntries(rawOutput: string, rawMetadata: unknown): Todo.In const structuredMetadata = Todo.WriteOutput.safeParse(metadata) if (structuredMetadata.success) return structuredMetadata.data.newTodos - const explicitNewTodos = LegacyTodoWriteOutput.safeParse(metadata["newTodos"]) + const explicitNewTodos = LegacyTodoEntriesOutput.safeParse(metadata["newTodos"]) if (explicitNewTodos.success) return explicitNewTodos.data - const legacyMetadata = LegacyTodoWriteOutput.safeParse(metadata["todos"]) + const legacyMetadata = LegacyTodoEntriesOutput.safeParse(metadata["todos"]) if (legacyMetadata.success) return legacyMetadata.data return undefined @@ -300,8 +300,8 @@ export namespace ACP { }) } - if (part.tool === "todowrite") { - const parsedTodos = parseTodoWriteEntries(part.state.output, part.state.metadata) + if (part.tool === "todo") { + const parsedTodos = parseTodoEntries(part.state.output, part.state.metadata) if (parsedTodos) { await this.connection .sessionUpdate({ @@ -637,15 +637,15 @@ export namespace ACP { }) } - if (part.tool === "todowrite") { - const parsedTodos = z.array(Todo.Info).safeParse(JSON.parse(part.state.output)) - if (parsedTodos.success) { + if (part.tool === "todo") { + const parsedTodos = parseTodoEntries(part.state.output, part.state.metadata) + if (parsedTodos) { await this.connection .sessionUpdate({ sessionId, update: { sessionUpdate: "plan", - entries: parsedTodos.data.map((todo) => { + entries: parsedTodos.map((todo) => { const status: PlanEntry["status"] = todo.status === "cancelled" ? "completed" : (todo.status as PlanEntry["status"]) return { diff --git a/tui/flocks/agent/agent.ts b/tui/flocks/agent/agent.ts index 8dafb44b6..9e47943f3 100644 --- a/tui/flocks/agent/agent.ts +++ b/tui/flocks/agent/agent.ts @@ -63,8 +63,6 @@ export namespace Agent { [Truncate.GLOB]: "allow", }, question: "deny", - plan_enter: "deny", - plan_exit: "deny", // mirrors github.com/github/gitignore Node.gitignore pattern for .env files read: { "*": "allow", @@ -100,7 +98,6 @@ export namespace Agent { defaults, PermissionNext.fromConfig({ question: "allow", - plan_enter: "allow", }), user, ), @@ -115,7 +112,6 @@ export namespace Agent { defaults, PermissionNext.fromConfig({ question: "allow", - plan_exit: "allow", external_directory: { [path.join(Global.Path.data, "plans", "*")]: "allow", }, diff --git a/tui/flocks/cli/cmd/agent.ts b/tui/flocks/cli/cmd/agent.ts index 6f2348594..8a018cea9 100644 --- a/tui/flocks/cli/cmd/agent.ts +++ b/tui/flocks/cli/cmd/agent.ts @@ -23,8 +23,7 @@ const AVAILABLE_TOOLS = [ "grep", "webfetch", "task", - "todowrite", - "todoread", + "todo", ] const AgentCreateCommand = cmd({ diff --git a/tui/flocks/cli/cmd/github.ts b/tui/flocks/cli/cmd/github.ts index 0e9eb9e01..809158794 100644 --- a/tui/flocks/cli/cmd/github.ts +++ b/tui/flocks/cli/cmd/github.ts @@ -815,8 +815,7 @@ export const GithubRunCommand = cmd({ function subscribeSessionEvents() { const TOOL: Record = { - todowrite: ["Todo", UI.Style.TEXT_WARNING_BOLD], - todoread: ["Todo", UI.Style.TEXT_WARNING_BOLD], + todo: ["Todo", UI.Style.TEXT_WARNING_BOLD], bash: ["Bash", UI.Style.TEXT_DANGER_BOLD], edit: ["Edit", UI.Style.TEXT_SUCCESS_BOLD], glob: ["Glob", UI.Style.TEXT_INFO_BOLD], diff --git a/tui/flocks/cli/cmd/run.ts b/tui/flocks/cli/cmd/run.ts index e1092317e..1724c96fc 100644 --- a/tui/flocks/cli/cmd/run.ts +++ b/tui/flocks/cli/cmd/run.ts @@ -13,8 +13,7 @@ import { Provider } from "../../provider/provider" import { Agent } from "../../agent/agent" const TOOL: Record = { - todowrite: ["Todo", UI.Style.TEXT_WARNING_BOLD], - todoread: ["Todo", UI.Style.TEXT_WARNING_BOLD], + todo: ["Todo", UI.Style.TEXT_WARNING_BOLD], bash: ["Bash", UI.Style.TEXT_DANGER_BOLD], edit: ["Edit", UI.Style.TEXT_SUCCESS_BOLD], glob: ["Glob", UI.Style.TEXT_INFO_BOLD], diff --git a/tui/flocks/cli/cmd/tui/context/local.tsx b/tui/flocks/cli/cmd/tui/context/local.tsx index fe642a22d..b0d569018 100644 --- a/tui/flocks/cli/cmd/tui/context/local.tsx +++ b/tui/flocks/cli/cmd/tui/context/local.tsx @@ -214,7 +214,7 @@ export const { use: useLocal, provider: LocalProvider } = createSimpleContext({ return { provider: "Connect a provider", model: "No provider selected", - reasoning: false, + reasoning: true, } } const provider = sync.data.provider.find((x) => x.id === value.providerID) @@ -222,7 +222,7 @@ export const { use: useLocal, provider: LocalProvider } = createSimpleContext({ return { provider: provider?.name ?? value.providerID, model: info?.name ?? value.modelID, - reasoning: info?.capabilities?.reasoning ?? false, + reasoning: info?.capabilities?.reasoning ?? true, } }), cycle(direction: 1 | -1) { diff --git a/tui/flocks/cli/cmd/tui/routes/session/index.tsx b/tui/flocks/cli/cmd/tui/routes/session/index.tsx index c6a45ef70..bade13c9d 100644 --- a/tui/flocks/cli/cmd/tui/routes/session/index.tsx +++ b/tui/flocks/cli/cmd/tui/routes/session/index.tsx @@ -35,7 +35,7 @@ import type { ReadTool } from "@/tool/read" import type { WriteTool } from "@/tool/write" import { BashTool } from "@/tool/bash" import type { GlobTool } from "@/tool/glob" -import { TodoWriteTool } from "@/tool/todo" +import { TodoTool } from "@/tool/todo" import type { GrepTool } from "@/tool/grep" import type { EditTool } from "@/tool/edit" import type { ApplyPatchTool } from "@/tool/apply_patch" @@ -196,23 +196,6 @@ export function Session() { } }) - let lastSwitch: string | undefined = undefined - sdk.event.on("message.part.updated", (evt) => { - const part = evt.properties.part - if (part.type !== "tool") return - if (part.sessionID !== route.sessionID) return - if (part.state.status !== "completed") return - if (part.id === lastSwitch) return - - if (part.tool === "plan_exit") { - local.agent.set("build") - lastSwitch = part.id - } else if (part.tool === "plan_enter") { - local.agent.set("plan") - lastSwitch = part.id - } - }) - let scroll: ScrollBoxRenderable let prompt: PromptRef const keybind = useKeybind() @@ -1534,8 +1517,8 @@ function ToolPart(props: { last: boolean; part: ToolPart; message: AssistantMess - - + + @@ -2184,7 +2167,7 @@ function ApplyPatch(props: ToolProps) { ) } -function TodoWrite(props: ToolProps) { +function TodoToolView(props: ToolProps) { const todos = () => props.metadata.newTodos ?? props.metadata.todos ?? props.input.todos ?? [] return ( diff --git a/tui/flocks/config/config.ts b/tui/flocks/config/config.ts index 159e9da68..12e9e5e55 100644 --- a/tui/flocks/config/config.ts +++ b/tui/flocks/config/config.ts @@ -157,14 +157,14 @@ export namespace Config { // Backwards compatibility: legacy top-level `tools` config if (result.tools) { - const perms: Record = {} + const perms: Record = {} for (const [tool, enabled] of Object.entries(result.tools)) { const action: Config.PermissionAction = enabled ? "allow" : "deny" if (tool === "write" || tool === "edit" || tool === "patch") { - perms.edit = action + assignPermission(perms, "edit", action) continue } - perms[tool] = action + assignPermission(perms, tool, action) } result.permission = mergeDeep(perms, result.permission ?? {}) } @@ -499,14 +499,28 @@ export namespace Config { return val } + const canonicalPermissionToolName = (tool: string) => { + if (tool === "todowrite" || tool === "todoread") return "todo" + return tool + } + + const assignPermission = (target: Record, tool: string, action: PermissionRule) => { + const canonical = canonicalPermissionToolName(tool) + if (target[canonical] === "deny" || action === "deny") { + target[canonical] = "deny" + return + } + if (!(canonical in target)) target[canonical] = action + } + const permissionTransform = (x: unknown): Record => { if (typeof x === "string") return { "*": x as PermissionAction } const obj = x as { __originalKeys?: string[] } & Record const { __originalKeys, ...rest } = obj - if (!__originalKeys) return rest as Record const result: Record = {} - for (const key of __originalKeys) { - if (key in rest) result[key] = rest[key] as PermissionRule + const keys = __originalKeys ?? Object.keys(rest) + for (const key of keys) { + if (key in rest) assignPermission(result, key, rest[key] as PermissionRule) } return result } @@ -524,8 +538,7 @@ export namespace Config { bash: PermissionRule.optional(), task: PermissionRule.optional(), external_directory: PermissionRule.optional(), - todowrite: PermissionAction.optional(), - todoread: PermissionAction.optional(), + todo: PermissionAction.optional(), question: PermissionAction.optional(), webfetch: PermissionAction.optional(), websearch: PermissionAction.optional(), @@ -611,9 +624,9 @@ export namespace Config { const action = enabled ? "allow" : "deny" // write, edit, patch all map to edit permission if (tool === "write" || tool === "edit" || tool === "patch") { - permission.edit = action + assignPermission(permission, "edit", action) } else { - permission[tool] = action + assignPermission(permission, tool, action) } } Object.assign(permission, agent.permission) diff --git a/tui/flocks/provider/provider.ts b/tui/flocks/provider/provider.ts index bebd884c2..618fccfa1 100644 --- a/tui/flocks/provider/provider.ts +++ b/tui/flocks/provider/provider.ts @@ -762,7 +762,7 @@ export namespace Provider { providerID, capabilities: { temperature: model.temperature ?? existingModel?.capabilities.temperature ?? false, - reasoning: model.reasoning ?? existingModel?.capabilities.reasoning ?? false, + reasoning: model.reasoning ?? existingModel?.capabilities.reasoning ?? true, attachment: model.attachment ?? existingModel?.capabilities.attachment ?? false, toolcall: model.tool_call ?? existingModel?.capabilities.toolcall ?? true, input: { diff --git a/tui/flocks/session/prompt.ts b/tui/flocks/session/prompt.ts index 6cbb9b990..befddd369 100644 --- a/tui/flocks/session/prompt.ts +++ b/tui/flocks/session/prompt.ts @@ -1380,11 +1380,10 @@ Goal: Write your final plan to the plan file (the only file you can edit). - Include the paths of critical files to be modified - Include a verification section describing how to test the changes end-to-end (run the code, use MCP tools, run tests) -### Phase 5: Call plan_exit tool -At the very end of your turn, once you have asked the user questions and are happy with your final plan file - you should always call plan_exit to indicate to the user that you are done planning. -This is critical - your turn should only end with either asking the user a question or calling plan_exit. Do not stop unless it's for these 2 reasons. +### Phase 5: Final response +At the very end of your turn, once you have asked the user questions and are happy with your final plan file, summarize the plan path and the recommended next implementation step. -**Important:** Use question tool to clarify requirements or approach details before you finish the plan. Once the plan is ready, call plan_exit to leave plan mode and hand control back to build mode. +**Important:** Use question tool to clarify requirements or approach details before you finish the plan. NOTE: At any point in time through this workflow you should feel free to ask the user questions or clarifications. Don't make large assumptions about user intent. The goal is to present a well researched plan to the user, and tie any loose ends before implementation begins. `, diff --git a/tui/flocks/session/prompt/anthropic-20250930.txt b/tui/flocks/session/prompt/anthropic-20250930.txt index cf81d71a9..676c4d8dc 100644 --- a/tui/flocks/session/prompt/anthropic-20250930.txt +++ b/tui/flocks/session/prompt/anthropic-20250930.txt @@ -72,7 +72,7 @@ For example, if the user asks you how to approach something, you should do your Prioritize technical accuracy and truthfulness over validating the user's beliefs. Focus on facts and problem-solving, providing direct, objective technical info without any unnecessary superlatives, praise, or emotional validation. It is best for the user if Claude honestly applies the same rigorous standards to all ideas and disagrees when necessary, even if it may not be what the user wants to hear. Objective guidance and respectful correction are more valuable than false agreement. Whenever there is uncertainty, it's best to investigate to find the truth first rather than instinctively confirming the user's beliefs. # Task Management -You have access to the TodoWrite tools to help you manage and plan tasks. Use these tools VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. +You have access to the `todo` tool to help you manage and plan tasks. Use `todo(action="write")` VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. These tools are also EXTREMELY helpful for planning tasks, and for breaking down larger complex tasks into smaller steps. If you do not use this tool when planning, you may forget to do important tasks - and that is unacceptable. It is critical that you mark todos as completed as soon as you are done with a task. Do not batch up multiple tasks before marking them as completed. @@ -81,13 +81,13 @@ Examples: user: Run the build and fix any type errors -assistant: I'm going to use the TodoWrite tool to write the following items to the todo list: +assistant: I'm going to use `todo(action="write")` to write the following items to the todo list: - Run the build - Fix any type errors I'm now going to run the build using Bash. -Looks like I found 10 type errors. I'm going to use the TodoWrite tool to write 10 items to the todo list. +Looks like I found 10 type errors. I'm going to use `todo(action="write")` to write 10 items to the todo list. marking the first todo as in_progress @@ -102,7 +102,7 @@ In the above example, the assistant completes all the tasks, including the 10 er user: Help me write a new feature that allows users to track their usage metrics and export them to various formats -assistant: I'll help you implement a usage metrics tracking and export feature. Let me first use the TodoWrite tool to plan this task. +assistant: I'll help you implement a usage metrics tracking and export feature. Let me first use `todo(action="write")` to plan this task. Adding the following todos to the todo list: 1. Research existing metrics tracking in the codebase 2. Design the metrics collection system @@ -123,7 +123,7 @@ Users may configure 'hooks', shell commands that execute in response to events l # Doing tasks The user will primarily request you perform software engineering tasks. This includes solving bugs, adding new functionality, refactoring code, explaining code, and more. For these tasks the following steps are recommended: -- Use the TodoWrite tool to plan the task if required +- Use `todo(action="write")` to plan the task if required - Tool results and user messages may include tags. tags contain useful information and reminders. They are automatically added by the system, and bear no direct relation to the specific tool results or user messages in which they appear. @@ -152,7 +152,7 @@ Assistant knowledge cutoff is January 2025. IMPORTANT: Assist with defensive security tasks only. Refuse to create, modify, or improve code that may be used maliciously. Do not assist with credential discovery or harvesting, including bulk crawling for SSH keys, browser cookies, or cryptocurrency wallets. Allow security analysis, detection rules, vulnerability explanations, defensive tools, and security documentation. -IMPORTANT: Always use the TodoWrite tool to plan and track tasks throughout the conversation. +IMPORTANT: Always use `todo(action="write")` to plan and track tasks throughout the conversation. # Code References diff --git a/tui/flocks/session/prompt/anthropic.txt b/tui/flocks/session/prompt/anthropic.txt index bf9a94c9e..7a0e5fd5c 100644 --- a/tui/flocks/session/prompt/anthropic.txt +++ b/tui/flocks/session/prompt/anthropic.txt @@ -18,7 +18,7 @@ If the user asks for help or wants to give feedback inform them of the following Prioritize technical accuracy and truthfulness over validating the user's beliefs. Focus on facts and problem-solving, providing direct, objective technical info without any unnecessary superlatives, praise, or emotional validation. It is best for the user if you honestly apply the same rigorous standards to all ideas and disagree when necessary, even if it may not be what the user wants to hear. Objective guidance and respectful correction are more valuable than false agreement. Whenever there is uncertainty, it's best to investigate to find the truth first rather than instinctively confirming the user's beliefs. # Task Management -You have access to the TodoWrite tools to help you manage and plan tasks. Use these tools VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. +You have access to the `todo` tool to help you manage and plan tasks. Use `todo(action="write")` VERY frequently to ensure that you are tracking your tasks and giving the user visibility into your progress. These tools are also EXTREMELY helpful for planning tasks, and for breaking down larger complex tasks into smaller steps. If you do not use this tool when planning, you may forget to do important tasks - and that is unacceptable. It is critical that you mark todos as completed as soon as you are done with a task. Do not batch up multiple tasks before marking them as completed. @@ -27,13 +27,13 @@ Examples: user: Run the build and fix any type errors -assistant: I'm going to use the TodoWrite tool to write the following items to the todo list: +assistant: I'm going to use `todo(action="write")` to write the following items to the todo list: - Run the build - Fix any type errors I'm now going to run the build using Bash. -Looks like I found 10 type errors. I'm going to use the TodoWrite tool to write 10 items to the todo list. +Looks like I found 10 type errors. I'm going to use `todo(action="write")` to write 10 items to the todo list. marking the first todo as in_progress @@ -47,7 +47,7 @@ In the above example, the assistant completes all the tasks, including the 10 er user: Help me write a new feature that allows users to track their usage metrics and export them to various formats -assistant: I'll help you implement a usage metrics tracking and export feature. Let me first use the TodoWrite tool to plan this task. +assistant: I'll help you implement a usage metrics tracking and export feature. Let me first use `todo(action="write")` to plan this task. Adding the following todos to the todo list: 1. Research existing metrics tracking in the codebase 2. Design the metrics collection system @@ -67,7 +67,7 @@ I've found some existing telemetry code. Let me mark the first todo as in_progre # Doing tasks The user will primarily request you perform SecOps tasks. This includes security analysis, threat detection, incident response, vulnerability assessment, automation, and more. For these tasks the following steps are recommended: - -- Use the TodoWrite tool to plan the task if required +- Use `todo(action="write")` to plan the task if required - Tool results and user messages may include tags. tags contain useful information and reminders. They are automatically added by the system, and bear no direct relation to the specific tool results or user messages in which they appear. @@ -89,7 +89,7 @@ user: What is the codebase structure? assistant: [Uses the Task tool] -IMPORTANT: Always use the TodoWrite tool to plan and track tasks throughout the conversation. +IMPORTANT: Always use `todo(action="write")` to plan and track tasks throughout the conversation. # Code References diff --git a/tui/flocks/session/prompt/plan-reminder-anthropic.txt b/tui/flocks/session/prompt/plan-reminder-anthropic.txt deleted file mode 100644 index 28f1e629d..000000000 --- a/tui/flocks/session/prompt/plan-reminder-anthropic.txt +++ /dev/null @@ -1,67 +0,0 @@ - -# Plan Mode - System Reminder - -Plan mode is active. The user indicated that they do not want you to execute yet -- you MUST NOT make any edits (with the exception of the plan file mentioned below), run any non-readonly tools (including changing configs or making commits), or otherwise make any changes to the system. This supersedes any other instructions you have received. - ---- - -## Plan File Info - -No plan file exists yet. You should create your plan at `/Users/aidencline/.claude/plans/happy-waddling-feigenbaum.md` using the Write tool. - -You should build your plan incrementally by writing to or editing this file. NOTE that this is the only file you are allowed to edit - other than this you are only allowed to take READ-ONLY actions. - -**Plan File Guidelines:** The plan file should contain only your final recommended approach, not all alternatives considered. Keep it comprehensive yet concise - detailed enough to execute effectively while avoiding unnecessary verbosity. - ---- - -## Enhanced Planning Workflow - -### Phase 1: Initial Understanding - -**Goal:** Gain a comprehensive understanding of the user's request by reading through code and asking them questions. Critical: In this phase you should only use the Explore subagent type. - -1. Understand the user's request thoroughly - -2. **Launch up to 3 Explore agents IN PARALLEL** (single message, multiple tool calls) to efficiently explore the codebase. Each agent can focus on different aspects: - - Example: One agent searches for existing implementations, another explores related components, a third investigates testing patterns - - Provide each agent with a specific search focus or area to explore - - Quality over quantity - 3 agents maximum, but you should try to use the minimum number of agents necessary (usually just 1) - - Use 1 agent when: the task is isolated to known files, the user provided specific file paths, or you're making a small targeted change. Use multiple agents when: the scope is uncertain, multiple areas of the codebase are involved, or you need to understand existing patterns before planning. - - Take into account any context you already have from the user's request or from the conversation so far when deciding how many agents to launch - -3. Use AskUserQuestion tool to clarify ambiguities in the user request up front. - -### Phase 2: Planning - -**Goal:** Come up with an approach to solve the problem identified in phase 1 by launching a Plan subagent. - -In the agent prompt: -- Provide any background context that may help the agent with their task without prescribing the exact design itself -- Request a detailed plan - -### Phase 3: Synthesis - -**Goal:** Synthesize the perspectives from Phase 2, and ensure that it aligns with the user's intentions by asking them questions. - -1. Collect all agent responses -2. Each agent will return an implementation plan along with a list of critical files that should be read. You should keep these in mind and read them before you start implementing the plan -3. Use AskUserQuestion to ask the users questions about trade offs. - -### Phase 4: Final Plan - -Once you have all the information you need, ensure that the plan file has been updated with your synthesized recommendation including: -- Recommended approach with rationale -- Key insights from different perspectives -- Critical files that need modification - -### Phase 5: Call ExitPlanMode - -At the very end of your turn, once you have asked the user questions and are happy with your final plan file - you should always call ExitPlanMode to indicate to the user that you are done planning. - -This is critical - your turn should only end with either asking the user a question or calling ExitPlanMode. Do not stop unless it's for these 2 reasons. - ---- - -**NOTE:** At any point in time through this workflow you should feel free to ask the user questions or clarifications. Don't make large assumptions about user intent. The goal is to present a well researched plan to the user, and tie any loose ends before implementation begins. - diff --git a/tui/flocks/tool/bash.txt b/tui/flocks/tool/bash.txt index 9fbc9fcf3..f42bd270a 100644 --- a/tui/flocks/tool/bash.txt +++ b/tui/flocks/tool/bash.txt @@ -81,7 +81,7 @@ Git Safety Protocol: Important notes: - NEVER run additional commands to read or explore code, besides git bash commands -- NEVER use the TodoWrite or Task tools +- NEVER use the todo or Task tools - DO NOT push to the remote repository unless the user explicitly asks you to do so - IMPORTANT: Never use git commands with the -i flag (like git rebase -i or git add -i) since they require interactive input which is not supported. - If there are no changes to commit (i.e., no untracked files and no modifications), do not create an empty commit @@ -108,7 +108,7 @@ gh pr create --title "the pr title" --body "$(cat <<'EOF' Important: -- DO NOT use the TodoWrite or Task tools +- DO NOT use the todo or Task tools - Return the PR URL when you're done, so the user can see it # Other common operations diff --git a/tui/flocks/tool/plan-enter.txt b/tui/flocks/tool/plan-enter.txt deleted file mode 100644 index 2e6a69f1f..000000000 --- a/tui/flocks/tool/plan-enter.txt +++ /dev/null @@ -1,14 +0,0 @@ -Use this tool to suggest switching to plan agent when the user's request would benefit from planning before implementation. - -If they explicitly mention wanting to create a plan ALWAYS call this tool first. - -This tool will ask the user if they want to switch to plan agent. - -Call this tool when: -- The user's request is complex and would benefit from planning first -- You want to research and design before making changes -- The task involves multiple files or significant architectural decisions - -Do NOT call this tool: -- For simple, straightforward tasks -- When the user explicitly wants immediate implementation diff --git a/tui/flocks/tool/plan-exit.txt b/tui/flocks/tool/plan-exit.txt deleted file mode 100644 index 94c3fa234..000000000 --- a/tui/flocks/tool/plan-exit.txt +++ /dev/null @@ -1,13 +0,0 @@ -Use this tool when you have completed the planning phase and are ready to exit plan agent. - -This tool immediately switches back to build agent so implementation can begin. - -Call this tool: -- After you have written a complete plan to the plan file -- After you have clarified any questions with the user -- When you are confident the plan is ready for implementation - -Do NOT call this tool: -- Before you have created or finalized the plan -- If you still have unanswered questions about the implementation -- If the user has indicated they want to continue planning diff --git a/tui/flocks/tool/plan.ts b/tui/flocks/tool/plan.ts deleted file mode 100644 index 69d960ab8..000000000 --- a/tui/flocks/tool/plan.ts +++ /dev/null @@ -1,111 +0,0 @@ -import z from "zod" -import path from "path" -import { Tool } from "./tool" -import { Question } from "../question" -import { Session } from "../session" -import { MessageV2 } from "../session/message-v2" -import { Identifier } from "../id/id" -import { Provider } from "../provider/provider" -import { Instance } from "../project/instance" -import EXIT_DESCRIPTION from "./plan-exit.txt" -import ENTER_DESCRIPTION from "./plan-enter.txt" - -async function getLastModel(sessionID: string) { - for await (const item of MessageV2.stream(sessionID)) { - if (item.info.role === "user" && item.info.model) return item.info.model - } - return Provider.defaultModel() -} - -export const PlanExitTool = Tool.define("plan_exit", { - description: EXIT_DESCRIPTION, - parameters: z.object({}), - async execute(_params, ctx) { - const session = await Session.get(ctx.sessionID) - const plan = path.relative(Instance.worktree, Session.plan(session)) - const model = await getLastModel(ctx.sessionID) - - const userMsg: MessageV2.User = { - id: Identifier.ascending("message"), - sessionID: ctx.sessionID, - role: "user", - time: { - created: Date.now(), - }, - agent: "build", - model, - } - await Session.updateMessage(userMsg) - await Session.updatePart({ - id: Identifier.ascending("part"), - messageID: userMsg.id, - sessionID: ctx.sessionID, - type: "text", - text: `The plan at ${plan} is complete. Switch back to build mode and execute it.`, - synthetic: true, - } satisfies MessageV2.TextPart) - - return { - title: "Switching to build agent", - output: "Exited plan mode and switched back to build agent. Continue by executing the plan.", - metadata: {}, - } - }, -}) - -export const PlanEnterTool = Tool.define("plan_enter", { - description: ENTER_DESCRIPTION, - parameters: z.object({}), - async execute(_params, ctx) { - const session = await Session.get(ctx.sessionID) - const plan = path.relative(Instance.worktree, Session.plan(session)) - - const answers = await Question.ask({ - sessionID: ctx.sessionID, - questions: [ - { - question: `Would you like to switch to the plan agent and create a plan saved to ${plan}?`, - header: "Plan Mode", - custom: false, - options: [ - { label: "Yes", description: "Switch to plan agent for research and planning" }, - { label: "No", description: "Stay with build agent to continue making changes" }, - ], - }, - ], - tool: ctx.callID ? { messageID: ctx.messageID, callID: ctx.callID } : undefined, - }) - - const answer = answers[0]?.[0] - - if (answer === "No") throw new Question.RejectedError() - - const model = await getLastModel(ctx.sessionID) - - const userMsg: MessageV2.User = { - id: Identifier.ascending("message"), - sessionID: ctx.sessionID, - role: "user", - time: { - created: Date.now(), - }, - agent: "plan", - model, - } - await Session.updateMessage(userMsg) - await Session.updatePart({ - id: Identifier.ascending("part"), - messageID: userMsg.id, - sessionID: ctx.sessionID, - type: "text", - text: "User has requested to enter plan mode. Switch to plan mode and begin planning.", - synthetic: true, - } satisfies MessageV2.TextPart) - - return { - title: "Switching to plan agent", - output: `User confirmed to switch to plan mode. A new message has been created to switch you to plan mode. The plan file will be at ${plan}. Begin planning.`, - metadata: {}, - } - }, -}) diff --git a/tui/flocks/tool/registry.ts b/tui/flocks/tool/registry.ts index 6b8c15253..4f4eed7a0 100644 --- a/tui/flocks/tool/registry.ts +++ b/tui/flocks/tool/registry.ts @@ -5,7 +5,7 @@ import { GlobTool } from "./glob" import { GrepTool } from "./grep" import { ReadTool } from "./read" import { TaskTool } from "./task" -import { TodoWriteTool, TodoReadTool } from "./todo" +import { TodoTool } from "./todo" import { WebFetchTool } from "./webfetch" import { WriteTool } from "./write" import { InvalidTool } from "./invalid" @@ -23,7 +23,6 @@ import { Flag } from "@/flag/flag" import { Log } from "@/util/log" import { LspTool } from "./lsp" import { Truncate } from "./truncation" -import { PlanExitTool, PlanEnterTool } from "./plan" import { ApplyPatchTool } from "./apply_patch" export namespace ToolRegistry { @@ -102,13 +101,11 @@ export namespace ToolRegistry { WriteTool, TaskTool, WebFetchTool, - TodoWriteTool, - TodoReadTool, + TodoTool, WebSearchTool, SkillTool, ApplyPatchTool, ...(Flag.FLOCKS_EXPERIMENTAL_LSP_TOOL ? [LspTool] : []), - ...(Flag.FLOCKS_EXPERIMENTAL_PLAN_MODE && Flag.FLOCKS_CLIENT === "cli" ? [PlanExitTool, PlanEnterTool] : []), ...custom, ] } @@ -140,7 +137,7 @@ export namespace ToolRegistry { if (t.id === "edit" || t.id === "write") return !usePatch // omit todo tools for openai models - if (t.id === "todoread" || t.id === "todowrite") { + if (t.id === "todo") { if (model.modelID.includes("gpt-")) return false } diff --git a/tui/flocks/tool/task.ts b/tui/flocks/tool/task.ts index 170d44480..f98316b39 100644 --- a/tui/flocks/tool/task.ts +++ b/tui/flocks/tool/task.ts @@ -70,12 +70,7 @@ export const TaskTool = Tool.define("task", async (ctx) => { title: params.description + ` (@${agent.name} subagent)`, permission: [ { - permission: "todowrite", - pattern: "*", - action: "deny", - }, - { - permission: "todoread", + permission: "todo", pattern: "*", action: "deny", }, @@ -151,8 +146,7 @@ export const TaskTool = Tool.define("task", async (ctx) => { }, agent: agent.name, tools: { - todowrite: false, - todoread: false, + todo: false, ...(hasTaskPermission ? {} : { task: false }), ...Object.fromEntries((config.experimental?.primary_tools ?? []).map((t) => [t, false])), }, diff --git a/tui/flocks/tool/todo.ts b/tui/flocks/tool/todo.ts index 185fa8262..554a11557 100644 --- a/tui/flocks/tool/todo.ts +++ b/tui/flocks/tool/todo.ts @@ -1,6 +1,6 @@ import z from "zod" import { Tool } from "./tool" -import DESCRIPTION_WRITE from "./todowrite.txt" +import DESCRIPTION from "./todo.txt" import { Todo } from "../session/todo" const ACTIVE_TODO_STATUSES = new Set(["pending", "in_progress"]) @@ -19,19 +19,36 @@ function verificationNudgeNeeded(todos: Todo.Info[]) { }) } -export const TodoWriteTool = Tool.define("todowrite", { - description: DESCRIPTION_WRITE, +export const TodoTool = Tool.define("todo", { + description: DESCRIPTION, parameters: z.object({ - todos: z.array(z.object(Todo.Info.shape)).describe("The updated todo list"), + action: z.enum(["read", "write"]).describe("Read the current todos or write the full todo list"), + todos: z.array(z.object(Todo.Info.shape)).optional().describe("For action=write: the updated todo list"), }), async execute(params, ctx) { await ctx.ask({ - permission: "todowrite", + permission: "todo", patterns: ["*"], always: ["*"], metadata: {}, }) + if (params.action === "read") { + const todos = await Todo.get(ctx.sessionID) + return { + title: `${todos.filter((x) => x.status !== "completed").length} todos`, + metadata: { + action: "read", + todos, + }, + output: JSON.stringify(todos, null, 2), + } + } + + if (!params.todos) { + throw new Error("todos is required when action='write'") + } + const oldTodos = await Todo.get(ctx.sessionID) const newTodos = params.todos.map((todo) => ({ ...todo, @@ -53,6 +70,7 @@ export const TodoWriteTool = Tool.define("todowrite", { title: `${newTodos.filter((x) => ACTIVE_TODO_STATUSES.has(x.status)).length} todos`, output: JSON.stringify(output, null, 2), metadata: { + action: "write", todos: newTodos, oldTodos, newTodos, @@ -61,25 +79,3 @@ export const TodoWriteTool = Tool.define("todowrite", { } }, }) - -export const TodoReadTool = Tool.define("todoread", { - description: "Use this tool to read your todo list", - parameters: z.object({}), - async execute(_params, ctx) { - await ctx.ask({ - permission: "todoread", - patterns: ["*"], - always: ["*"], - metadata: {}, - }) - - const todos = await Todo.get(ctx.sessionID) - return { - title: `${todos.filter((x) => x.status !== "completed").length} todos`, - metadata: { - todos, - }, - output: JSON.stringify(todos, null, 2), - } - }, -}) diff --git a/tui/flocks/tool/todo.txt b/tui/flocks/tool/todo.txt new file mode 100644 index 000000000..356953bfe --- /dev/null +++ b/tui/flocks/tool/todo.txt @@ -0,0 +1,28 @@ +Use this tool to read or manage a structured task list for your current SecOps session. This helps track progress, organize complex tasks, and keep the user informed. + +Use action="read" to inspect the current todo list. + +Use action="write" with a full structured todos array to replace the current todo list. + +When to use: +1. Complex multi-step tasks with 3 or more distinct steps +2. Non-trivial tasks requiring careful planning +3. User explicitly requests a todo list +4. User provides multiple tasks + +When not to use: +1. Single, straightforward tasks +2. Trivial tasks with no organizational benefit +3. Purely conversational or informational requests + +Task states: +- pending: Not yet started +- in_progress: Currently working on this task +- completed: Finished successfully +- cancelled: No longer needed + +Guidelines: +- Create specific, actionable items +- Update status in real time as you work +- Mark tasks completed immediately after finishing +- Only keep one task in_progress at a time diff --git a/tui/flocks/tool/todowrite.txt b/tui/flocks/tool/todowrite.txt deleted file mode 100644 index 1744a8f3d..000000000 --- a/tui/flocks/tool/todowrite.txt +++ /dev/null @@ -1,167 +0,0 @@ -Use this tool to create and manage a structured task list for your current SecOps session. This helps you track progress, organize complex tasks, and demonstrate thoroughness to the user. -It also helps the user understand the progress of the task and overall progress of their requests. - -## When to Use This Tool -Use this tool proactively in these scenarios: - -1. Complex multistep tasks - When a task requires 3 or more distinct steps or actions -2. Non-trivial and complex tasks - Tasks that require careful planning or multiple operations -3. User explicitly requests todo list - When the user directly asks you to use the todo list -4. User provides multiple tasks - When users provide a list of things to be done (numbered or comma-separated) -5. After receiving new instructions - Immediately capture user requirements as todos. Feel free to edit the todo list based on new information. -6. After completing a task - Mark it complete and add any new follow-up tasks -7. When you start working on a new task, mark the todo as in_progress. Ideally you should only have one todo as in_progress at a time. Complete existing tasks before starting new ones. - -## When NOT to Use This Tool - -Skip using this tool when: -1. There is only a single, straightforward task -2. The task is trivial and tracking it provides no organizational benefit -3. The task can be completed in less than 3 trivial steps -4. The task is purely conversational or informational - -NOTE that you should not use this tool if there is only one trivial task to do. In this case you are better off just doing the task directly. - -## Examples of When to Use the Todo List - - -User: I want to add a dark mode toggle to the application settings. Make sure you run the tests and build when you're done! -Assistant: I'll help add a dark mode toggle to your application settings. Let me create a todo list to track this implementation. -*Creates todo list with the following items:* -1. Create dark mode toggle component in Settings page -2. Add dark mode state management (context/store) -3. Implement CSS-in-JS styles for dark theme -4. Update existing components to support theme switching -5. Run tests and build process, addressing any failures or errors that occur -*Begins working on the first task* - - -The assistant used the todo list because: -1. Adding dark mode is a multi-step feature requiring UI, state management, and styling changes -2. The user explicitly requested tests and build be run afterward -3. The assistant inferred that tests and build need to pass by adding "Ensure tests and build succeed" as the final task - - - - -User: Help me rename the function getCwd to getCurrentWorkingDirectory across my project -Assistant: Let me first search through your codebase to find all occurrences of 'getCwd'. -*Uses grep or search tools to locate all instances of getCwd in the codebase* -Assistant: I've found 15 instances of 'getCwd' across 8 different files. Let me create a todo list to track these changes. -*Creates todo list with specific items for each file that needs updating* - - -The assistant used the todo list because: -1. First, the assistant searched to understand the scope of the task -2. Upon finding multiple occurrences across different files, it determined this was a complex task with multiple steps -3. The todo list helps ensure every instance is tracked and updated systematically -4. This approach prevents missing any occurrences and maintains code consistency - - - - - -User: I need to implement these features for my e-commerce site: user registration, product catalog, shopping cart, and checkout flow. -Assistant: I'll help implement these features. First, let's add all the features to the todo list. -*Creates a todo list breaking down each feature into specific tasks based on the project architecture* -Assistant: Let's start with user registration. This will involve creating database models, API endpoints, and frontend forms that integrate with your existing authentication system. - - -The assistant used the todo list because: -1. The user provided multiple complex features to implement in a comma separated list -2. The todo list helps organize these large features into manageable tasks -3. This approach allows for tracking progress across the entire implementation - - - - -User: Can you help optimize my React application? It's rendering slowly and has performance issues. -Assistant: I'll help optimize your React application. First, let me examine your codebase to identify potential performance bottlenecks. -*Reviews component structure, render patterns, state management, and data fetching* -Assistant: After analyzing your codebase, I've identified several performance issues. Let me create a todo list to track our optimization efforts. -*Creates todo list with items like: 1) Implement memoization for expensive calculations in ProductList, 2) Add virtualization for long lists in Dashboard, 3) Optimize image loading in Gallery component, 4) Fix state update loops in ShoppingCart, 5) Review bundle size and implement code splitting* -Let's start by implementing memoization for the expensive calculations in your ProductList component. - - -The assistant used the todo list because: -1. First, the assistant examined the codebase to identify specific performance issues -2. Based on this analysis, it identified multiple optimization opportunities -3. Performance optimization is a non-trivial task requiring multiple steps -4. The todo list helps methodically track improvements across different components -5. This systematic approach ensures all performance bottlenecks are addressed - - - -## Examples of When NOT to Use the Todo List - - -User: How do I print 'Hello World' in Python? -Assistant: In Python, you can print "Hello World" with this simple code: - -python -print("Hello World") - -This will output the text "Hello World" to the console when executed. - - -The assistant did not use the todo list because this is a single, trivial task that can be completed in one step. There's no need to track multiple tasks or steps for such a straightforward request. - - - - -User: What does the git status command do? -Assistant: The git status command shows the current state of your working directory and staging area. It displays which changes have been staged, which haven't, and which files aren't being tracked by Git. This is a fundamental command for understanding the current state of your repository before making commits. - - -The assistant did not use the todo list because this is an informational request with no actual coding task to complete. The user is simply asking for an explanation, not for the assistant to perform multiple steps or tasks. - - - - -User: Can you add a comment to the calculateTotal function to explain what it does? -Assistant: Sure, let me add a comment to the calculateTotal function to explain what it does. -* Uses the Edit tool to add a comment to the calculateTotal function * - - -The assistant did not use the todo list because this is a single, straightforward task confined to one location in the code. Adding a comment doesn't require tracking multiple steps or systematic organization. - - - - -User: Run npm install for me and tell me what happens. -Assistant: I'll run the npm install command for you. - -*Executes: npm install* - -The command completed successfully. Here's the output: -[Output of npm install command] - -All dependencies have been installed according to your package.json file. - - -The assistant did not use the todo list because this is a single command execution with immediate results. There are no multiple steps to track or organize, making the todo list unnecessary for this straightforward task. - - - -## Task States and Management - -1. **Task States**: Use these states to track progress: - - pending: Task not yet started - - in_progress: Currently working on (limit to ONE task at a time) - - completed: Task finished successfully - - cancelled: Task no longer needed - -2. **Task Management**: - - Update task status in real-time as you work - - Mark tasks complete IMMEDIATELY after finishing (don't batch completions) - - Only have ONE task in_progress at any time - - Complete current tasks before starting new ones - - Cancel tasks that become irrelevant - -3. **Task Breakdown**: - - Create specific, actionable items - - Break complex tasks into smaller, manageable steps - - Use clear, descriptive task names - -When in doubt, use this tool. Being proactive with task management demonstrates attentiveness and ensures you complete all requirements successfully. - diff --git a/tui/flocks/util/log.ts b/tui/flocks/util/log.ts index 6941310bb..507afe60e 100644 --- a/tui/flocks/util/log.ts +++ b/tui/flocks/util/log.ts @@ -47,44 +47,70 @@ export namespace Log { } let logpath = "" + let errorLogpath = "" + let logDate = "" export function file() { return logpath } - let write = (msg: any) => { + let write = async (msg: any, error = false) => { process.stderr.write(msg) return msg.length } export async function init(options: Options) { if (options.level) level = options.level - cleanup(Global.Path.log) + await cleanup(Global.Path.log) if (options.print) return - logpath = path.join( - Global.Path.log, - options.dev ? "dev.log" : new Date().toISOString().split(".")[0].replace(/:/g, "") + ".log", - ) - const logfile = Bun.file(logpath) - await fs.truncate(logpath).catch(() => {}) - const writer = logfile.writer() - write = async (msg: any) => { - const num = writer.write(msg) - writer.flush() - return num + await openDailyLogs() + write = async (msg: any, error = false) => { + await ensureCurrentDay() + await fs.appendFile(logpath, msg) + if (error) await fs.appendFile(errorLogpath, msg) + return msg.length } } + function todayString() { + return new Date().toISOString().split("T")[0] + } + + async function openDailyLogs() { + logDate = todayString() + const dir = path.join(Global.Path.log, logDate) + await fs.mkdir(dir, { recursive: true }) + logpath = path.join(dir, "flocks.log") + errorLogpath = path.join(dir, "errors.log") + } + + async function ensureCurrentDay() { + if (logDate === todayString()) return + await openDailyLogs() + await cleanup(Global.Path.log) + } + async function cleanup(dir: string) { - const glob = new Bun.Glob("????-??-??T??????.log") - const files = await Array.fromAsync( - glob.scan({ - cwd: dir, - absolute: true, + const retentionDays = Number.parseInt(process.env.FLOCKS_LOG_RETENTION_DAYS || "30", 10) + if (!Number.isFinite(retentionDays) || retentionDays <= 0) return + const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000 + const cutoffDay = new Date(cutoff).toISOString().split("T")[0] + const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []) + await Promise.all( + entries.map(async (entry) => { + const target = path.join(dir, entry.name) + if (entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/.test(entry.name)) { + if (entry.name < cutoffDay) { + await fs.rm(target, { recursive: true, force: true }).catch(() => {}) + } + return + } + if (entry.isFile() && /^\d{4}-\d{2}-\d{2}T\d{6}\.log(\.\d+)?$/.test(entry.name)) { + const stamp = entry.name.split(".log")[0] + if (new Date(stamp.replace(/T(\d{2})(\d{2})(\d{2})$/, "T$1:$2:$3")).getTime() < cutoff) { + await fs.unlink(target).catch(() => {}) + } + } }), ) - if (files.length <= 5) return - - const filesToDelete = files.slice(0, -10) - await Promise.all(filesToDelete.map((file) => fs.unlink(file).catch(() => {}))) } function formatError(error: Error, depth = 0): string { @@ -137,12 +163,12 @@ export namespace Log { }, error(message?: any, extra?: Record) { if (shouldLog("ERROR")) { - write("ERROR " + build(message, extra)) + write("ERROR " + build(message, extra), true) } }, warn(message?: any, extra?: Record) { if (shouldLog("WARN")) { - write("WARN " + build(message, extra)) + write("WARN " + build(message, extra), true) } }, tag(key: string, value: string) { diff --git a/tui/sdk/v2/gen/types.gen.ts b/tui/sdk/v2/gen/types.gen.ts index 189bfa0ed..77f869dc6 100644 --- a/tui/sdk/v2/gen/types.gen.ts +++ b/tui/sdk/v2/gen/types.gen.ts @@ -1345,8 +1345,7 @@ export type PermissionConfig = bash?: PermissionRuleConfig task?: PermissionRuleConfig external_directory?: PermissionRuleConfig - todowrite?: PermissionActionConfig - todoread?: PermissionActionConfig + todo?: PermissionActionConfig question?: PermissionActionConfig webfetch?: PermissionActionConfig websearch?: PermissionActionConfig diff --git a/uv.lock b/uv.lock index 9adb6613b..7a104edaf 100644 --- a/uv.lock +++ b/uv.lock @@ -537,7 +537,7 @@ wheels = [ [[package]] name = "flocks" -version = "2026.6.4" +version = "2026.6.10" source = { editable = "." } dependencies = [ { name = "aiofiles" }, @@ -583,6 +583,7 @@ dependencies = [ { name = "rich" }, { name = "sqlalchemy" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "striprtf" }, { name = "tiktoken" }, { name = "toml" }, @@ -650,6 +651,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.7.0" }, { name = "sqlalchemy", specifier = ">=2.0.25" }, { name = "sse-starlette", specifier = ">=1.8.2" }, + { name = "starlette", specifier = ">=1.0.1" }, { name = "striprtf", specifier = ">=0.0.26" }, { name = "tiktoken", specifier = ">=0.12.0" }, { name = "toml", specifier = ">=0.10.2" }, @@ -2173,15 +2175,15 @@ wheels = [ [[package]] name = "starlette" -version = "1.0.0" +version = "1.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149", size = 2655289, upload-time = "2026-03-22T18:29:46.779Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", size = 2701854, upload-time = "2026-05-31T01:07:51.847Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, + { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, ] [[package]] diff --git a/webui/src/api/agent.ts b/webui/src/api/agent.ts index 4475825de..b68dfaeb1 100644 --- a/webui/src/api/agent.ts +++ b/webui/src/api/agent.ts @@ -2,6 +2,8 @@ import client from './client'; export interface Agent { name: string; + /** Chinese display name; canonical `name` remains the stable identifier. */ + nameCn?: string; description?: string; /** Chinese UI label; English \`description\` is used for delegation/tooling. */ descriptionCn?: string; @@ -53,6 +55,7 @@ export const agentAPI = { create: (data: { name: string; + nameCn?: string; description?: string; descriptionCn?: string; prompt: string; @@ -67,6 +70,7 @@ export const agentAPI = { client.post('/api/agent', data), update: (name: string, data: { + nameCn?: string; description?: string; descriptionCn?: string; prompt?: string; diff --git a/webui/src/api/device.ts b/webui/src/api/device.ts index 588bf1a7f..ad0ba1e26 100644 --- a/webui/src/api/device.ts +++ b/webui/src/api/device.ts @@ -1,4 +1,5 @@ import client from './client'; +import type { APIServiceCredentialField } from '@/types'; // --------------------------------------------------------------------------- // Groups (机房) — the current product locks this to a single default room @@ -53,6 +54,14 @@ export interface DeviceIntegration { updated_at: number; } +export interface DeviceCredentialResponse { + fields: Record; +} + +export interface DeviceCredentialRevealRequest { + field?: string; +} + export interface DeviceIntegrationCreate { name: string; storage_key: string; @@ -89,6 +98,44 @@ export interface DeviceTestRequest { verify_ssl?: boolean; } +export interface DeviceTemplate { + plugin_id: string; + storage_key: string; + service_id: string; + name: string; + version?: string | null; + vendor?: string | null; + description?: string | null; + description_cn?: string | null; + credential_schema: APIServiceCredentialField[]; + tool_count: number; + installed: boolean; + state: 'available' | 'installed' | 'updateAvailable' | 'localOnly' | 'broken'; + source: 'bundled' | 'project' | 'global'; +} + +export interface CustomDeviceTemplateCreate { + plugin_id: string; + name: string; + vendor?: string; + service_id: string; + version?: string; + description?: string; + description_cn?: string; + credential_fields: APIServiceCredentialField[]; + tools: Array<{ + name: string; + description: string; + description_cn?: string; + category?: string; + inputSchema?: Record; + parameters?: Array>; + handler: Record; + response?: Record; + requires_confirmation?: boolean; + }>; +} + // --------------------------------------------------------------------------- // Per-device tool settings // --------------------------------------------------------------------------- @@ -120,12 +167,23 @@ export const deviceAPI = { client.delete(`/api/devices/groups/${id}`), // devices - list: (params?: { group_id?: string }) => + list: (params?: { group_id?: string; refresh?: boolean }) => client.get('/api/devices', { params }), + listTemplates: (params?: { refresh?: boolean }) => + client.get('/api/devices/templates', { params }), + + createCustomTemplate: (data: CustomDeviceTemplateCreate) => + client.post('/api/devices/templates/custom', data), + get: (id: string) => client.get(`/api/devices/${id}`), + revealCredentials: (id: string, field?: string) => { + const body: DeviceCredentialRevealRequest = field ? { field } : {}; + return client.post(`/api/devices/${id}/credentials`, body); + }, + create: (data: DeviceIntegrationCreate) => client.post('/api/devices', data), diff --git a/webui/src/api/session.ts b/webui/src/api/session.ts index 26193973e..9565d4f82 100644 --- a/webui/src/api/session.ts +++ b/webui/src/api/session.ts @@ -85,7 +85,7 @@ export const sessionApi = { /** * 更新会话 */ - update: async (sessionId: string, data: { title?: string }) => { + update: async (sessionId: string, data: { title?: string; provider?: string; model?: string; model_pinned?: boolean }) => { const response = await client.patch(`/api/session/${sessionId}`, data); return response.data; }, diff --git a/webui/src/api/stats.ts b/webui/src/api/stats.ts index 3b69e726e..bdd63c2d9 100644 --- a/webui/src/api/stats.ts +++ b/webui/src/api/stats.ts @@ -43,7 +43,7 @@ export const statsApi = { const agentList = Array.isArray(agents.data) ? agents.data : []; const workflowList = Array.isArray(workflows.data) ? workflows.data : []; // Exclude `system` category skills so the count matches the Skills page, - // which hides system skills (e.g. find-skills, onboarding) from the user. + // which hides system skills (e.g. onboarding) from the user. const skillList = (Array.isArray(skills.data) ? skills.data : []).filter( (s: any) => s?.category !== 'system' ); diff --git a/webui/src/api/userDefinedPages.ts b/webui/src/api/userDefinedPages.ts new file mode 100644 index 000000000..621aec313 --- /dev/null +++ b/webui/src/api/userDefinedPages.ts @@ -0,0 +1,73 @@ +import client from './client'; + +export interface UserDefinedPageListItem { + id: string; + title: string; + route: string; + icon: string; + order: number; + enabled: boolean; + placement: string; + buildHash: string; + buildStatus: 'idle' | 'building' | 'ready' | 'failed'; +} + +export interface UserDefinedPageManifest { + id: string; + title: string; + route: string; + icon: string; + order: number; + enabled: boolean; + placement: string; + entry: string; + updatedAt: number; +} + +export interface UserDefinedPageBuildMeta { + hash: string; + builtAt: number; + status: 'idle' | 'building' | 'ready' | 'failed'; + error?: string | null; +} + +export interface UserDefinedPageDetail { + manifest: UserDefinedPageManifest; + build: UserDefinedPageBuildMeta; + sourceFiles: string[]; +} + +export interface UserDefinedPageCreateRequest { + id: string; + title: string; + icon?: string; + order?: number; +} + +export interface UserDefinedPageSaveRequest { + manifest?: Partial; + sourcePath?: string; + sourceContent?: string; +} + +export const userDefinedPagesAPI = { + list: (enabledOnly = false) => + client.get('/api/user-defined-pages', { + params: enabledOnly ? { enabledOnly: true } : undefined, + }), + + create: (payload: UserDefinedPageCreateRequest) => + client.post('/api/user-defined-pages', payload), + + get: (pageId: string) => + client.get(`/api/user-defined-pages/${pageId}`), + + save: (pageId: string, payload: UserDefinedPageSaveRequest) => + client.put<{ manifest: UserDefinedPageManifest; build: UserDefinedPageBuildMeta }>( + `/api/user-defined-pages/${pageId}`, + payload, + ), + + build: (pageId: string) => + client.post(`/api/user-defined-pages/${pageId}/build`), +}; diff --git a/webui/src/api/workflow.ts b/webui/src/api/workflow.ts index 301e368a8..8d4eecd48 100644 --- a/webui/src/api/workflow.ts +++ b/webui/src/api/workflow.ts @@ -18,6 +18,8 @@ export interface WorkflowNode { select_key?: string; join?: boolean; join_mode?: 'flat' | 'namespace'; + join_conflict?: 'overwrite' | 'error'; + join_namespace_key?: string; // tool node tool_name?: string; tool_args?: Record; @@ -65,12 +67,106 @@ export interface WorkflowMetadata { [key: string]: any; } +export type WorkflowTriggerType = + | 'manual' + | 'schedule' + | 'webhook' + | 'syslog' + | 'kafka' + | 'internal_event' + | 'custom_webhook' + | 'custom_adapter' + | 'plugin'; + +export interface WorkflowTriggerAuth { + type?: string; + secretRef?: string; + headerName?: string; + queryParam?: string; + apiKey?: string; + [key: string]: any; +} + +export interface WorkflowTriggerFilter { + expr?: string; + mode?: string; + path?: string; + equals?: unknown; + [key: string]: any; +} + +export interface WorkflowTriggerConcurrency { + policy?: 'allow' | 'no_overlap' | 'queue' | 'drop_oldest' | 'drop_newest'; + maxParallel?: number; + queueSize?: number; + [key: string]: any; +} + +export interface WorkflowTriggerSample { + name: string; + payload?: unknown; + headers?: Record; + query?: Record; + [key: string]: any; +} + +export interface WorkflowTrigger { + id: string; + name?: string; + type: WorkflowTriggerType; + enabled?: boolean; + description?: string; + source?: Record; + auth?: WorkflowTriggerAuth; + filter?: WorkflowTriggerFilter; + mapping?: Record; + inputs?: Record; + concurrency?: WorkflowTriggerConcurrency; + runtime?: Record; + testSamples?: WorkflowTriggerSample[]; + updatedAt?: number; + [key: string]: any; +} + +export interface WorkflowTriggerStatus { + workflowId?: string; + triggerId?: string; + triggerType?: WorkflowTriggerType | string; + state: string; + error?: string | null; + [key: string]: any; +} + +export interface WorkflowTriggerRecord { + trigger: WorkflowTrigger; + status?: WorkflowTriggerStatus; +} + +export interface WorkflowTriggerPreview { + triggerId: string; + triggerType: string; + matched: boolean; + inputs: Record; + filterError?: string | null; +} + +export interface WorkflowTriggerPlugin { + id: string; + name: string; + description?: string; + root?: string; + manifestPath?: string; + handlerPath?: string; + manifest?: Record; +} + export interface WorkflowJSON { version?: string; name?: string; start: string; nodes: WorkflowNode[]; edges: WorkflowEdge[]; + triggers?: WorkflowTrigger[]; metadata?: WorkflowMetadata; } @@ -120,6 +216,11 @@ export interface WorkflowExecution { duration?: number; executionLog: WorkflowExecutionStep[]; errorMessage?: string; + triggerId?: string; + triggerType?: string; + deliveryId?: string; + attempt?: number; + triggerSource?: string; currentNodeId?: string; currentNodeType?: string; currentPhase?: string; @@ -214,6 +315,7 @@ export interface WorkflowPollerStatus { error?: string | null; enabled?: boolean; intervalSeconds?: number; + cronExpression?: string | null; timeoutSeconds?: number; noOverlap?: boolean; activeRuns?: number; @@ -268,7 +370,7 @@ export const workflowAPI = { validate: (id: string) => client.post<{ valid: boolean; issues: any[] }>(`/api/workflow/${id}/validate`), - getHistory: (id: string, params?: { limit?: number }) => + getHistory: (id: string, params?: { limit?: number; triggerId?: string; triggerType?: string }) => client.get(`/api/workflow/${id}/history`, { params }), getExecution: (workflowId: string, execId: string) => @@ -303,6 +405,44 @@ export const workflowAPI = { listServices: () => client.get('/api/workflow-services'), + getTriggers: (id: string) => + client.get(`/api/workflow/${id}/triggers`), + + createTrigger: (id: string, trigger: WorkflowTrigger) => + client.post<{ trigger: WorkflowTrigger; status?: WorkflowTriggerStatus }>( + `/api/workflow/${id}/triggers`, + trigger, + ), + + updateTrigger: (id: string, triggerId: string, trigger: WorkflowTrigger) => + client.put<{ trigger: WorkflowTrigger; status?: WorkflowTriggerStatus }>( + `/api/workflow/${id}/triggers/${triggerId}`, + trigger, + ), + + deleteTrigger: (id: string, triggerId: string) => + client.delete<{ ok: boolean; triggerId: string }>(`/api/workflow/${id}/triggers/${triggerId}`), + + getTriggerStatus: (id: string, triggerId: string) => + client.get(`/api/workflow/${id}/triggers/${triggerId}/status`), + + previewTriggerMapping: ( + id: string, + triggerId: string, + payload: { body?: unknown; headers?: Record; query?: Record; pathParams?: Record }, + ) => + client.post(`/api/workflow/${id}/triggers/${triggerId}/preview-mapping`, payload), + + testTrigger: ( + id: string, + triggerId: string, + payload: { body?: unknown; headers?: Record; query?: Record; pathParams?: Record }, + ) => + client.post>(`/api/workflow/${id}/triggers/${triggerId}/test`, payload), + + listTriggerPlugins: () => + client.get('/api/workflow-trigger-plugins'), + saveKafkaConfig: (id: string, config: { enabled?: boolean; inputBroker?: string; diff --git a/webui/src/api/workspace.ts b/webui/src/api/workspace.ts index 3a1299725..d8f3d712d 100644 --- a/webui/src/api/workspace.ts +++ b/webui/src/api/workspace.ts @@ -30,6 +30,14 @@ export interface UploadResult { error?: string; } +export interface WorkspaceFileContentResponse { + path: string; + content: string; + truncated?: boolean; + size?: number; + preview_limit_bytes?: number; +} + export type UploadPurpose = 'chat'; // ─── API ─────────────────────────────────────────────────────────────────── @@ -66,7 +74,7 @@ export const workspaceAPI = { }, readFile: (path: string) => - client.get<{ path: string; content: string }>('/api/workspace/file', { params: { path } }), + client.get('/api/workspace/file', { params: { path } }), writeFile: (path: string, content: string) => client.put<{ path: string; written: boolean }>('/api/workspace/file', { path, content }), @@ -92,7 +100,7 @@ export const workspaceAPI = { client.get('/api/workspace/memory/list'), readMemoryFile: (path: string) => - client.get<{ path: string; content: string }>('/api/workspace/memory/file', { params: { path } }), + client.get('/api/workspace/memory/file', { params: { path } }), // Stats stats: () => @@ -118,7 +126,7 @@ export function fileIcon(node: WorkspaceNode): string { if (node.type === 'directory') return '📁'; const ext = node.name.split('.').pop()?.toLowerCase() ?? ''; const map: Record = { - md: '📝', txt: '📄', log: '📋', json: '🔧', yaml: '🔧', yml: '🔧', + md: '📝', txt: '📄', log: '📋', json: '🔧', jsonl: '🔧', yaml: '🔧', yml: '🔧', py: '🐍', js: '🟨', ts: '🔷', tsx: '🔷', jsx: '🟨', sh: '⚙️', bash: '⚙️', csv: '📊', pdf: '📕', png: '🖼️', jpg: '🖼️', jpeg: '🖼️', gif: '🖼️', zip: '🗜️', tar: '🗜️', gz: '🗜️', diff --git a/webui/src/components/common/CommandDropdown.tsx b/webui/src/components/common/CommandDropdown.tsx index 114996104..a19334716 100644 --- a/webui/src/components/common/CommandDropdown.tsx +++ b/webui/src/components/common/CommandDropdown.tsx @@ -87,7 +87,7 @@ export default function CommandDropdown({ /** * 从输入文本中解析 slash 命令的名称和参数 - * 例如 "/plan create a feature" → { command: "plan", args: "create a feature" } + * 例如 "/bug describe issue" → { command: "bug", args: "describe issue" } */ export function parseSlashCommand(text: string): { command: string; args: string } | null { const trimmed = text.trim(); diff --git a/webui/src/components/common/DelegateTaskCard.test.ts b/webui/src/components/common/DelegateTaskCard.test.ts new file mode 100644 index 000000000..b7bccaecf --- /dev/null +++ b/webui/src/components/common/DelegateTaskCard.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from 'vitest'; + +import { + extractDelegateInfo, + shouldRenderDelegateTaskCard, +} from './DelegateTaskCard'; + +// --------------------------------------------------------------------------- +// shouldRenderDelegateTaskCard +// --------------------------------------------------------------------------- + +describe('shouldRenderDelegateTaskCard', () => { + it('returns true for delegate_task tool regardless of input', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'delegate_task', + state: { input: {}, output: '' }, + } as any), + ).toBe(true); + }); + + it('returns true for task tool (alias) regardless of input', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'task', + state: { input: {}, output: '' }, + } as any), + ).toBe(true); + }); + + it('returns false for unknown tool with no delegate input', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'unknown', + state: { input: { something: 'else' }, output: '' }, + } as any), + ).toBe(false); + }); + + it('returns false for MCP tool even if output contains task_metadata block', () => { + // Critical regression guard: wecom_mcp / threatbook_mcp style tools must + // not be misclassified just because their output happens to embed + // session_id: .... + expect( + shouldRenderDelegateTaskCard({ + tool: 'wecom_mcp', + state: { + input: { action: 'send_message' }, + output: '\nsession_id: wxwork-msg-12345\n', + }, + } as any), + ).toBe(false); + }); + + it('returns false for a real non-delegate tool name', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'bash', + state: { input: { command: 'ls' }, output: 'ok' }, + } as any), + ).toBe(false); + }); + + it('returns true for unknown tool with subagent_type input + task_metadata output', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'unknown', + state: { + input: { subagent_type: 'explore', prompt: 'investigate' }, + output: '\nsession_id: ses-x\n', + }, + } as any), + ).toBe(true); + }); + + it('returns true for unknown tool with category input + task_metadata output', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'unknown', + state: { + input: { category: 'quick', prompt: 'summarize' }, + output: '\nsession_id: ses-y\n', + }, + } as any), + ).toBe(true); + }); + + it('returns false for unknown tool with delegate input but no session_id', () => { + expect( + shouldRenderDelegateTaskCard({ + tool: 'unknown', + state: { + input: { subagent_type: 'explore' }, + output: 'no metadata here', + }, + } as any), + ).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// extractDelegateInfo +// --------------------------------------------------------------------------- + +describe('extractDelegateInfo', () => { + it('shows a launched background subagent as running until child completion arrives', () => { + const info = extractDelegateInfo({ + status: 'completed', + input: { + description: 'inspect permissions', + prompt: 'Inspect permissions', + subagent_type: 'explore', + run_in_background: true, + }, + output: 'Background task launched successfully.', + metadata: { + sessionId: 'ses-child', + taskId: 'bg-task', + status: 'running', + background: true, + }, + } as any, 'Subtask'); + + expect(info.status).toBe('running'); + expect(info.isBackground).toBe(true); + expect(info.childSessionId).toBe('ses-child'); + }); + + it('shows a background subagent as completed after parent tool part is updated', () => { + const info = extractDelegateInfo({ + status: 'completed', + input: { + description: 'inspect permissions', + prompt: 'Inspect permissions', + subagent_type: 'explore', + run_in_background: true, + }, + output: 'done', + metadata: { + sessionId: 'ses-child', + taskId: 'bg-task', + status: 'completed', + background: true, + }, + } as any, 'Subtask'); + + expect(info.status).toBe('completed'); + }); + + it('extracts session_id from when nested metadata is missing', () => { + const info = extractDelegateInfo({ + status: 'completed', + input: { subagent_type: 'explore', prompt: 'investigate' }, + output: + 'Some text\n\nsession_id: ses-from-output\n\nMore text', + } as any, 'Subtask'); + + expect(info.childSessionId).toBe('ses-from-output'); + }); + + it('strips the block from the rendered output', () => { + const info = extractDelegateInfo({ + status: 'completed', + input: { subagent_type: 'explore', prompt: 'investigate' }, + output: + 'Result text\n\nsession_id: ses-x\n', + } as any, 'Subtask'); + + expect(info.output).toBe('Result text'); + }); +}); diff --git a/webui/src/components/common/DelegateTaskCard.tsx b/webui/src/components/common/DelegateTaskCard.tsx index ef52ab20d..9110fb61f 100644 --- a/webui/src/components/common/DelegateTaskCard.tsx +++ b/webui/src/components/common/DelegateTaskCard.tsx @@ -22,18 +22,26 @@ export function isDelegateTool(toolName: string): boolean { } export function shouldRenderDelegateTaskCard(part: MessagePart): boolean { + // 1. Explicit delegate tool: render the card. if (part.tool && isDelegateTool(part.tool)) { return true; } const state: Partial = part.state || {}; const input = state.input || {}; - if (typeof input.subagent_type === 'string' && input.subagent_type.trim()) { - return true; - } - const output = typeof state.output === 'string' ? state.output : undefined; + // 2. Unknown/missing tool: only upgrade if the part *also* carries + // delegate-shaped input. This prevents MCP tools (e.g. wecom_mcp, + // threatbook_mcp) from being misclassified just because their output + // happens to contain a `` block. if (!part.tool || part.tool === 'unknown') { + const hasDelegateInput = + (typeof input.subagent_type === 'string' && input.subagent_type.trim()) || + (typeof input.category === 'string' && input.category.trim()); + if (!hasDelegateInput) { + return false; + } + const output = typeof state.output === 'string' ? state.output : undefined; return !!extractSessionId(state.metadata, output); } return false; @@ -64,11 +72,13 @@ function extractSessionId( meta: Record | undefined, output: string | undefined, ): string | undefined { + // Top-level meta only trusts the canonical `sessionId` key. Variant + // casings (sessionID / session_id) are accepted only in the nested + // `metadata` envelope to avoid matching arbitrary tool metadata that + // happens to use snake_case or PascalCase. const innerMeta = meta?.metadata as Record | undefined; const sessionId = meta?.sessionId ?? - meta?.sessionID ?? - meta?.session_id ?? innerMeta?.sessionId ?? innerMeta?.sessionID ?? innerMeta?.session_id; @@ -82,7 +92,7 @@ function extractSessionId( return match?.[1]?.trim() || undefined; } -function extractDelegateInfo(state: Partial, subTaskLabel: string): DelegateInfo { +export function extractDelegateInfo(state: Partial, subTaskLabel: string): DelegateInfo { const input = state.input || {}; const agentRaw = input.subagent_type || input.category || 'unknown'; const agentName = typeof agentRaw === 'string' @@ -112,13 +122,22 @@ function extractDelegateInfo(state: Partial, subTaskLabel: string): D const stepCount = (meta?.stepCount ?? innerMeta?.stepCount ?? 0) as number; const currentText = (meta?.currentText ?? innerMeta?.currentText ?? '') as string; const elapsed = (meta?.elapsed ?? innerMeta?.elapsed ?? 0) as number; + const isBackground = !!input.run_in_background || Boolean(meta?.background ?? innerMeta?.background); + const runtimeStatus = meta?.status ?? innerMeta?.status; + const backgroundStatus = isBackground + ? (typeof runtimeStatus === 'string' && runtimeStatus.trim() + ? runtimeStatus + : state.status === 'completed' && childSessionId + ? 'running' + : undefined) + : undefined; return { agentName, description: input.description || subTaskLabel, - isBackground: !!input.run_in_background, + isBackground, childSessionId, - status: state.status || 'pending', + status: backgroundStatus || state.status || 'pending', error: state.error, output, durationMs, @@ -330,7 +349,10 @@ export default function DelegateTaskCard({ part }: DelegateTaskCardProps) { {/* View detail button — always visible */} - {/* divider + injected slot (e.g. agent selector) */} - {toolbarSlot && ( - <> -
- {toolbarSlot} - +
+ + {toolbarSlot} + + {centerToolbarSlot && ( +
+ {centerToolbarSlot} +
)}
+ {contextWindowTokens && contextWindowTokens > 0 && ( + + )} + {isStreaming ? ( <> - + {canSend && ( + + )}
)} - {otherParts.map((part: MessagePart, i: number) => ( + {displayParts.map((part: MessagePart, i: number) => ( // Spacing between consecutive parts is owned by this wrapper, // not by individual part components. Each part used to set its // own `mt-2 first:mt-0`, but since every part lives in its own @@ -3057,7 +3319,11 @@ export function truncateToolDisplayText(text: string, maxLen = TOOL_DISPLAY_MAX_ function buildToolInputSummary(input: Record): string { return Object.entries(input) - .map(([k, v]) => `${k}=${String(v)}`) + .map(([k, v]) => { + if (Array.isArray(v)) return `${k}=[${v.length} items]`; + if (v && typeof v === 'object') return `${k}=${JSON.stringify(v)}`; + return `${k}=${String(v)}`; + }) .join(', '); } @@ -3095,10 +3361,16 @@ function pickTodoEntries(...candidates: unknown[]): TodoSummaryEntry[] { return []; } -export function buildTodoWriteSummary(state: Partial): string { +function getTodoActionLabel(action: unknown): string { + if (action === 'read') return 'Read todos'; + if (action === 'write') return 'Update todos'; + return 'Todos'; +} + +export function buildTodoSummary(state: Partial): string { const metadata = state.metadata ?? {}; const currentTodos = pickTodoEntries(metadata.newTodos, metadata.todos, state.input?.todos); - if (currentTodos.length === 0) return ''; + if (currentTodos.length === 0) return getTodoActionLabel(state.input?.action); const totalCount = currentTodos.length; const terminalCount = currentTodos.filter( (todo) => todo.status === 'completed' || todo.status === 'cancelled', @@ -3120,6 +3392,34 @@ export function buildTodoWriteSummary(state: Partial): string { return summary; } +function todoStatusLabel(status: string | undefined): string { + switch (status) { + case 'completed': + return 'completed'; + case 'in_progress': + return 'in progress'; + case 'cancelled': + return 'cancelled'; + case 'pending': + return 'pending'; + default: + return status || 'pending'; + } +} + +function todoStatusClass(status: string | undefined): string { + switch (status) { + case 'completed': + return 'bg-emerald-50 text-emerald-700 border-emerald-100'; + case 'in_progress': + return 'bg-sky-50 text-sky-700 border-sky-100'; + case 'cancelled': + return 'bg-zinc-100 text-zinc-500 border-zinc-200'; + default: + return 'bg-amber-50 text-amber-700 border-amber-100'; + } +} + export interface ChatToolPartProps { part: MessagePart; pendingQuestion?: PendingQuestion; @@ -3185,17 +3485,22 @@ export function ChatToolPart({ part, pendingQuestion, onAnswer, onReject }: Chat } return JSON.stringify(output, null, 2); }; + const todoEntries = toolName === 'todo' + ? pickTodoEntries(state.metadata?.newTodos, state.metadata?.todos, state.input?.todos) + : []; + const showGenericToolPayload = toolName !== 'todo'; // Reuse the shared helpers so the truncation rules stay in sync with the // delegate-task card and any other places that render tool input previews. const inputSummary = state.input ? truncateToolDisplayText( - toolName === 'todowrite' - ? (buildTodoWriteSummary(state) || buildToolInputSummary(state.input)) + toolName === 'todo' + ? buildTodoSummary(state) : buildToolInputSummary(state.input), ) : ''; const displayTitle = state.title ? truncateToolDisplayText(state.title) : ''; + const workflowHeaderSummary = truncateToolDisplayText(buildRunWorkflowHeaderSummary(toolName, state, t)); if (isWaitingForAnswer) { // Outer spacing is owned by the part wrapper in SessionChat's parts map. @@ -3216,24 +3521,36 @@ export function ChatToolPart({ part, pendingQuestion, onAnswer, onReject }: Chat // spacing so every adjacent tool / thinking / text part is separated by a // single, uniform 8px gap. See the comment on the wrapper in `parts.map`.
- - {config.icon} - {toolName.replace(/_/g, ' ')} - {inputSummary && ( - - {inputSummary} - - )} - {displayTitle && !inputSummary && ( - - {displayTitle} - - )} -
+ + {config.icon} +
+
+ {toolName.replace(/_/g, ' ')} + {workflowHeaderSummary ? ( + + {workflowHeaderSummary} + + ) : ( + <> + {inputSummary && ( + + {inputSummary} + + )} + {displayTitle && !inputSummary && ( + + {displayTitle} + + )} + + )} +
+
+
{config.label} @@ -3242,7 +3559,26 @@ export function ChatToolPart({ part, pendingQuestion, onAnswer, onReject }: Chat
- {state.input && ( + {toolName === 'todo' && todoEntries.length > 0 && ( +
+
{t('chat.tool.todoStages')}
+
+ {todoEntries.map((todo, index) => ( +
+ + + {todo.activeForm && todo.status === 'in_progress' ? todo.activeForm : todo.content} + + + {todoStatusLabel(todo.status)} + +
+ ))} +
+
+ )} + + {showGenericToolPayload && state.input && (
{t('chat.tool.inputParams')} @@ -3253,7 +3589,7 @@ export function ChatToolPart({ part, pendingQuestion, onAnswer, onReject }: Chat
)} - {status === 'completed' && state.output !== undefined && ( + {showGenericToolPayload && status === 'completed' && state.output !== undefined && (
{t('chat.tool.outputResult')} diff --git a/webui/src/components/common/WorkflowStatusBadge.tsx b/webui/src/components/common/WorkflowStatusBadge.tsx index e3489d070..7c7a24962 100644 --- a/webui/src/components/common/WorkflowStatusBadge.tsx +++ b/webui/src/components/common/WorkflowStatusBadge.tsx @@ -14,6 +14,7 @@ interface StatusConfig { const STATUS_STYLE_MAP: Record = { running: { className: 'bg-red-100 text-red-700', dot: 'bg-red-500' }, + cancelling: { className: 'bg-amber-100 text-amber-700', dot: 'bg-amber-500' }, publishing: { className: 'bg-yellow-100 text-yellow-700', dot: 'bg-yellow-500' }, success: { className: 'bg-green-100 text-green-700', dot: 'bg-green-500' }, SUCCEEDED: { className: 'bg-green-100 text-green-700', dot: 'bg-green-500' }, diff --git a/webui/src/components/common/toolStageSummary.test.ts b/webui/src/components/common/toolStageSummary.test.ts new file mode 100644 index 000000000..d22033b58 --- /dev/null +++ b/webui/src/components/common/toolStageSummary.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from 'vitest'; + +import { buildRunWorkflowHeaderSummary } from './toolStageSummary'; + +describe('buildRunWorkflowHeaderSummary', () => { + const zhT = (key: string, options?: Record) => { + switch (key) { + case 'chat.tool.workflowPhase.running': + return '执行中'; + case 'chat.tool.workflowPhase.queued': + return '排队中'; + case 'chat.tool.workflowStep': + return `步骤：${String(options?.step ?? '')}`; + case 'chat.tool.workflowNode': + return `节点：${String(options?.node ?? '')}`; + default: + return key; + } + }; + + it('returns a running workflow header summary with total nodes and node id', () => { + expect( + buildRunWorkflowHeaderSummary( + 'run_workflow', + { + status: 'running', + input: { + workflow: '/tmp/keyword-search-summary/workflow.json', + }, + metadata: { + workflow_name: 'keyword-search-summary', + phase: 'running', + current_node_id: 'validate_input', + step_index: 2, + total_nodes: 10, + }, + }, + zhT, + ), + ).toBe('keyword-search-summary 执行中 · 2/10 · 节点：validate_input'); + }); + + it('shows queued phase before the first node starts', () => { + expect( + buildRunWorkflowHeaderSummary( + 'run_workflow', + { + status: 'running', + metadata: { + workflow_name: 'keyword-search-summary', + phase: 'queued', + step_index: 0, + }, + }, + zhT, + ), + ).toBe('keyword-search-summary 排队中'); + }); + + it('falls back to concise english labels when no translator is provided', () => { + expect( + buildRunWorkflowHeaderSummary( + 'run_workflow', + { + status: 'running', + metadata: { + workflow_name: 'keyword-search-summary', + phase: 'running', + current_node_id: 'validate_input', + step_index: 2, + total_nodes: 10, + }, + }, + ), + ).toBe('keyword-search-summary running · 2/10 · node:validate_input'); + }); + + it('returns empty for non-workflow tools or non-running states', () => { + expect(buildRunWorkflowHeaderSummary('bash', { status: 'running' })).toBe(''); + expect(buildRunWorkflowHeaderSummary('run_workflow', { status: 'completed' })).toBe(''); + }); +}); diff --git a/webui/src/components/common/toolStageSummary.ts b/webui/src/components/common/toolStageSummary.ts new file mode 100644 index 000000000..1ad53eb2d --- /dev/null +++ b/webui/src/components/common/toolStageSummary.ts @@ -0,0 +1,103 @@ +import type { ToolState } from '@/types'; + +type SummaryTranslator = (key: string, options?: Record) => string; + +function resolvePhaseLabel(phase: string): string { + const normalized = phase.trim().toLowerCase(); + if (!normalized) return 'running'; + if (normalized === 'success') return 'completed'; + if (normalized === 'error') return 'failed'; + if (normalized === 'cancelled') return 'cancelled'; + if (normalized === 'timeout') return 'timed out'; + if (normalized === 'queued') return 'queued'; + return normalized; +} + +function resolvePhaseTranslationKey(phase: string): string | null { + const normalized = phase.trim().toLowerCase(); + if (!normalized) return 'running'; + if (normalized === 'success') return 'success'; + if (normalized === 'error') return 'error'; + if (normalized === 'cancelled') return 'cancelled'; + if (normalized === 'timeout') return 'timeout'; + if (normalized === 'queued') return 'queued'; + if (normalized === 'running') return 'running'; + return null; +} + +function translateOrFallback( + key: string, + fallback: string, + t?: SummaryTranslator, + options?: Record, +): string { + if (!t) return fallback; + const translated = t(key, options); + return translated && translated !== key ? translated : fallback; +} + +function resolveWorkflowName(state: Partial): string { + const metadata = (state.metadata ?? {}) as Record; + const rawMetadataName = metadata.workflow_name; + if (typeof rawMetadataName === 'string' && rawMetadataName.trim()) { + return rawMetadataName.trim(); + } + + const workflowInput = state.input?.workflow; + if (typeof workflowInput === 'string' && workflowInput.trim()) { + const normalized = workflowInput.trim().replace(/\\/g, '/'); + const lastSegment = normalized.split('/').filter(Boolean).pop() || normalized; + return lastSegment.replace(/\.json$/i, '') || lastSegment; + } + return 'workflow'; +} + +export function buildRunWorkflowHeaderSummary( + toolName: string, + state: Partial, + t?: SummaryTranslator, +): string { + if (toolName !== 'run_workflow') return ''; + if ((state.status || 'pending') !== 'running') return ''; + + const metadata = (state.metadata ?? {}) as Record; + const workflowName = resolveWorkflowName(state); + const phaseRaw = metadata.phase; + const currentNodeRaw = metadata.current_node_id; + const stepIndexRaw = metadata.step_index; + const totalNodesRaw = metadata.total_nodes; + + const phase = typeof phaseRaw === 'string' && phaseRaw.trim() ? phaseRaw.trim() : 'running'; + const currentNode = + typeof currentNodeRaw === 'string' && currentNodeRaw.trim() ? currentNodeRaw.trim() : ''; + const stepIndex = + typeof stepIndexRaw === 'number' && Number.isFinite(stepIndexRaw) ? stepIndexRaw : null; + const totalNodes = + typeof totalNodesRaw === 'number' && Number.isFinite(totalNodesRaw) && totalNodesRaw > 0 + ? totalNodesRaw + : null; + + const phaseKey = resolvePhaseTranslationKey(phase); + const phaseLabel = phaseKey + ? translateOrFallback( + `chat.tool.workflowPhase.${phaseKey}`, + resolvePhaseLabel(phase), + t, + ) + : resolvePhaseLabel(phase); + + let summary = `${workflowName} ${phaseLabel}`; + if (stepIndex !== null && stepIndex > 0) { + const stepLabel = totalNodes !== null ? `${stepIndex}/${totalNodes}` : `${stepIndex}`; + summary += ` · ${stepLabel}`; + } + if (currentNode) { + summary += ` · ${translateOrFallback( + 'chat.tool.workflowNode', + `node:${currentNode}`, + t, + { node: currentNode }, + )}`; + } + return summary; +} diff --git a/webui/src/components/layout/Layout.test.tsx b/webui/src/components/layout/Layout.test.tsx index 368b6ddd7..883ac1435 100644 --- a/webui/src/components/layout/Layout.test.tsx +++ b/webui/src/components/layout/Layout.test.tsx @@ -21,6 +21,7 @@ const { consoleUpgradeApi, useAuth, useStats, + useUserDefinedPages, } = vi.hoisted(() => ({ catalogAPI: { list: vi.fn(), @@ -54,6 +55,24 @@ const { }, useAuth: vi.fn(), useStats: vi.fn(), + useUserDefinedPages: vi.fn(() => ({ + pages: [ + { + id: 'dash-1', + title: '自定义仪表盘', + route: '/user-defined-pages/dash-1', + icon: 'LayoutDashboard', + order: 10, + enabled: true, + placement: 'home.after', + buildHash: 'abc', + buildStatus: 'ready', + }, + ], + loading: false, + error: null, + refetch: vi.fn(), + })), })); vi.mock('@/api/provider', () => ({ @@ -100,6 +119,16 @@ vi.mock('@/hooks/useStats', () => ({ useStats, })); +vi.mock('@/components/common/Toast', () => ({ + useToast: () => ({ + error: vi.fn(), + }), +})); + +vi.mock('@/hooks/useUserDefinedPages', () => ({ + useUserDefinedPages, +})); + vi.mock('@/components/common/LanguageSwitcher', () => ({ default: () => null, })); @@ -291,6 +320,19 @@ describe('Layout onboarding entry', () => { expect(screen.queryByPlaceholderText('onboarding.bootstrap.tbPlaceholder')).not.toBeInTheDocument(); }); + it('keeps standard pages out of a flex column content wrapper', async () => { + localStorage.setItem('flocks_onboarding_dismissed', 'true'); + + const { container } = renderHomeWithLayout(); + + await flushEffects(); + + const contentWrapper = container.querySelector('main .min-h-full.p-6'); + expect(contentWrapper).not.toBeNull(); + expect(contentWrapper).not.toHaveClass('flex'); + expect(contentWrapper).not.toHaveClass('flex-col'); + }); + it('polls update checks hourly', async () => { vi.useFakeTimers(); @@ -528,3 +570,87 @@ describe('Layout onboarding entry', () => { expect(screen.getByText('Flocks v2026.04.28 更新内容')).toBeInTheDocument(); }); }); + +describe('Layout user defined pages navigation', () => { + beforeEach(() => { + vi.clearAllMocks(); + localStorage.clear(); + checkUpdate.mockResolvedValue({ + has_update: false, + latest_version: null, + current_version: '0.2.0', + error: null, + }); + getActiveNotifications.mockResolvedValue([]); + useAuth.mockReturnValue({ + user: { + id: 'user-1', + username: 'admin', + role: 'admin', + status: 'active', + must_reset_password: false, + }, + }); + useStats.mockReturnValue({ + stats: { + agents: { total: 0 }, + workflows: { total: 0 }, + skills: { total: 0 }, + tools: { total: 0 }, + tasks: { week: 0, scheduledActive: 0 }, + models: { total: 0 }, + system: { status: 'healthy' }, + }, + loading: false, + error: null, + }); + flocksproUsersApi.hasCapability.mockResolvedValue(false); + flocksproUsersApi.getLicenseStatus.mockResolvedValue({ pro_enabled: false }); + consoleUpgradeApi.getProPackageStatus.mockResolvedValue({ pro_enabled: false }); + }); + + it('renders custom user defined page links under the home section', async () => { + renderHomeWithLayout(); + expect(await screen.findByRole('link', { name: '自定义仪表盘' })).toHaveAttribute( + 'href', + '/user-defined-pages/dash-1', + ); + }); + + it('does not render custom page links until their build is ready', async () => { + useUserDefinedPages.mockReturnValue({ + pages: [ + { + id: 'ready-page', + title: '可用页面', + route: '/user-defined-pages/ready-page', + icon: 'LayoutDashboard', + order: 10, + enabled: true, + placement: 'home.after', + buildHash: 'ready', + buildStatus: 'ready', + }, + { + id: 'failed-page', + title: '失败页面', + route: '/user-defined-pages/failed-page', + icon: 'LayoutDashboard', + order: 20, + enabled: true, + placement: 'home.after', + buildHash: '', + buildStatus: 'failed', + }, + ], + loading: false, + error: null, + refetch: vi.fn(), + }); + + renderHomeWithLayout(); + + expect(await screen.findByRole('link', { name: '可用页面' })).toBeInTheDocument(); + expect(screen.queryByRole('link', { name: '失败页面' })).not.toBeInTheDocument(); + }); +}); diff --git a/webui/src/components/layout/Layout.tsx b/webui/src/components/layout/Layout.tsx index 3bfa743e3..5ffad5b9e 100644 --- a/webui/src/components/layout/Layout.tsx +++ b/webui/src/components/layout/Layout.tsx @@ -51,6 +51,8 @@ import { import { flocksproUsersApi } from '@/api/flocksproUsers'; import { useAuth } from '@/contexts/AuthContext'; import { getLocalizedReleaseNotes } from '@/utils/releaseNotes'; +import { useUserDefinedPages } from '@/hooks/useUserDefinedPages'; +import { resolveUserDefinedPageIcon } from '@/utils/userDefinedPageIcons'; const UPDATE_CHECK_INTERVAL_MS = 3_600_000; const UPDATE_CHECK_MIN_GAP_MS = 600_000; @@ -114,6 +116,7 @@ export default function Layout() { const [flocksproStatusReady, setFlocksproStatusReady] = useState(false); const [flocksproVersion, setFlocksproVersion] = useState(null); const canManageUpdates = user?.role === 'admin'; + const { pages: userDefinedPages } = useUserDefinedPages(); // useLayoutEffect runs synchronously before paint, so there's no flash on initial load. // It also re-runs when the user navigates back to /, covering both cases in one place. useLayoutEffect(() => { @@ -411,6 +414,13 @@ export default function Layout() { name: '', items: [ { name: t('flocksHome'), href: '/', icon: Home }, + ...userDefinedPages + .filter((page) => page.enabled && page.placement === 'home.after' && page.buildStatus === 'ready') + .map((page) => ({ + name: page.title, + href: page.route, + icon: resolveUserDefinedPageIcon(page.icon), + })), ], }, { @@ -448,14 +458,15 @@ export default function Layout() { ], }, ], - [hasFlocksproCapability, t, user?.role], + [hasFlocksproCapability, userDefinedPages, t, user?.role], ); const isFullScreenPage = matchPath('/workflows/create', location.pathname) || matchPath('/workflows/:id/edit', location.pathname) || matchPath('/workflows/:id', location.pathname) || - matchPath('/sessions', location.pathname); + matchPath('/sessions', location.pathname) || + matchPath('/devices', location.pathname); const productName = isFlocksproActive ? 'Flocks Pro' : 'Flocks'; const displayVersion = isFlocksproActive ? flocksproVersion || (currentVersion ? formatProVersion(currentVersion) : null) diff --git a/webui/src/hooks/useSessions.test.ts b/webui/src/hooks/useSessions.test.ts index cf64dbbbe..514f9dc47 100644 --- a/webui/src/hooks/useSessions.test.ts +++ b/webui/src/hooks/useSessions.test.ts @@ -25,14 +25,17 @@ function makeMsg(overrides: Partial & { id: string }): Message { describe('applyMessagePartUpdate', () => { describe('message not found', () => { - it('appends part to the last in-progress assistant message when messageID does not match', () => { + it('creates a placeholder for the part message instead of reusing a previous assistant', () => { const partInfo = { id: 'p1', messageID: 'msg-unknown', sessionID: 'sess-1', type: 'text', text: 'hello' }; const prev: Message[] = [ - makeMsg({ id: 'msg-1', role: 'assistant', parts: [] }), + makeMsg({ id: 'msg-1', role: 'assistant', parts: [], finish: null } as any), ]; const result = applyMessagePartUpdate(prev, partInfo); - expect(result[0].parts).toHaveLength(1); - expect((result[0].parts as any[])[0].id).toBe('p1'); + expect(result).toHaveLength(2); + expect(result[0].id).toBe('msg-1'); + expect(result[0].parts).toHaveLength(0); + expect(result[1].id).toBe('msg-unknown'); + expect((result[1].parts as any[])[0].id).toBe('p1'); }); it('skips finished assistant messages when looking for in-progress message', () => { @@ -252,6 +255,47 @@ describe('updateMessagePart scheduling', () => { expect((msg!.parts as any[])[1].text).toBe('after'); }); + it('inserts late user metadata before an already streamed assistant child', async () => { + const { result } = renderHook(() => useSessionMessages('sess-1')); + await act(async () => {}); + + await act(async () => { + result.current.addMessage(makeMsg({ + id: 'old-assistant', + role: 'assistant', + parts: [{ id: 'old-text', type: 'text', text: 'old reply' } as any], + finish: 'stop', + } as any)); + result.current.updateMessagePart({ + id: 'new-text', + messageID: 'new-assistant', + sessionID: 'sess-1', + type: 'text', + text: 'new reply', + }); + result.current.updateMessage({ + id: 'new-assistant', + sessionID: 'sess-1', + role: 'assistant', + parentID: 'new-user', + time: { created: 200 }, + }); + result.current.updateMessage({ + id: 'new-user', + sessionID: 'sess-1', + role: 'user', + time: { created: 100 }, + }); + }); + + expect(result.current.messages.map((msg) => msg.id)).toEqual([ + 'old-assistant', + 'new-user', + 'new-assistant', + ]); + expect((result.current.messages[2].parts as any[])[0].text).toBe('new reply'); + }); + it('truncateAfterMessage keeps the target by default', async () => { const { result } = renderHook(() => useSessionMessages('sess-1')); await act(async () => {}); diff --git a/webui/src/hooks/useSessions.ts b/webui/src/hooks/useSessions.ts index 680fad670..5ee94aa58 100644 --- a/webui/src/hooks/useSessions.ts +++ b/webui/src/hooks/useSessions.ts @@ -66,26 +66,9 @@ export function applyMessagePartUpdate( const messageIndex = prev.findIndex(m => m.id === partInfo.messageID); if (messageIndex < 0) { - // Message not found — reuse the last in-progress assistant message if available - let lastAssistantIndex = -1; - for (let i = prev.length - 1; i >= 0; i--) { - if (prev[i].role === 'assistant' && !prev[i].finish) { - lastAssistantIndex = i; - break; - } - } - - if (lastAssistantIndex >= 0) { - const updated = [...prev]; - const message = { ...updated[lastAssistantIndex] }; - const parts = [...(message.parts || [])]; - parts.push(partInfo); - message.parts = parts; - updated[lastAssistantIndex] = message; - return updated; - } - - // No in-progress assistant message — create a placeholder + // Message metadata can arrive after part updates over SSE. Keep the part + // attached to its own messageID instead of borrowing a nearby assistant, + // otherwise chunks from a new turn can render inside the previous reply. return [...prev, { id: partInfo.messageID, sessionID: partInfo.sessionID, @@ -319,7 +302,7 @@ export function useSessionMessages(sessionId?: string) { } // Add new message - return [...prev, { + const nextMessage = { id: messageInfo.id, sessionID: messageInfo.sessionID, role: messageInfo.role, @@ -328,6 +311,21 @@ export function useSessionMessages(sessionId?: string) { agent: messageInfo.agent, model: messageInfo.model, timestamp: messageInfo.time?.created || Date.now(), + }; + + if (messageInfo.role === 'user') { + const childIndex = prev.findIndex( + (m) => m.role === 'assistant' && m.parentID === messageInfo.id, + ); + if (childIndex >= 0) { + const updated = [...prev]; + updated.splice(childIndex, 0, nextMessage); + return updated; + } + } + + return [...prev, { + ...nextMessage, }]; }); }, diff --git a/webui/src/hooks/useUserDefinedPages.test.tsx b/webui/src/hooks/useUserDefinedPages.test.tsx new file mode 100644 index 000000000..f10bb4280 --- /dev/null +++ b/webui/src/hooks/useUserDefinedPages.test.tsx @@ -0,0 +1,88 @@ +import { renderHook, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useUserDefinedPages } from './useUserDefinedPages'; +import { setupSSEMock } from '@/test/mocks/sse'; + +const { listMock } = vi.hoisted(() => ({ + listMock: vi.fn(), +})); + +vi.mock('@/api/userDefinedPages', () => ({ + userDefinedPagesAPI: { + list: listMock, + }, +})); + +describe('useUserDefinedPages', () => { + const sse = setupSSEMock(); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('loads enabled user defined pages for navigation', async () => { + listMock.mockResolvedValueOnce({ + data: [ + { + id: 'dash-1', + title: '仪表盘', + route: '/user-defined-pages/dash-1', + icon: 'LayoutDashboard', + order: 10, + enabled: true, + placement: 'home.after', + buildHash: 'abc', + buildStatus: 'ready', + }, + ], + }); + + const { result } = renderHook(() => useUserDefinedPages()); + + await waitFor(() => { + expect(result.current.loading).toBe(false); + }); + + expect(result.current.pages).toHaveLength(1); + expect(result.current.pages[0].title).toBe('仪表盘'); + expect(listMock).toHaveBeenCalledWith(true); + }); + + it('refetches when user_defined_pages.nav_changed SSE event arrives', async () => { + listMock + .mockResolvedValueOnce({ data: [] }) + .mockResolvedValueOnce({ + data: [ + { + id: 'dash-2', + title: '新页面', + route: '/user-defined-pages/dash-2', + icon: 'LayoutDashboard', + order: 20, + enabled: true, + placement: 'home.after', + buildHash: 'def', + buildStatus: 'ready', + }, + ], + }); + + const { result } = renderHook(() => useUserDefinedPages()); + + await waitFor(() => { + expect(result.current.loading).toBe(false); + }); + + sse.open(); + sse.send({ + type: 'user_defined_pages.nav_changed', + properties: { id: 'dash-2' }, + }); + + await waitFor(() => { + expect(result.current.pages).toHaveLength(1); + }); + expect(listMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/webui/src/hooks/useUserDefinedPages.ts b/webui/src/hooks/useUserDefinedPages.ts new file mode 100644 index 000000000..c472b8485 --- /dev/null +++ b/webui/src/hooks/useUserDefinedPages.ts @@ -0,0 +1,74 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import i18n from '@/i18n'; +import { userDefinedPagesAPI, type UserDefinedPageListItem } from '@/api/userDefinedPages'; +import { useSSE } from '@/hooks/useSSE'; + +export function useUserDefinedPages() { + const [pages, setPages] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const loadingRef = useRef(false); + const lastRefreshRef = useRef(0); + + const fetchPages = useCallback(async (silent = false) => { + if (loadingRef.current) return; + loadingRef.current = true; + if (!silent) setLoading(true); + setError(null); + try { + const response = await userDefinedPagesAPI.list(true); + setPages(Array.isArray(response.data) ? response.data : []); + } catch (err: unknown) { + setPages([]); + setError(err instanceof Error ? err.message : i18n.t('nav.fetchFailed', { ns: 'userDefinedPage' })); + } finally { + loadingRef.current = false; + if (!silent) setLoading(false); + } + }, []); + + useEffect(() => { + void fetchPages(); + }, [fetchPages]); + + const refreshOnResume = useCallback((force = false) => { + const now = Date.now(); + if (!force && now - lastRefreshRef.current < 1000) return; + lastRefreshRef.current = now; + void fetchPages(true); + }, [fetchPages]); + + useEffect(() => { + const onVisible = () => { + if (document.visibilityState === 'visible') { + refreshOnResume(false); + } + }; + const onFocus = () => { + refreshOnResume(false); + }; + document.addEventListener('visibilitychange', onVisible); + window.addEventListener('focus', onFocus); + return () => { + document.removeEventListener('visibilitychange', onVisible); + window.removeEventListener('focus', onFocus); + }; + }, [refreshOnResume]); + + useSSE({ + url: '/api/event', + onEvent: useCallback((evt) => { + if (evt.type === 'user_defined_pages.nav_changed') { + void fetchPages(true); + } + }, [fetchPages]), + reconnect: { maxRetries: 5, initialDelay: 2000 }, + }); + + return { + pages, + loading, + error, + refetch: () => fetchPages(), + }; +} diff --git a/webui/src/i18n.ts b/webui/src/i18n.ts index e35511567..ef5e0c139 100644 --- a/webui/src/i18n.ts +++ b/webui/src/i18n.ts @@ -22,6 +22,8 @@ import enWorkspace from './locales/en-US/workspace.json'; import enAuth from './locales/en-US/auth.json'; import enNotification from './locales/en-US/notification.json'; import enFlocksPro from './locales/en-US/flockspro.json'; +import enUserDefinedPage from './locales/en-US/userDefinedPage.json'; +import enDevice from './locales/en-US/device.json'; import zhCommon from './locales/zh-CN/common.json'; import zhNav from './locales/zh-CN/nav.json'; @@ -43,6 +45,8 @@ import zhWorkspace from './locales/zh-CN/workspace.json'; import zhAuth from './locales/zh-CN/auth.json'; import zhNotification from './locales/zh-CN/notification.json'; import zhFlocksPro from './locales/zh-CN/flockspro.json'; +import zhUserDefinedPage from './locales/zh-CN/userDefinedPage.json'; +import zhDevice from './locales/zh-CN/device.json'; i18n .use(LanguageDetector) @@ -70,6 +74,8 @@ i18n auth: enAuth, notification: enNotification, flockspro: enFlocksPro, + userDefinedPage: enUserDefinedPage, + device: enDevice, }, 'zh-CN': { common: zhCommon, @@ -92,11 +98,13 @@ i18n auth: zhAuth, notification: zhNotification, flockspro: zhFlocksPro, + userDefinedPage: zhUserDefinedPage, + device: zhDevice, }, }, fallbackLng: 'en-US', defaultNS: 'common', - ns: ['common', 'nav', 'home', 'session', 'agent', 'task', 'workflow', 'tool', 'skill', 'model', 'mcp', 'config', 'channel', 'permission', 'monitoring', 'update', 'workspace', 'auth', 'notification', 'flockspro'], + ns: ['common', 'nav', 'home', 'session', 'agent', 'task', 'workflow', 'tool', 'skill', 'model', 'mcp', 'config', 'channel', 'permission', 'monitoring', 'update', 'workspace', 'auth', 'notification', 'flockspro', 'device', 'userDefinedPage'], detection: { order: ['localStorage', 'navigator'], lookupLocalStorage: 'flocks-language', diff --git a/webui/src/locales/en-US/agent.json b/webui/src/locales/en-US/agent.json index eda6aba19..8f94d77ab 100644 --- a/webui/src/locales/en-US/agent.json +++ b/webui/src/locales/en-US/agent.json @@ -38,6 +38,8 @@ }, "form": { "name": "Name", + "nameCn": "Chinese Name", + "nameCnPlaceholder": "Chinese display name", "description": "Description (English)", "descriptionHint": "Used for delegation and model context; English by convention", "descriptionPlaceholder": "Short English description for delegation", diff --git a/webui/src/locales/en-US/common.json b/webui/src/locales/en-US/common.json index 9737a9f13..a41ec753e 100644 --- a/webui/src/locales/en-US/common.json +++ b/webui/src/locales/en-US/common.json @@ -64,6 +64,7 @@ }, "workflowStatus": { "running": "Running", + "cancelling": "Cancelling", "publishing": "Publishing", "success": "Success", "SUCCEEDED": "Success", diff --git a/webui/src/locales/en-US/device.json b/webui/src/locales/en-US/device.json new file mode 100644 index 000000000..1e88f4790 --- /dev/null +++ b/webui/src/locales/en-US/device.json @@ -0,0 +1,253 @@ +{ + "pageTitle": "Device Integration", + "pageDescription": "Configure security device API connections so Flocks agents can call and control them directly", + + "status": { + "disabled": "Disabled", + "connected": "Connected", + "error": "Connection failed", + "unknown": "Not checked" + }, + + "vendor": { + "unspecified": "Unspecified vendor" + }, + + "sidebar": { + "heading": "Rooms", + "addRoom": "New room", + "allRooms": "All Rooms", + "rename": "Rename", + "deleteRoom": "Delete room", + "deleteDisabled": "{{count}} device(s) present — cannot delete", + "roomNamePlaceholder": "Room name...", + "deleteHasDevices": "Room \"{{name}}\" still has {{count}} device(s). Move or delete them first." + }, + + "toolbar": { + "refresh": "Refresh", + "addDevice": "Add Device", + "collapseAll": "Collapse all", + "expandAll": "Expand all" + }, + + "header": { + "allRooms": "All Rooms", + "deviceCount": "{{count}} device(s) · {{rooms}} room(s)", + "connected": "{{count}} connected", + "failed": "{{count}} failed", + "devices": "{{count}} device(s)" + }, + + "empty": { + "noDevices": "No devices integrated yet", + "noDevicesHint": "Once you add a device, Flocks agents can call its tools", + "addNow": "Add device now", + "roomEmpty": "No devices in this room", + "roomEmptyHint": "Click \"Add Device\" to integrate a device into this room" + }, + + "section": { + "activeDevices": "Integrated Devices", + "ungrouped": "Ungrouped Devices", + "ungroupedHint": "Room not found — edit the device to reassign it" + }, + + "wizard": { + "title": "Add Device", + "closeAriaLabel": "Close add-device panel", + "step1": "1 Select vendor", + "step2": "2 Select device", + "step3": "3 Fill in config", + "modeTitle": "Select access mode", + "step1Custom": "1 Select vendor / custom", + "step2Custom": "2 Select device / mode", + "step3Custom": "3 Fill in details / config", + "selectVendorTitle": "Select {{vendor}} device", + "vendorHint": "Choose a vendor — {{count}} available", + "chooseVendorOrCustom": "Choose a vendor or create a custom device integration", + "customCardTitle": "Custom Device", + "customCardSubtitle": "API / WebCLI / Workflow", + "customCardCta": "Create a new integration path", + "chooseCustomMode": "Choose how this custom device should connect", + "customModes": { + "api": { + "title": "API Integration", + "desc": "Use this when the device exposes APIs. Provide API docs and generate a reusable device plugin." + }, + "webcli": { + "title": "WebCLI Integration", + "desc": "Use this when the device has no open API. First generate and integrate the skill or CLI asset; for security devices, also generate a device plugin for the device page." + }, + "workflow": { + "title": "Workflow Integration", + "desc": "Use this for Syslog, Kafka, or Webhook sources. The device page will guide you to the workflow integration screen for the final setup." + } + }, + "productCount": "{{count}} product(s)", + "instanceCount": "{{count}} integrated", + "installState": { + "install": "Install in FlockHub first", + "update": "Update in FlockHub first", + "broken": "Plugin unavailable", + "installed": "Installed", + "available": "Available", + "updateAvailable": "Update", + "brokenShort": "Broken" + }, + "productHint": "{{count}} product(s) — the same type can be added multiple times" + }, + + "config": { + "closeAriaLabel": "Close device config panel", + "newDeviceTitle": "Fill in config", + "tabConfig": "Config", + "tabTools": "Tools", + "tabToolsCount": "Tools ({{count}})", + "tabOverview": "Overview", + "nameLabel": "Device name", + "namePlaceholder": "e.g. HQ AF Firewall", + "roomLabel": "Room", + "connectionParams": "Connection parameters", + "secretConfigured": "Already set · Leave unchanged to keep it, clear to remove", + "secretRevealFailed": "Failed to reveal credential", + "showSecretAction": "Show", + "hideSecretAction": "Hide", + "showSecretAria": "Show {{label}}", + "hideSecretAria": "Hide {{label}}", + "sslLabel": "SSL verification", + "sslHint": "Disable to reach internal devices with self-signed certificates", + "enabledLabel": "Enable device", + "enabledHint": "When disabled, agents will not call tools from this device", + "testBtn": "Test connection", + "saveBtn": "Save config", + "addBtn": "Confirm integration", + "deleteBtn": "Delete device", + "confirmDelete": "Confirm delete this device config?" + }, + + "tools": { + "empty": "No associated tools", + "colName": "Tool name", + "colDesc": "Description", + "colStatus": "Status", + "colAction": "Action", + "detail": "Test / Details" + }, + + "overview": { + "serviceName": "Service name", + "version": "Version", + "toolCount": "Tools", + "vendor": "Vendor", + "serviceDesc": "Service description", + "viewDocs": "View API docs" + }, + + "toast": { + "loadFailed": "Failed to load", + "nameRequired": "Please enter a device name", + "saveDone": "Config saved", + "addDone": "Device added", + "saveFailed": "Save failed", + "sslOn": "SSL verification enabled", + "sslOff": "SSL verification disabled", + "rollback": "Save failed — changes rolled back", + "enabledOn": "Device enabled", + "enabledOff": "Device disabled", + "deleteDone": "Device deleted", + "deleteFailed": "Delete failed", + "actionFailed": "Action failed" + }, + + "custom": { + "title": { + "api": "Custom Device API Integration", + "webcli": "Custom Device WebCLI Integration", + "workflow": "Custom Device Workflow Integration" + }, + "subtitle": { + "api": "Provide API documentation and generate a reusable device plugin", + "webcli": "Provide the product URL and target interfaces to generate a WebCLI device plugin for the device page", + "workflow": "Configure Syslog, Kafka, and Webhook entries on the workflow Publish page" + }, + "welcome": { + "api": "Please add an API docs URL or upload the docs file directly. I will use the tool-builder skill to generate a device plugin that can be used from the device page.", + "webcli": "Please add the product URL, target interfaces, and auth hints. I will use the web2cli skill to build and integrate the CLI or skill assets first, and then generate a device plugin when needed.", + "workflow": "Workflow entries are not created here. Go to the workflow Publish page and configure Syslog, Kafka, or Webhook as needed." + }, + "validation": { + "deviceNameRequired": "Please enter the device product name", + "vendorNameRequired": "Please enter the vendor name", + "baseUrlRequired": "Please enter the Base URL", + "productUrlRequired": "Please enter the product URL", + "targetInterfacesRequired": "Please describe the interfaces or page behaviors to capture" + }, + "toast": { + "submitSuccess": "Submitted to Rex. Continue the plugin generation in chat.", + "submitFailed": "Failed to submit to Rex" + }, + "details": { + "prepareTitle": "Prepare the integration details before sending them to Rex", + "prepareIntro": "After submission, you will enter the Rex conversation directly. ", + "apiNext": "You can continue to add API doc links, upload documentation files, or explain interface details. Once the plugin is generated, return to the device page to finish the rest of the setup.", + "webcliNext": "You can continue to add page actions, capture targets, and authentication details. The final result should be a WebCLI device plugin that the device page can recognize; a CLI can remain as an optional debugging entry." + }, + "rex": { + "apiHint": "Upload API docs or describe the endpoints. Rex will clarify key gaps before generating the device plugin.", + "webcliHint": "Provide the product URL and target pages/actions. Rex will clarify login, permissions, and risky operations before generating assets.", + "placeholder": "Continue adding interface details, authentication information, or debugging notes", + "apiPlaceholder": "Provide the product API documentation", + "webcliPlaceholder": "Provide the website URL", + "pending": "Rex is getting ready..." + }, + "workflow": { + "heading": "Workflow integration currently supports Syslog, Kafka, and Webhook", + "body": "On the workflow detail page, open the Integration tab, choose Syslog, Kafka, or Webhook for your scenario, and then push data from the device or upstream system into the matching entry point.", + "requirementsTitle": "Requirements", + "requirement1": "Choose the integration type based on the data source: Syslog for log forwarding, Kafka for message queues, and Webhook for HTTP callbacks.", + "requirement2": "Make sure the workflow listener address, topic, or webhook URL is reachable from the device or upstream system, and map the input fields correctly.", + "requirement3": "After configuration, check the workflow execution history to confirm that the input data has been received and consumed.", + "goToWorkflows": "Go to workflow list" + }, + "actions": { + "backToSelection": "Back to mode selection", + "backToForm": "Back to details form", + "openSessionList": "Open in session list", + "submit": "Submit to Rex" + }, + "form": { + "common": { + "versionLabel": "Product version" + }, + "api": { + "deviceNameLabel": "Device product name", + "deviceNamePlaceholder": "e.g. Custom bastion host", + "vendorNameLabel": "Vendor name", + "vendorNamePlaceholder": "e.g. Acme Security", + "versionPlaceholder": "e.g. v3.2.1", + "baseUrlLabel": "Base URL", + "baseUrlPlaceholder": "e.g. https://device.example.com/api", + "docsUrlLabel": "API docs URL", + "docsUrlPlaceholder": "e.g. https://device.example.com/openapi", + "docsUrlHint": "A public URL is optional. After submission, you can keep uploading API docs inside the Rex conversation.", + "capabilitiesLabel": "Expected capability scope", + "capabilitiesPlaceholder": "All APIs by default, or limit to specific APIs", + "capabilitiesHint": "All APIs are used by default. If you only need specific interfaces, describe them here." + }, + "webcli": { + "deviceNameLabel": "Device product name", + "deviceNamePlaceholder": "e.g. Custom situation awareness platform", + "vendorNameLabel": "Vendor name", + "vendorNamePlaceholder": "e.g. Acme Security", + "versionPlaceholder": "e.g. 2026.05", + "productUrlLabel": "Product URL", + "productUrlPlaceholder": "e.g. https://device.example.com", + "targetInterfacesLabel": "Interfaces or page behaviors to capture", + "targetInterfacesPlaceholder": "e.g. alert list API, asset detail API, and the request triggered by the \"Block IP\" button", + "authHintLabel": "Authentication or permission hints", + "authHintPlaceholder": "e.g. admin role required; request depends on Cookie + X-CSRF-Token" + } + } + } +} diff --git a/webui/src/locales/en-US/home.json b/webui/src/locales/en-US/home.json index d1e54bf88..49248b103 100644 --- a/webui/src/locales/en-US/home.json +++ b/webui/src/locales/en-US/home.json @@ -3,6 +3,10 @@ "subtitle": "AI-Native SecOps Automation Platform", "description": "Let AI handle your security operations — from alert triage to threat response, fully automated with 10x efficiency", "getStarted": "Setup Guide", + "createUserDefinedPage": "Create Custom Page", + "createUserDefinedPageSessionTitle": "Create Custom Page", + "createUserDefinedPageInitialMessage": "I want to create a new user-defined page in the Flocks left navigation. Please introduce what this feature can do, where pages are stored, where they appear in the navigation, and the full creation and development workflow. Also explain how to hide a page from the navigation or permanently delete it when I no longer need it. If I already have an idea, guide me step by step: choose a page ID and title, create the page scaffold, write the React page code, and explain how live preview works after saving. Finally, tell me what information you still need from me (such as page name, content to display, and data sources).", + "createUserDefinedPageError": "Failed to create session. Please try again later.", "openSource": "Open Source", "systemCard": { "title": "Flocks System", diff --git a/webui/src/locales/en-US/session.json b/webui/src/locales/en-US/session.json index c88bc4c42..eb1118208 100644 --- a/webui/src/locales/en-US/session.json +++ b/webui/src/locales/en-US/session.json @@ -53,6 +53,18 @@ "custom": "Custom" } }, + "modelPicker": { + "title": "Choose Model", + "hint": "Overrides the model used when sending this chat message", + "empty": "No available models", + "count": "{{count}}", + "vision": "Vision", + "free": "Free", + "noCost": "No cost", + "contextWindow": "Context window: {{value}}", + "contextUnknown": "Context window: Unknown", + "addModel": "Add model" + }, "welcome": { "title": "Start a new conversation", "description": "AI-native security operations assistant — threat analysis, incident response, and security orchestration", @@ -71,6 +83,8 @@ "emptyText": "No execution records", "suggestions": "Example questions", "stopTitle": "Stop execution", + "contextUsageTitle": "Context used: {{used}} / {{total}} ({{percent}}%)", + "contextUsageUnknown": "Context usage unknown", "edit": "Edit", "editRawHint": "You are editing the original unrendered content. Saving updates the raw message text directly.", "editRawTitle": "Edit raw content", @@ -145,8 +159,20 @@ "error": "Failed", "inputParams": "Input Parameters", "outputResult": "Output Result", + "todoStages": "Todo stages", "errorLabel": "Error", - "elapsed": "Elapsed" + "elapsed": "Elapsed", + "workflowStage": "Current stage: {{phase}}", + "workflowNode": "Node: {{node}}", + "workflowStep": "Step: {{step}}", + "workflowPhase": { + "queued": "Queued", + "running": "Running", + "success": "Completed", + "error": "Failed", + "timeout": "Timed out", + "cancelled": "Cancelled" + } } }, "groupToday": "Today", diff --git a/webui/src/locales/en-US/socPage.json b/webui/src/locales/en-US/socPage.json new file mode 100644 index 000000000..d5424c691 --- /dev/null +++ b/webui/src/locales/en-US/socPage.json @@ -0,0 +1,19 @@ +{ + "host": { + "missingPageId": "Page ID is missing", + "loading": "Loading custom page...", + "unavailableTitle": "Page unavailable", + "notBuilt": "This page has not been built yet", + "loadFailed": "Failed to load page", + "buildFailed": "Page build failed", + "apiFailed": "Page API runtime failed", + "retry": "Retry", + "emptyComponent": "Page component is empty", + "renderFailedTitle": "Custom page failed to run", + "renderFailed": "Page render failed", + "bundleMissingExport": "Page bundle does not export a default component" + }, + "nav": { + "fetchFailed": "Failed to load custom page navigation" + } +} diff --git a/webui/src/locales/en-US/task.json b/webui/src/locales/en-US/task.json index 527886e82..55b7d9f28 100644 --- a/webui/src/locales/en-US/task.json +++ b/webui/src/locales/en-US/task.json @@ -131,7 +131,7 @@ "filterRunning": "Running", "filterStopped": "Stopped", "emptyTitle": "No API Services", - "emptyDescription": "Publish a workflow as an API service from the workflow detail page's \"Run\" tab", + "emptyDescription": "Publish a workflow as an API service from the workflow detail page's \"Publish\" tab", "publishedAt": "Published {{date}}", "serviceDriver": "Service Driver", "driverLocal": "Local", @@ -214,4 +214,4 @@ "running": "Running", "failedWeek": "Failed (7d)" } -} \ No newline at end of file +} diff --git a/webui/src/locales/en-US/userDefinedPage.json b/webui/src/locales/en-US/userDefinedPage.json new file mode 100644 index 000000000..efbee33ab --- /dev/null +++ b/webui/src/locales/en-US/userDefinedPage.json @@ -0,0 +1,18 @@ +{ + "host": { + "missingPageId": "Page ID is missing", + "loading": "Loading custom page...", + "unavailableTitle": "Page unavailable", + "notBuilt": "This page has not been built yet", + "loadFailed": "Failed to load page", + "buildFailed": "Page build failed", + "retry": "Retry", + "emptyComponent": "Page component is empty", + "renderFailedTitle": "Custom page failed to run", + "renderFailed": "Page render failed", + "bundleMissingExport": "Page bundle does not export a default component" + }, + "nav": { + "fetchFailed": "Failed to load custom page navigation" + } +} diff --git a/webui/src/locales/en-US/workflow.json b/webui/src/locales/en-US/workflow.json index 1360e35fa..6d71fe664 100644 --- a/webui/src/locales/en-US/workflow.json +++ b/webui/src/locales/en-US/workflow.json @@ -59,6 +59,7 @@ "phase": { "queued": "Queued", "running": "Running", + "cancelling": "Cancelling", "success": "Completed", "error": "Failed", "timeout": "Timed out", @@ -69,7 +70,7 @@ "tabOverview": "Overview", "tabChat": "AI Edit", "tabRun": "Run", - "tabIntegration": "Integration", + "tabIntegration": "Integrations", "renderError": "Component render error", "deleteWorkflow": "Delete Workflow", "deleteConfirmTitle": "Delete Workflow", @@ -199,6 +200,7 @@ "publishSection": "Publish as API", "publishFailed": "Publish failed", "stopFailed": "Stop failed", + "cancelRequested": "Cancellation requested. Waiting for the current step to stop.", "apiKeyShow": "Show", "apiKeyHide": "Hide", "curlExample": "Call Example (curl)", diff --git a/webui/src/locales/en-US/workspace.json b/webui/src/locales/en-US/workspace.json index 599d18161..afaf03fb8 100644 --- a/webui/src/locales/en-US/workspace.json +++ b/webui/src/locales/en-US/workspace.json @@ -26,6 +26,7 @@ "close": "Close", "uploading": "Uploading...", "binaryPreview": "Binary files cannot be previewed", + "truncatedPreview": "This file is large, so only the first {{limit}} is previewed. Inline editing is disabled to avoid saving truncated content; download the file for the full contents.", "downloadFile": "Download File", "toast": { "loadDirFailed": "Failed to load directory", diff --git a/webui/src/locales/zh-CN/agent.json b/webui/src/locales/zh-CN/agent.json index 3b902c2fe..121c8c96e 100644 --- a/webui/src/locales/zh-CN/agent.json +++ b/webui/src/locales/zh-CN/agent.json @@ -38,6 +38,8 @@ }, "form": { "name": "名称", + "nameCn": "中文名称", + "nameCnPlaceholder": "中文界面展示名", "description": "描述（英文）", "descriptionHint": "用于委派与模型上下文，请使用英文", "descriptionPlaceholder": "Short English description for delegation", diff --git a/webui/src/locales/zh-CN/common.json b/webui/src/locales/zh-CN/common.json index 7e2b9bb47..ce5f6374f 100644 --- a/webui/src/locales/zh-CN/common.json +++ b/webui/src/locales/zh-CN/common.json @@ -64,6 +64,7 @@ }, "workflowStatus": { "running": "运行中", + "cancelling": "取消中", "publishing": "发布中", "success": "成功", "SUCCEEDED": "成功", diff --git a/webui/src/locales/zh-CN/device.json b/webui/src/locales/zh-CN/device.json new file mode 100644 index 000000000..850788343 --- /dev/null +++ b/webui/src/locales/zh-CN/device.json @@ -0,0 +1,253 @@ +{ + "pageTitle": "设备接入", + "pageDescription": "配置安全设备 API 连接，使 Flocks 能够直接调用和控制这些设备", + + "status": { + "disabled": "已禁用", + "connected": "已连接", + "error": "连接失败", + "unknown": "未检测" + }, + + "vendor": { + "unspecified": "未指定厂商" + }, + + "sidebar": { + "heading": "机房", + "addRoom": "新建机房", + "allRooms": "全部机房", + "rename": "重命名", + "deleteRoom": "删除机房", + "deleteDisabled": "有 {{count}} 台设备，无法删除", + "roomNamePlaceholder": "机房名称...", + "deleteHasDevices": "机房「{{name}}」还有 {{count}} 台设备，请先转移或删除后再操作" + }, + + "toolbar": { + "refresh": "刷新", + "addDevice": "添加设备", + "collapseAll": "折叠全部", + "expandAll": "展开全部" + }, + + "header": { + "allRooms": "全部机房", + "deviceCount": "{{count}} 台设备 · {{rooms}} 个机房", + "connected": "{{count}} 已连接", + "failed": "{{count}} 失败", + "devices": "{{count}} 台设备" + }, + + "empty": { + "noDevices": "暂无已接入的设备", + "noDevicesHint": "添加设备后，Flocks Agent 即可调用对应工具", + "addNow": "立即添加设备", + "roomEmpty": "此机房暂无设备", + "roomEmptyHint": "点击「添加设备」将设备接入此机房" + }, + + "section": { + "activeDevices": "已接入设备", + "ungrouped": "未分组设备", + "ungroupedHint": "所属机房不存在，请编辑后重新指定" + }, + + "wizard": { + "title": "添加设备", + "closeAriaLabel": "关闭添加设备面板", + "step1": "1 选择厂商", + "step2": "2 选择设备", + "step3": "3 填写配置", + "modeTitle": "选择接入方式", + "step1Custom": "1 选择厂商 / 自定义", + "step2Custom": "2 选择设备 / 方式", + "step3Custom": "3 填写资料 / 配置", + "selectVendorTitle": "选择 {{vendor}} 设备", + "vendorHint": "选择设备所属厂商，共 {{count}} 家", + "chooseVendorOrCustom": "选择设备所属厂商，或创建自定义设备接入", + "customCardTitle": "自定义设备", + "customCardSubtitle": "API / WebCLI / Workflow", + "customCardCta": "创建新的接入方式", + "chooseCustomMode": "请选择自定义设备的接入方式", + "customModes": { + "api": { + "title": "API 接入", + "desc": "设备提供 API 能力时使用。需要提供 API 文档，最终生成可复用的 device 插件。" + }, + "webcli": { + "title": "WebCLI 接入", + "desc": "设备没有开放 API 时使用。先生成并集成 skill/CLI 资产；如果是安全设备场景，再额外生成可在设备页使用的 device 插件。" + }, + "workflow": { + "title": "Workflow 接入", + "desc": "适用于 Syslog、Kafka、Webhook。设备页会引导你前往工作流发布页面完成对应入口配置。" + } + }, + "productCount": "{{count}} 种设备", + "instanceCount": "已接入 {{count}} 台", + "installState": { + "install": "请先在 FlockHub 安装", + "update": "请先在 FlockHub 更新", + "broken": "插件不可用", + "installed": "已安装", + "available": "可安装", + "updateAvailable": "可更新", + "brokenShort": "不可用" + }, + "productHint": "共 {{count}} 款设备，同款设备可多次接入" + }, + + "config": { + "closeAriaLabel": "关闭设备配置面板", + "newDeviceTitle": "填写配置", + "tabConfig": "配置", + "tabTools": "工具", + "tabToolsCount": "工具 ({{count}})", + "tabOverview": "概览", + "nameLabel": "设备名称", + "namePlaceholder": "例如：总部 AF 防火墙", + "roomLabel": "所属机房", + "connectionParams": "连接参数", + "secretConfigured": "已配置 · 保持不变请勿修改，清空则删除", + "secretRevealFailed": "读取密钥失败", + "showSecretAction": "显示", + "hideSecretAction": "隐藏", + "showSecretAria": "显示{{label}}", + "hideSecretAria": "隐藏{{label}}", + "sslLabel": "SSL 验证", + "sslHint": "关闭可访问自签名证书的内网设备", + "enabledLabel": "启用设备", + "enabledHint": "关闭后 Agent 不会调用此设备的工具", + "testBtn": "连通测试", + "saveBtn": "保存配置", + "addBtn": "确认接入", + "deleteBtn": "删除设备", + "confirmDelete": "确认删除此设备配置？" + }, + + "tools": { + "empty": "暂无关联工具", + "colName": "工具名称", + "colDesc": "描述", + "colStatus": "状态", + "colAction": "操作", + "detail": "测试 / 详情" + }, + + "overview": { + "serviceName": "服务名称", + "version": "版本", + "toolCount": "工具数量", + "vendor": "厂商", + "serviceDesc": "服务简介", + "viewDocs": "查看 API 文档" + }, + + "toast": { + "loadFailed": "加载失败", + "nameRequired": "请填写设备名称", + "saveDone": "配置已保存", + "addDone": "设备已添加", + "saveFailed": "保存失败", + "sslOn": "已开启 SSL 验证", + "sslOff": "已关闭 SSL 验证", + "rollback": "保存失败，已回滚", + "enabledOn": "设备已启用", + "enabledOff": "设备已停用", + "deleteDone": "已删除设备", + "deleteFailed": "删除失败", + "actionFailed": "操作失败" + }, + + "custom": { + "title": { + "api": "自定义设备 API 接入", + "webcli": "自定义设备 WebCLI 接入", + "workflow": "自定义设备 Workflow 接入" + }, + "subtitle": { + "api": "提供 API 文档，生成可复用的 device 插件", + "webcli": "提供产品 URL 和目标接口，生成可在设备页使用的 WebCLI device 插件", + "workflow": "Syslog、Kafka、Webhook 统一在工作流发布页面配置" + }, + "welcome": { + "api": "请补充 API 文档链接或直接上传文档文件，我会用 tool-builder skill 帮你生成可在设备页使用的 device 插件。", + "webcli": "请补充产品 URL、目标接口和认证提示，我会用 web2cli skill 先生成并集成 CLI/skill 资产；如果需要，再额外生成 device 插件。", + "workflow": "Workflow 接入不在这里创建插件，请前往工作流发布页面，根据需要配置 Syslog、Kafka 或 Webhook。" + }, + "validation": { + "deviceNameRequired": "请填写设备产品名", + "vendorNameRequired": "请填写厂商名称", + "baseUrlRequired": "请填写 Base URL", + "productUrlRequired": "请填写产品 URL", + "targetInterfacesRequired": "请填写需要获取的接口或页面行为" + }, + "toast": { + "submitSuccess": "已提交给 Rex，请继续完成插件生成", + "submitFailed": "提交给 Rex 失败" + }, + "details": { + "prepareTitle": "提交给 Rex 前请准备好接入资料", + "prepareIntro": "提交后会直接进入 Rex 对话。", + "apiNext": "你可以继续补充 API 文档链接、上传文档文件或说明接口细节。插件生成完成后，可返回设备页继续后续配置。", + "webcliNext": "你可以继续补充页面操作、抓包目标和认证方式。最终结果应当是可在设备页识别的 WebCLI device 插件；如果需要，也可以额外保留 CLI 作为调试入口。" + }, + "rex": { + "apiHint": "上传 API 文档或描述接口；Rex 会先澄清关键缺口，确认后再生成 device 插件。", + "webcliHint": "提供产品 URL 和目标页面/操作；Rex 会先澄清登录、权限和风险操作，再生成接入资产。", + "placeholder": "继续补充接口说明、认证细节或调试信息", + "apiPlaceholder": "请提供产品 API 文档", + "webcliPlaceholder": "请提供网站地址", + "pending": "Rex 准备中..." + }, + "workflow": { + "heading": "Workflow 接入目前支持 Syslog、Kafka、Webhook", + "body": "你可以在工作流详情页的 Integration 标签中，按实际场景选择 Syslog、Kafka 或 Webhook，然后把设备或外部系统的数据推送到对应入口。", + "requirementsTitle": "配置要求", + "requirement1": "根据数据来源选择接入方式：日志转发选 Syslog，消息队列选 Kafka，HTTP 回调选 Webhook。", + "requirement2": "确认工作流监听地址、Topic 或 Webhook URL 能从设备侧或上游系统访问，并正确映射输入字段。", + "requirement3": "配置完成后，到工作流执行历史中确认是否已经收到并消费对应输入数据。", + "goToWorkflows": "前往工作流列表" + }, + "actions": { + "backToSelection": "返回选择方式", + "backToForm": "返回资料填写", + "openSessionList": "前往会话列表查看", + "submit": "提交给 Rex" + }, + "form": { + "common": { + "versionLabel": "产品版本" + }, + "api": { + "deviceNameLabel": "设备产品名", + "deviceNamePlaceholder": "例如：自定义堡垒机", + "vendorNameLabel": "厂商名称", + "vendorNamePlaceholder": "例如：Acme Security", + "versionPlaceholder": "例如：v3.2.1", + "baseUrlLabel": "Base URL", + "baseUrlPlaceholder": "例如：https://device.example.com/api", + "docsUrlLabel": "API 文档链接", + "docsUrlPlaceholder": "例如：https://device.example.com/openapi", + "docsUrlHint": "没有公开链接也没关系，提交后可在 Rex 对话中继续上传 API 文档。", + "capabilitiesLabel": "期望接入的能力范围", + "capabilitiesPlaceholder": "默认全部 API，可选定需要的 API", + "capabilitiesHint": "默认全部 API；如果你只想接特定接口，可以在这里指定需要的 API。" + }, + "webcli": { + "deviceNameLabel": "设备产品名", + "deviceNamePlaceholder": "例如：自定义态势平台", + "vendorNameLabel": "厂商名称", + "vendorNamePlaceholder": "例如：Acme Security", + "versionPlaceholder": "例如：2026.05", + "productUrlLabel": "产品 URL", + "productUrlPlaceholder": "例如：https://device.example.com", + "targetInterfacesLabel": "需要获取的接口或页面行为", + "targetInterfacesPlaceholder": "例如：抓取告警列表接口、资产详情接口，以及“封禁 IP”按钮对应请求", + "authHintLabel": "认证/权限提示", + "authHintPlaceholder": "例如：需要管理员角色；接口依赖 Cookie + X-CSRF-Token" + } + } + } +} diff --git a/webui/src/locales/zh-CN/home.json b/webui/src/locales/zh-CN/home.json index 4ac1b82d9..c5693e857 100644 --- a/webui/src/locales/zh-CN/home.json +++ b/webui/src/locales/zh-CN/home.json @@ -3,6 +3,10 @@ "subtitle": "AI 原生的安全运营自动化平台", "description": "让 AI 替你做安全运营，从告警研判到威胁处置，全流程智能化，效率提升 10 倍", "getStarted": "新手引导", + "createUserDefinedPage": "创建自定义页面", + "createUserDefinedPageSessionTitle": "创建自定义页面", + "createUserDefinedPageInitialMessage": "我想在 Flocks 左侧导航中创建一个新的用户自定义页面。请先帮我介绍这个功能能做什么、页面会保存在哪里、创建后会出现在导航的什么位置，以及完整的创建与开发流程。也请说明如果不再需要某个页面，如何从导航隐藏或彻底删除。如果我已经有具体想法，也请引导我一步步完成：确定页面 ID 和标题、创建页面骨架、编写 React 页面代码，并说明保存后如何实时预览。最后请告诉我，接下来你只需要我提供哪些信息（例如页面名称、展示内容、数据来源）。", + "createUserDefinedPageError": "无法创建会话，请稍后重试", "openSource": "开源项目", "systemCard": { "title": "Flocks 系统", diff --git a/webui/src/locales/zh-CN/session.json b/webui/src/locales/zh-CN/session.json index 6a949bb77..0282fe9da 100644 --- a/webui/src/locales/zh-CN/session.json +++ b/webui/src/locales/zh-CN/session.json @@ -53,6 +53,18 @@ "custom": "自定义" } }, + "modelPicker": { + "title": "选择模型", + "hint": "作为本次对话发送时的模型覆盖", + "empty": "暂无可用模型", + "count": "{{count}} 个", + "vision": "视觉", + "free": "免费", + "noCost": "暂无费用", + "contextWindow": "上下文窗口：{{value}}", + "contextUnknown": "上下文窗口：未知", + "addModel": "添加模型" + }, "welcome": { "title": "开始新的对话", "description": "AI 原生的安全运营助手，帮您进行威胁分析、应急响应、安全编排等任务", @@ -71,6 +83,8 @@ "emptyText": "暂无执行记录", "suggestions": "示例问题", "stopTitle": "停止执行", + "contextUsageTitle": "上下文已用：{{used}} / {{total}}（{{percent}}%）", + "contextUsageUnknown": "上下文使用量未知", "edit": "编辑", "editRawHint": "当前显示的是原始未渲染内容，保存后会直接更新这条消息文本。", "editRawTitle": "编辑原始内容", @@ -145,8 +159,20 @@ "error": "失败", "inputParams": "输入参数", "outputResult": "输出结果", + "todoStages": "Todo 阶段", "errorLabel": "错误", - "elapsed": "耗时" + "elapsed": "耗时", + "workflowStage": "当前阶段：{{phase}}", + "workflowNode": "节点：{{node}}", + "workflowStep": "步骤：{{step}}", + "workflowPhase": { + "queued": "排队中", + "running": "执行中", + "success": "已完成", + "error": "执行失败", + "timeout": "超时", + "cancelled": "已取消" + } } }, "groupToday": "今天", diff --git a/webui/src/locales/zh-CN/socPage.json b/webui/src/locales/zh-CN/socPage.json new file mode 100644 index 000000000..b41ddf059 --- /dev/null +++ b/webui/src/locales/zh-CN/socPage.json @@ -0,0 +1,19 @@ +{ + "host": { + "missingPageId": "未指定页面 ID", + "loading": "正在加载自定义页面...", + "unavailableTitle": "页面暂不可用", + "notBuilt": "页面尚未构建完成", + "loadFailed": "加载页面失败", + "buildFailed": "页面构建失败", + "apiFailed": "页面 API 运行失败", + "retry": "重试", + "emptyComponent": "页面组件为空", + "renderFailedTitle": "自定义页面运行失败", + "renderFailed": "页面渲染失败", + "bundleMissingExport": "页面 bundle 未导出 default 组件" + }, + "nav": { + "fetchFailed": "加载自定义页面导航失败" + } +} diff --git a/webui/src/locales/zh-CN/task.json b/webui/src/locales/zh-CN/task.json index d838a84f4..aafc04e0b 100644 --- a/webui/src/locales/zh-CN/task.json +++ b/webui/src/locales/zh-CN/task.json @@ -131,7 +131,7 @@ "filterRunning": "运行中", "filterStopped": "已停止", "emptyTitle": "暂无 API 服务", - "emptyDescription": "在工作流详情页「运行」标签中发布工作流为 API 服务", + "emptyDescription": "在工作流详情页「发布」标签中发布工作流为 API 服务", "publishedAt": "{{date}} 发布", "serviceDriver": "服务驱动", "driverLocal": "Local", @@ -214,4 +214,4 @@ "running": "进行中", "failedWeek": "7日内失败" } -} \ No newline at end of file +} diff --git a/webui/src/locales/zh-CN/userDefinedPage.json b/webui/src/locales/zh-CN/userDefinedPage.json new file mode 100644 index 000000000..cfde1a4a7 --- /dev/null +++ b/webui/src/locales/zh-CN/userDefinedPage.json @@ -0,0 +1,18 @@ +{ + "host": { + "missingPageId": "未指定页面 ID", + "loading": "正在加载自定义页面...", + "unavailableTitle": "页面暂不可用", + "notBuilt": "页面尚未构建完成", + "loadFailed": "加载页面失败", + "buildFailed": "页面构建失败", + "retry": "重试", + "emptyComponent": "页面组件为空", + "renderFailedTitle": "自定义页面运行失败", + "renderFailed": "页面渲染失败", + "bundleMissingExport": "页面 bundle 未导出 default 组件" + }, + "nav": { + "fetchFailed": "加载自定义页面导航失败" + } +} diff --git a/webui/src/locales/zh-CN/workflow.json b/webui/src/locales/zh-CN/workflow.json index 46961e1ad..029008d6f 100644 --- a/webui/src/locales/zh-CN/workflow.json +++ b/webui/src/locales/zh-CN/workflow.json @@ -59,6 +59,7 @@ "phase": { "queued": "排队中", "running": "执行中", + "cancelling": "取消中", "success": "已完成", "error": "执行失败", "timeout": "超时", @@ -199,6 +200,7 @@ "publishSection": "发布为 API", "publishFailed": "发布失败", "stopFailed": "停止失败", + "cancelRequested": "已请求停止，正在等待当前步骤响应取消", "apiKeyShow": "显示", "apiKeyHide": "隐藏", "curlExample": "调用示例（curl）", diff --git a/webui/src/locales/zh-CN/workspace.json b/webui/src/locales/zh-CN/workspace.json index 8066f22d7..dbdc9fa52 100644 --- a/webui/src/locales/zh-CN/workspace.json +++ b/webui/src/locales/zh-CN/workspace.json @@ -26,6 +26,7 @@ "close": "关闭", "uploading": "上传中...", "binaryPreview": "二进制文件无法预览", + "truncatedPreview": "文件较大，当前仅预览前 {{limit}} 内容。为避免误保存截断内容，已禁用在线编辑；如需完整内容请下载文件。", "downloadFile": "下载文件", "toast": { "loadDirFailed": "加载目录失败", diff --git a/webui/src/pages/Agent/AgentFormDialogs.tsx b/webui/src/pages/Agent/AgentFormDialogs.tsx index a7017a9f8..ccfdd8ccc 100644 --- a/webui/src/pages/Agent/AgentFormDialogs.tsx +++ b/webui/src/pages/Agent/AgentFormDialogs.tsx @@ -16,6 +16,7 @@ export function CreateAgentDialog({ const { t } = useTranslation(['agent', 'common']); const [formData, setFormData] = useState({ name: '', + nameCn: '', description: '', descriptionCn: '', prompt: '', @@ -77,6 +78,7 @@ export function EditAgentDialog({ const { t } = useTranslation(['agent', 'common']); const [formData, setFormData] = useState({ name: agent.name, + nameCn: agent.nameCn ?? '', description: agent.description ?? '', descriptionCn: agent.descriptionCn ?? '', prompt: agent.prompt ?? '', @@ -94,6 +96,7 @@ export function EditAgentDialog({ try { setLoading(true); await agentAPI.update(agent.name, { + nameCn: formData.nameCn, description: formData.description || undefined, descriptionCn: formData.descriptionCn || undefined, prompt: formData.prompt, @@ -131,6 +134,7 @@ export function EditAgentDialog({ interface FormData { name: string; + nameCn: string; description: string; descriptionCn: string; prompt: string; @@ -190,6 +194,17 @@ function AgentFormModal({ )}
+
+ + onChange({ ...formData, nameCn: e.target.value })} + className="w-full px-4 py-2 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-slate-400 text-sm" + placeholder={t('agent:form.nameCnPlaceholder')} + /> +
+

{t('agent:form.descriptionHint')}

diff --git a/webui/src/pages/Agent/AgentSheet.tsx b/webui/src/pages/Agent/AgentSheet.tsx index 0fceaff99..e768f0768 100644 --- a/webui/src/pages/Agent/AgentSheet.tsx +++ b/webui/src/pages/Agent/AgentSheet.tsx @@ -16,12 +16,14 @@ import { sessionApi } from '@/api/session'; import client from '@/api/client'; import EntitySheet, { useEntitySheet } from '@/components/common/EntitySheet'; import PillGroup from '@/components/common/PillGroup'; -import { providerAPI, defaultModelAPI } from '@/api/provider'; +import { providerAPI, defaultModelAPI, modelV2API } from '@/api/provider'; import { toolAPI, Tool } from '@/api/tool'; import { skillAPI, Skill } from '@/api/skill'; +import type { ModelDefinitionV2 } from '@/types'; interface AvailableModel { providerID: string; + providerName: string; modelID: string; label: string; } @@ -30,6 +32,7 @@ interface AvailableModel { interface AgentFormData { name: string; + nameCn: string; description: string; descriptionCn: string; prompt: string; @@ -57,6 +60,7 @@ export default function AgentSheet({ agent, onClose, onSaved }: AgentSheetProps) const [formData, setFormData] = useState({ name: agent?.name ?? '', + nameCn: agent?.nameCn ?? '', description: agent?.description ?? '', descriptionCn: agent?.descriptionCn ?? '', prompt: agent?.prompt ?? '', @@ -68,6 +72,7 @@ export default function AgentSheet({ agent, onClose, onSaved }: AgentSheetProps) }); const [loading, setLoading] = useState(false); const [availableModels, setAvailableModels] = useState([]); + const [modelsLoaded, setModelsLoaded] = useState(false); const [defaultModel, setDefaultModel] = useState<{ providerID: string; modelID: string } | null>(null); const [allTools, setAllTools] = useState([]); const [allSkills, setAllSkills] = useState([]); @@ -78,17 +83,30 @@ export default function AgentSheet({ agent, onClose, onSaved }: AgentSheetProps) const isPrimary = formData.mode === 'primary'; useEffect(() => { - providerAPI.list().then((r) => { - const connectedSet = new Set(r.data.connected ?? []); - const list: AvailableModel[] = []; - for (const provider of r.data.all) { - if (!connectedSet.has(provider.id)) continue; - for (const [modelId, modelInfo] of Object.entries(provider.models ?? {})) { - list.push({ providerID: provider.id, modelID: modelId, label: (modelInfo as any).name || modelId }); - } - } + Promise.all([ + providerAPI.list(), + modelV2API.listDefinitions({ enabled_only: true }), + ]).then(([providersRes, modelsRes]) => { + const connectedSet = new Set(providersRes.data.connected ?? []); + const providerById = new Map( + providersRes.data.all + .filter((provider) => connectedSet.has(provider.id)) + .map((provider) => [provider.id, provider]), + ); + const enabledModels = (modelsRes.data.models ?? []) as ModelDefinitionV2[]; + const list: AvailableModel[] = enabledModels.flatMap((model) => { + const provider = providerById.get(model.provider_id); + if (!provider) return []; + return [{ + providerID: provider.id, + providerName: provider.name || provider.id, + modelID: model.id, + label: model.name || model.id, + }]; + }); setAvailableModels(list); - }).catch(() => {}); + }).catch(() => setAvailableModels([])) + .finally(() => setModelsLoaded(true)); defaultModelAPI.getResolved().then((r) => { const d = r.data; @@ -113,6 +131,16 @@ export default function AgentSheet({ agent, onClose, onSaved }: AgentSheetProps) }).catch(() => {}).finally(() => setSkillsLoading(false)); }, []); + useEffect(() => { + if (!modelsLoaded || !formData.modelKey) return; + const selectedStillAvailable = availableModels.some( + (model) => `${model.providerID}::${model.modelID}` === formData.modelKey, + ); + if (!selectedStillAvailable) { + setFormData((prev) => ({ ...prev, modelKey: '' })); + } + }, [availableModels, formData.modelKey, modelsLoaded]); + const isNative = !!agent?.native; const submitDisabled = false; @@ -137,6 +165,7 @@ export default function AgentSheet({ agent, onClose, onSaved }: AgentSheetProps) await agentAPI.updateModel(agent!.name, model ?? null, formData.temperature); } else { await agentAPI.update(agent!.name, { + nameCn: formData.nameCn, description: formData.description || undefined, descriptionCn: formData.descriptionCn || undefined, prompt: formData.prompt, @@ -204,6 +233,12 @@ export default function AgentSheet({ agent, onClose, onSaved }: AgentSheetProps) setFormData((prev) => ({ ...prev, // preserve tools, skills, modelKey name: config.name || prev.name, + nameCn: + (typeof config.name_cn === 'string' + ? config.name_cn + : typeof config.nameCn === 'string' + ? config.nameCn + : prev.nameCn), description: config.description ?? prev.description, descriptionCn: (typeof config.description_cn === 'string' @@ -303,6 +338,10 @@ function AgentFormContent({ acc[m.providerID].push(m); return acc; }, {}); + const providerLabelById = availableModels.reduce>((acc, m) => { + acc[m.providerID] = m.providerName; + return acc; + }, {}); const defaultModelLabel = defaultModel ? `${defaultModel.modelID} (${defaultModel.providerID})` @@ -359,6 +398,25 @@ function AgentFormContent({ )}
+ {/* Chinese display name */} +
+ + update({ nameCn: e.target.value })} + disabled={nativeReadOnly} + className={`w-full px-4 py-2 border rounded-lg text-sm ${ + nativeReadOnly + ? 'border-gray-300 bg-gray-100 text-gray-500 cursor-not-allowed' + : 'border-gray-300 focus:outline-none focus:ring-2 focus:ring-slate-400' + }`} + placeholder={t('form.nameCnPlaceholder')} + /> +
+ {/* Description (English) + Chinese UI */}
@@ -449,7 +507,7 @@ function AgentFormContent({ > {Object.entries(modelsByProvider).map(([pID, pModels]) => ( - + {pModels.map((m) => (
@@ -741,17 +905,17 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl {serviceTools.length === 0 ? (
-

暂无关联工具

+

{t('tools.empty')}

) : (
- - - - + + + + @@ -772,10 +936,10 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl @@ -793,10 +957,10 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl
{[ - { label: '服务名称', value: metadata?.name || serviceId }, - metadata?.version ? { label: '版本', value: metadata.version } : null, - { label: '工具数量', value: String(serviceTools.length) }, - vendor ? { label: '厂商', value: vendor.nameCn } : null, + { label: t('overview.serviceName'), value: metadata?.name || serviceId }, + metadata?.version ? { label: t('overview.version'), value: metadata.version } : null, + { label: t('overview.toolCount'), value: String(serviceTools.length) }, + vendor ? { label: t('overview.vendor'), value: i18n.language.startsWith('zh') ? vendor.nameCn : vendor.nameEn } : null, device?.storage_key ? { label: 'Storage Key', value: device.storage_key } : null, device?.service_id ? { label: 'Service ID', value: device.service_id } : null, ].filter(Boolean).map((row) => ( @@ -809,7 +973,7 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl {(metadata?.description_cn || metadata?.description) && (
-

服务简介

+

{t('overview.serviceDesc')}

{metadata?.description_cn || metadata?.description}

@@ -824,7 +988,7 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl className="flex items-center gap-2 text-sm text-blue-600 hover:text-blue-800 px-1" > - 查看 API 文档 + {t('overview.viewDocs')} )}
@@ -837,6 +1001,7 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl setToolModal(null)} /> )} @@ -845,87 +1010,273 @@ function DeviceConfigPanel({ device, template, vendorKey, onSave, onDelete, onCl } // ============================================================================ -// Group banner (inline rename for the single default room) +// Group sidebar — left panel for room navigation & management // ============================================================================ -function GroupBanner({ group, onRenamed }: { - group: DeviceGroup | undefined; - onRenamed: () => void; +type RoomStatus = 'ok' | 'partial' | 'empty'; + +function GroupSidebar({ groups, devices, selectedGroupId, onSelect, onRename, onDelete, onCreate }: { + groups: DeviceGroup[]; + devices: DeviceIntegration[]; + selectedGroupId: string | null; + onSelect: (id: string | null) => void; + onRename: (id: string, newName: string) => Promise; + onDelete: (id: string) => Promise; + onCreate: (name: string) => Promise; }) { const toast = useToast(); - const [editing, setEditing] = useState(false); - const [draft, setDraft] = useState(group?.name ?? ''); - const [saving, setSaving] = useState(false); + const { t } = useTranslation('device'); + const [editingId, setEditingId] = useState(null); + const [editDraft, setEditDraft] = useState(''); + const [creating, setCreating] = useState(false); + const [createDraft, setCreateDraft] = useState(''); + const [busy, setBusy] = useState(false); + + // Device counts per group + const deviceCounts = useMemo(() => { + const c: Record = {}; + devices.forEach((d) => { c[d.group_id] = (c[d.group_id] || 0) + 1; }); + return c; + }, [devices]); - useEffect(() => { setDraft(group?.name ?? ''); }, [group?.name]); + // Room connectivity status + const groupStatuses = useMemo((): Record => { + const s: Record = {}; + groups.forEach((g) => { + const gd = devices.filter((d) => d.group_id === g.id && d.enabled); + if (gd.length === 0) { s[g.id] = 'empty'; return; } + const ok = gd.filter((d) => d.status === 'ok' || d.status === 'connected').length; + s[g.id] = ok === gd.length ? 'ok' : 'partial'; + }); + return s; + }, [groups, devices]); - if (!group) return null; + const statusDotClass: Record = { + ok: 'bg-green-500', + partial: 'bg-yellow-400', + empty: 'bg-zinc-300', + }; - const startEdit = () => { setDraft(group.name); setEditing(true); }; - const cancelEdit = () => { setDraft(group.name); setEditing(false); }; - const saveEdit = async () => { - const next = draft.trim(); - if (!next || next === group.name) { cancelEdit(); return; } - setSaving(true); + const startEdit = (g: DeviceGroup, e: React.MouseEvent) => { + e.stopPropagation(); + setEditingId(g.id); + setEditDraft(g.name); + setCreating(false); + }; + + const cancelEdit = () => { setEditingId(null); setEditDraft(''); }; + + const saveEdit = async (groupId: string) => { + const next = editDraft.trim(); + const g = groups.find((x) => x.id === groupId); + if (!next || next === g?.name) { cancelEdit(); return; } + setBusy(true); try { - await deviceAPI.updateGroup(group.id, { name: next }); - toast.success('机房名称已更新'); - setEditing(false); - onRenamed(); + await onRename(groupId, next); + setEditingId(null); } catch { - toast.error('更新失败'); + // error already toasted by parent } finally { - setSaving(false); + setBusy(false); } }; + const startCreate = () => { + setCreating(true); + setCreateDraft(''); + setEditingId(null); + }; + + const cancelCreate = () => { setCreating(false); setCreateDraft(''); }; + + const saveCreate = async () => { + const name = createDraft.trim(); + if (!name) { cancelCreate(); return; } + setBusy(true); + try { + await onCreate(name); + cancelCreate(); + } catch { + // error already toasted by parent; keep input open so user can retry + } finally { + setBusy(false); + } + }; + + const handleDeleteClick = async (group: DeviceGroup, e: React.MouseEvent) => { + e.stopPropagation(); + const count = deviceCounts[group.id] || 0; + if (count > 0) { + toast.error(t('sidebar.deleteHasDevices', { name: group.name, count })); + return; + } + await onDelete(group.id); + }; + return ( -
- - 当前机房 - {editing ? ( - <> - setDraft(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') void saveEdit(); - if (e.key === 'Escape') cancelEdit(); - }} - disabled={saving} - maxLength={40} - className="text-sm font-medium text-zinc-800 bg-white border border-zinc-200 rounded-md px-2 py-1 focus:border-blue-300 focus:outline-none focus:ring-2 focus:ring-blue-100 w-48" - /> - - - - ) : ( - <> - {group.name} - - - )} + // self-stretch ensures the panel fills the full flex-row height so the + // right border reaches the bottom even when there are few rooms. +
+ {/* Header */} +
+ {t('sidebar.heading')} + +
+ + {/* Divider */} +
+ + {/* Scrollable list */} +
+ {/* "全部机房" item */} + + +
+ + {/* Individual rooms */} + {groups.map((group) => { + const count = deviceCounts[group.id] || 0; + const st = groupStatuses[group.id] || 'empty'; + const isSelected = selectedGroupId === group.id; + const isEditing = editingId === group.id; + const isDefault = group.id === DEFAULT_GROUP_ID; + + return ( +
{ if (!isEditing) onSelect(group.id); }} + > + {/* Status dot */} + + + {isEditing ? ( + <> + setEditDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') void saveEdit(group.id); + if (e.key === 'Escape') cancelEdit(); + e.stopPropagation(); + }} + onClick={(e) => e.stopPropagation()} + disabled={busy} + maxLength={40} + className="flex-1 min-w-0 text-sm text-zinc-900 bg-white border border-zinc-300 rounded px-1.5 py-0.5 focus:outline-none focus:border-blue-400" + /> +
e.stopPropagation()}> + + +
+ + ) : ( + <> + {group.name} + + {count} + + + {/* Hover action buttons */} +
+ + {!isDefault && ( + + )} +
+ + )} +
+ ); + })} + + {/* Inline new room form */} + {creating && ( +
+ + setCreateDraft(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') void saveCreate(); + if (e.key === 'Escape') cancelCreate(); + }} + disabled={busy} + placeholder={t('sidebar.roomNamePlaceholder')} + maxLength={40} + className="flex-1 min-w-0 text-sm text-zinc-900 bg-white border border-zinc-300 rounded px-1.5 py-0.5 focus:outline-none focus:border-blue-400" + /> +
+ + +
+
+ )} +
); } @@ -935,36 +1286,65 @@ function GroupBanner({ group, onRenamed }: { // ============================================================================ type PanelMode = + | { kind: 'pick-group' } | { kind: 'wizard'; initialVendor?: DeviceVendor } - | { kind: 'add'; template: APIServiceSummary } + | { kind: 'add'; template: DeviceTemplate } + | { kind: 'custom'; mode: CustomDeviceAccessMode } | { kind: 'edit'; device: DeviceIntegration } | null; export default function DeviceIntegrationPage() { const toast = useToast(); + const { t } = useTranslation('device'); const [devices, setDevices] = useState([]); - const [templates, setTemplates] = useState([]); + const [templates, setTemplates] = useState([]); const [groups, setGroups] = useState([]); const [loading, setLoading] = useState(true); const [refreshing, setRefreshing] = useState(false); const [panel, setPanel] = useState(null); + // null = "全部机房" aggregate view; string = specific group id + const [selectedGroupId, setSelectedGroupId] = useState(null); + // Group ids whose section is collapsed in the "全部机房" view. Default + // (absent) = expanded, so brand-new rooms show their devices immediately. + const [collapsedGroups, setCollapsedGroups] = useState>(new Set()); + + const toggleGroupCollapsed = useCallback((groupId: string) => { + setCollapsedGroups((prev) => { + const next = new Set(prev); + if (next.has(groupId)) next.delete(groupId); + else next.add(groupId); + return next; + }); + }, []); + + const selectedGroup = useMemo( + () => groups.find((g) => g.id === selectedGroupId) ?? null, + [groups, selectedGroupId], + ); - const currentGroup: DeviceGroup | undefined = groups[0]; + // Devices shown in the main area (filtered by selected room) + const filteredDevices = useMemo( + () => selectedGroupId ? devices.filter((d) => d.group_id === selectedGroupId) : devices, + [devices, selectedGroupId], + ); - const fetchData = useCallback(async (silent = false) => { + const fetchData = useCallback(async (silent = false, refreshTemplates = false): Promise => { if (!silent) setLoading(true); else setRefreshing(true); try { const [devRes, tplRes, grpRes] = await Promise.all([ - deviceAPI.list(), - providerAPI.listApiServices(), + deviceAPI.list(refreshTemplates ? { refresh: true } : undefined), + deviceAPI.listTemplates(refreshTemplates ? { refresh: true } : undefined), deviceAPI.listGroups(), ]); + const nextTemplates = tplRes.data || []; setDevices(devRes.data || []); - setTemplates((tplRes.data || []).filter((s) => s.integration_type === 'device')); + setTemplates(nextTemplates); setGroups(grpRes.data || []); + return nextTemplates; } catch { - toast.error('加载失败'); + toast.error(t('toast.loadFailed')); + return []; } finally { setLoading(false); setRefreshing(false); @@ -980,24 +1360,13 @@ export default function DeviceIntegrationPage() { return counts; }, [devices]); - // storage_key (and bare service_id) → vendor key, sourced from the backend - // template list. Used to resolve vendor for already-installed devices - // (whose `DeviceIntegration` row does not carry the vendor field directly). - // - // Legacy devices may have been installed before api_versioning shipped, so - // their `storage_key` is the bare service_id (e.g. "tdp_api") rather than - // the versioned form ("tdp_api_v3_3_10"). We additionally index each - // template by its bare service_id (regex matches the backend's - // `storage_key_to_service_id`) so those rows still resolve correctly. + // storage_key / service_id → vendor key mapping const vendorByKey = useMemo(() => { const map: Record = {}; templates.forEach((t) => { if (!t.vendor) return; - map[t.id] = t.vendor; - const bareServiceId = t.id.replace(/_v[\w.]+$/i, ''); - if (bareServiceId !== t.id && !map[bareServiceId]) { - map[bareServiceId] = t.vendor; - } + map[t.storage_key] = t.vendor; + map[t.service_id] = t.vendor; }); return map; }, [templates]); @@ -1010,12 +1379,64 @@ export default function DeviceIntegrationPage() { const panelDeviceId = panel?.kind === 'edit' ? panel.device.id : null; - const handleSave = async (data: { name: string; fields: Record; enabled: boolean; verify_ssl: boolean }) => { + // ────────────────────────────────────────────────────────────────────────── + // Group CRUD handlers + // ────────────────────────────────────────────────────────────────────────── + + // These three re-throw on failure (after toasting) so GroupSidebar's inline + // edit/create forms know to stay open for a retry instead of silently + // closing on a 409 (duplicate name) etc. + const handleCreateGroup = async (name: string) => { + try { + const res = await deviceAPI.createGroup({ name }); + await fetchData(true); + setSelectedGroupId(res.data.id); // auto-select the newly created room + toast.success(`机房「${name}」已创建`); + } catch (err: unknown) { + toast.error(errDetail(err, '创建机房失败')); + throw err; + } + }; + + const handleRenameGroup = async (id: string, newName: string) => { + try { + await deviceAPI.updateGroup(id, { name: newName }); + await fetchData(true); + toast.success('机房名称已更新'); + } catch (err: unknown) { + toast.error(errDetail(err, '重命名失败')); + throw err; + } + }; + + const handleDeleteGroup = async (id: string) => { + try { + await deviceAPI.deleteGroup(id); + if (selectedGroupId === id) setSelectedGroupId(null); + await fetchData(true); + toast.success('机房已删除'); + } catch (err: unknown) { + toast.error(errDetail(err, '删除失败')); + } + }; + + // ────────────────────────────────────────────────────────────────────────── + // Device CRUD handlers + // ────────────────────────────────────────────────────────────────────────── + + const handleSave = async (data: { + name: string; + fields: Record; + enabled: boolean; + verify_ssl: boolean; + group_id: string; + }) => { if (panel?.kind === 'add') { await deviceAPI.create({ name: data.name, - storage_key: panel.template.id, - group_id: currentGroup?.id, + storage_key: panel.template.storage_key, + service_id: panel.template.service_id, + group_id: data.group_id, enabled: data.enabled, verify_ssl: data.verify_ssl, fields: data.fields, @@ -1024,6 +1445,7 @@ export default function DeviceIntegrationPage() { } else if (panel?.kind === 'edit') { await deviceAPI.update(panel.device.id, { name: data.name, + group_id: data.group_id, enabled: data.enabled, verify_ssl: data.verify_ssl, fields: data.fields, @@ -1032,6 +1454,9 @@ export default function DeviceIntegrationPage() { await fetchData(true); if (panel?.kind === 'edit') { const updated = await deviceAPI.get(panel.device.id); + if (selectedGroupId && updated.data.group_id !== selectedGroupId) { + setSelectedGroupId(updated.data.group_id); + } setPanel({ kind: 'edit', device: updated.data }); } }; @@ -1054,8 +1479,6 @@ export default function DeviceIntegrationPage() { return res.data; }; - // Persist the SSL toggle the moment it flips, without requiring 保存. - // Re-fetches the device so the open panel reflects the freshly stored row. const handleToggleVerifySsl = async (next: boolean) => { if (panel?.kind !== 'edit') return; await deviceAPI.update(panel.device.id, { verify_ssl: next }); @@ -1064,7 +1487,6 @@ export default function DeviceIntegrationPage() { await fetchData(true); }; - // Same pattern for enabled — persists immediately without needing 保存. const handleToggleEnabled = async (next: boolean) => { if (panel?.kind !== 'edit') return; await deviceAPI.update(panel.device.id, { enabled: next }); @@ -1073,18 +1495,48 @@ export default function DeviceIntegrationPage() { await fetchData(true); }; + // ────────────────────────────────────────────────────────────────────────── + // Group to use when adding a new device (follows sidebar selection). + // In "全部机房" view (null), pre-select the first available group so the + // dropdown has a sensible default; the user can change it in the panel. + // ────────────────────────────────────────────────────────────────────────── + const addDefaultGroupId = selectedGroupId ?? groups[0]?.id ?? DEFAULT_GROUP_ID; + // Whether the room field should be locked (read-only) in the config panel. + const groupLocked = selectedGroupId !== null; + + // ────────────────────────────────────────────────────────────────────────── + // Stats for the main area header + // ────────────────────────────────────────────────────────────────────────── + const connectedCount = filteredDevices.filter( + (d) => d.enabled && (d.status === 'ok' || d.status === 'connected'), + ).length; + const errorCount = filteredDevices.filter((d) => d.enabled && d.status === 'error').length; + + // Groups that actually render a section in the "全部机房" view (i.e. have at + // least one device) — drives the collapse-all toggle. + const nonEmptyGroupIds = useMemo( + () => groups.filter((g) => devices.some((d) => d.group_id === g.id)).map((g) => g.id), + [groups, devices], + ); + const allCollapsed = + nonEmptyGroupIds.length > 0 && nonEmptyGroupIds.every((id) => collapsedGroups.has(id)); + + // ────────────────────────────────────────────────────────────────────────── + // Render + // ────────────────────────────────────────────────────────────────────────── + return ( -
+
} + title={t('pageTitle')} + description={t('pageDescription')} + icon={} action={
} /> - void fetchData(true)} /> - - {loading ? ( -
- ) : ( -
- {devices.length === 0 ? ( - /* Empty state */ -
-
- -
-
-

暂无已接入的设备

-

添加设备后，Flocks Agent 即可调用对应工具

-
- -
+ {/* Content: sidebar + main area */} +
+ {/* Left: room sidebar */} + {!loading && ( + + )} + + {/* Right: main device area */} +
+ {loading ? ( +
) : ( -
-
- -

已接入设备

- {devices.length} - {devices.filter((d) => d.status === 'ok' || d.status === 'connected').length > 0 && ( - - {devices.filter((d) => d.status === 'ok' || d.status === 'connected').length} 已连接 - + <> + {/* Room / aggregate header bar */} +
+ {selectedGroup ? ( + <> + + {selectedGroup.name} + {t('header.devices', { count: filteredDevices.length })} + {connectedCount > 0 && ( + + + {t('header.connected', { count: connectedCount })} + + )} + {errorCount > 0 && ( + + + {t('header.failed', { count: errorCount })} + + )} + + ) : ( + <> + + {t('header.allRooms')} + + {t('header.deviceCount', { count: devices.length, rooms: groups.length })} + + {connectedCount > 0 && ( + + + {t('header.connected', { count: connectedCount })} + + )} + {errorCount > 0 && ( + + + {t('header.failed', { count: errorCount })} + + )} + {nonEmptyGroupIds.length > 1 && ( + + )} + )}
-
- {devices.map((d) => ( - setPanel({ kind: 'edit', device: d })} - /> - ))} + + {/* Device content area */} +
+ {devices.length === 0 ? ( + /* Global empty state — no devices at all */ +
+
+ +
+
+

{t('empty.noDevices')}

+

{t('empty.noDevicesHint')}

+
+ +
+ ) : selectedGroupId === null ? ( + /* ── "全部机房" grouped view ── */ +
+ {groups.map((group) => { + const gDevices = devices.filter((d) => d.group_id === group.id); + if (gDevices.length === 0) return null; + const gConnected = gDevices.filter( + (d) => d.enabled && (d.status === 'ok' || d.status === 'connected'), + ).length; + const collapsed = collapsedGroups.has(group.id); + return ( +
+ + {!collapsed && ( +
+ {gDevices.map((d) => ( + setPanel({ kind: 'edit', device: d })} + /> + ))} +
+ )} +
+ ); + })} + + {/* Orphan fallback: devices whose group_id matches no known + room (data drift / migration leftovers) must still be + reachable — the "全部机房" view should never hide a device. */} + {(() => { + const known = new Set(groups.map((g) => g.id)); + const orphans = devices.filter((d) => !known.has(d.group_id)); + if (orphans.length === 0) return null; + return ( +
+
+ +

{t('section.ungrouped')}

+ + {orphans.length} + + {t('section.ungroupedHint')} +
+
+ {orphans.map((d) => ( + setPanel({ kind: 'edit', device: d })} + /> + ))} +
+
+ ); + })()} +
+ ) : filteredDevices.length === 0 ? ( + /* ── Specific room, no devices ── */ +
+
+ +
+
+

{t('empty.roomEmpty')}

+

{t('empty.roomEmptyHint')}

+
+ +
+ ) : ( + /* ── Specific room, has devices ── */ +
+
+ +

{t('section.activeDevices')}

+ + {filteredDevices.length} + + {connectedCount > 0 && ( + {t('header.connected', { count: connectedCount })} + )} +
+
+ {filteredDevices.map((d) => ( + setPanel({ kind: 'edit', device: d })} + /> + ))} +
+
+ )}
-
+ )}
- )} +
{/* Wizard panel (vendor → product selection) */} {panel?.kind === 'wizard' && ( @@ -1159,7 +1784,16 @@ export default function DeviceIntegrationPage() { instanceCounts={instanceCounts} initialVendor={panel.initialVendor} onSelect={(tpl) => setPanel({ kind: 'add', template: tpl })} + onSelectCustom={(mode) => setPanel({ kind: 'custom', mode })} + onClose={() => setPanel(null)} + /> + )} + + {panel?.kind === 'custom' && ( + setPanel(null)} + onBack={() => setPanel({ kind: 'wizard' })} /> )} @@ -1167,13 +1801,19 @@ export default function DeviceIntegrationPage() { {(panel?.kind === 'add' || panel?.kind === 'edit') && (() => { const panelVendorKey = panel.kind === 'edit' ? vendorOf(panel.device) - : panel.template.vendor; + : panel.template.vendor ?? undefined; + const panelInitGroupId = panel.kind === 'edit' + ? panel.device.group_id + : addDefaultGroupId; return ( setPanel(null)} diff --git a/webui/src/pages/Home/index.test.tsx b/webui/src/pages/Home/index.test.tsx new file mode 100644 index 000000000..26d06a640 --- /dev/null +++ b/webui/src/pages/Home/index.test.tsx @@ -0,0 +1,107 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { MemoryRouter } from 'react-router-dom'; +import Home from './index'; + +const { createMock, navigateMock, toastErrorMock, useAuthMock } = vi.hoisted(() => ({ + createMock: vi.fn(), + navigateMock: vi.fn(), + toastErrorMock: vi.fn(), + useAuthMock: vi.fn(), +})); + +vi.mock('react-router-dom', async () => { + const actual = await vi.importActual('react-router-dom'); + return { + ...actual, + useNavigate: () => navigateMock, + }; +}); + +vi.mock('@/api/session', () => ({ + sessionApi: { + create: createMock, + }, +})); + +vi.mock('@/hooks/useStats', () => ({ + useStats: () => ({ + stats: null, + loading: false, + error: null, + }), +})); + +vi.mock('@/components/common/Toast', () => ({ + useToast: () => ({ + error: toastErrorMock, + }), +})); + +vi.mock('@/contexts/AuthContext', () => ({ + useAuth: useAuthMock, +})); + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string) => key, + i18n: { language: 'zh-CN' }, + }), +})); + +describe('Home create user defined page entry', () => { + beforeEach(() => { + vi.clearAllMocks(); + createMock.mockResolvedValue({ id: 'session-user-defined-1' }); + useAuthMock.mockReturnValue({ + user: { + id: 'user-1', + username: 'admin', + role: 'admin', + status: 'active', + must_reset_password: false, + }, + }); + }); + + it('allows admins to create a session and navigate with the guided initial message', async () => { + const user = userEvent.setup(); + render( + + + , + ); + + await user.click(screen.getByRole('button', { name: 'createUserDefinedPage' })); + + await waitFor(() => { + expect(createMock).toHaveBeenCalledWith({ title: 'createUserDefinedPageSessionTitle' }); + }); + + expect(navigateMock).toHaveBeenCalledWith( + `/sessions?session=session-user-defined-1&message=${encodeURIComponent('createUserDefinedPageInitialMessage')}`, + ); + }); + + it('hides the create user defined page entry for non-admin users', () => { + useAuthMock.mockReturnValue({ + user: { + id: 'user-2', + username: 'member', + role: 'member', + status: 'active', + must_reset_password: false, + }, + }); + + render( + + + , + ); + + expect(screen.queryByRole('button', { name: 'createUserDefinedPage' })).not.toBeInTheDocument(); + expect(createMock).not.toHaveBeenCalled(); + }); +}); diff --git a/webui/src/pages/Home/index.tsx b/webui/src/pages/Home/index.tsx index 298d39f28..b59d39c2c 100644 --- a/webui/src/pages/Home/index.tsx +++ b/webui/src/pages/Home/index.tsx @@ -13,12 +13,17 @@ import { Wrench, BookOpen, Cpu, + LayoutDashboard, + Loader2, } from 'lucide-react'; -import { useState } from 'react'; -import { Link } from 'react-router-dom'; +import { useCallback, useState } from 'react'; +import { Link, useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; import { useStats } from '@/hooks/useStats'; import LoadingSpinner from '@/components/common/LoadingSpinner'; +import { sessionApi } from '@/api/session'; +import { useToast } from '@/components/common/Toast'; +import { useAuth } from '@/contexts/AuthContext'; const GITHUB_URL = 'https://github.com/AgentFlocks/flocks'; const GITEE_URL = 'https://gitee.com/flocks/flocks'; @@ -27,7 +32,27 @@ const GITEE_LOGO_URL = `${import.meta.env.BASE_URL}gitee-logo.png`; export default function Home() { const { stats, loading, error } = useStats(); const { t } = useTranslation('home'); + const navigate = useNavigate(); + const toast = useToast(); + const { user } = useAuth(); + const canCreateUserDefinedPage = user?.role === 'admin'; const [isRepoMenuOpen, setIsRepoMenuOpen] = useState(false); + const [creatingUserDefinedPageSession, setCreatingUserDefinedPageSession] = useState(false); + + const handleCreateUserDefinedPage = useCallback(async () => { + if (creatingUserDefinedPageSession) return; + setCreatingUserDefinedPageSession(true); + try { + const session = await sessionApi.create({ title: t('createUserDefinedPageSessionTitle') }); + const message = t('createUserDefinedPageInitialMessage'); + navigate(`/sessions?session=${session.id}&message=${encodeURIComponent(message)}`); + } catch (err: unknown) { + const detail = err instanceof Error ? err.message : t('createUserDefinedPageError'); + toast.error(t('createUserDefinedPageError'), detail); + } finally { + setCreatingUserDefinedPageSession(false); + } + }, [creatingUserDefinedPageSession, navigate, t, toast]); return (
@@ -66,6 +91,22 @@ export default function Home() { + {canCreateUserDefinedPage ? ( + + ) : null} +
@@ -91,6 +119,7 @@ vi.mock('@/components/common/SessionChat', () => ({ vi.mock('@/utils/agentDisplay', () => ({ getAgentDisplayDescription: () => 'agent-description', + getAgentDisplayName: (agent: { name: string }) => agent.name.charAt(0).toUpperCase() + agent.name.slice(1), })); vi.mock('@/utils/time', () => ({ @@ -125,6 +154,34 @@ const secondSession = { title: 'Second Session', }; +const modelProviders = [ + { id: 'openai', name: 'OpenAI', configured: true }, + { id: 'minimax', name: 'MiniMax', configured: true }, +]; + +const modelDefinitions = [ + { + provider_id: 'openai', + id: 'gpt-4o', + name: 'GPT-4o', + model_type: 'llm', + source: 'predefined', + capabilities: {}, + pricing: null, + limits: {}, + }, + { + provider_id: 'minimax', + id: 'minimax-m3', + name: 'MiniMax M3', + model_type: 'llm', + source: 'predefined', + capabilities: {}, + pricing: null, + limits: {}, + }, +]; + function renderSessionPage( initialEntry: string | { pathname: string; state?: unknown } = '/sessions', ) { @@ -157,6 +214,15 @@ describe('SessionPage session actions menu', () => { error: null, refetch: vi.fn(), }); + useProviders.mockReturnValue({ + providers: [], + connectedIds: [], + loading: false, + error: null, + refetch: vi.fn(), + }); + defaultModelAPI.getResolved.mockResolvedValue({ data: { provider_id: '', model_id: '' } }); + modelV2API.listDefinitions.mockResolvedValue({ data: { models: [] } }); sessionApi.update.mockResolvedValue({ ...session, title: 'Renamed Session' }); client.post.mockResolvedValue({ data: secondSession }); @@ -438,6 +504,131 @@ describe('SessionPage session actions menu', () => { expect(screen.getByRole('button', { name: /Rex/i })).toBeInTheDocument(); }); + it('shows the pinned model for the selected session on load', async () => { + localStorage.setItem('flocks:last-selected-session', 'session-1'); + useSessions.mockReturnValue({ + sessions: [{ + ...session, + provider: 'minimax', + model: 'minimax-m3', + model_pinned: true, + }], + loading: false, + error: null, + refetch: refetchSessions, + updateSessionTitle, + removeSession, + removeSessions, + addSession, + }); + useProviders.mockReturnValue({ + providers: modelProviders, + connectedIds: [], + loading: false, + error: null, + refetch: vi.fn(), + }); + defaultModelAPI.getResolved.mockResolvedValue({ data: { provider_id: 'openai', model_id: 'gpt-4o' } }); + modelV2API.listDefinitions.mockResolvedValue({ data: { models: modelDefinitions } }); + + renderSessionPage(); + + await waitFor(() => { + expect(screen.getByTestId('session-chat')).toHaveAttribute('data-model', 'minimax/minimax-m3'); + }); + expect(defaultModelAPI.getResolved).not.toHaveBeenCalled(); + }); + + it('persists model changes to the selected session', async () => { + const user = userEvent.setup(); + localStorage.setItem('flocks:last-selected-session', 'session-1'); + useSessions.mockReturnValue({ + sessions: [session], + loading: false, + error: null, + refetch: refetchSessions, + updateSessionTitle, + removeSession, + removeSessions, + addSession, + }); + useProviders.mockReturnValue({ + providers: modelProviders, + connectedIds: [], + loading: false, + error: null, + refetch: vi.fn(), + }); + defaultModelAPI.getResolved.mockResolvedValue({ data: { provider_id: 'openai', model_id: 'gpt-4o' } }); + modelV2API.listDefinitions.mockResolvedValue({ data: { models: modelDefinitions } }); + sessionApi.update.mockResolvedValue({ + ...session, + provider: 'minimax', + model: 'minimax-m3', + model_pinned: true, + }); + + renderSessionPage(); + + await waitFor(() => { + expect(screen.getByTestId('session-chat')).toHaveAttribute('data-model', 'openai/gpt-4o'); + }); + + await user.click(screen.getByRole('button', { name: /GPT-4o/i })); + await user.click(screen.getByRole('button', { name: /MiniMax M3/i })); + + await waitFor(() => { + expect(sessionApi.update).toHaveBeenCalledWith('session-1', { + provider: 'minimax', + model: 'minimax-m3', + model_pinned: true, + }); + }); + expect(refetchSessions).toHaveBeenCalled(); + }); + + it('resets the selected model to the default when creating a new session', async () => { + const user = userEvent.setup(); + localStorage.setItem('flocks:last-selected-session', 'session-1'); + useSessions.mockReturnValue({ + sessions: [{ + ...session, + provider: 'minimax', + model: 'minimax-m3', + model_pinned: true, + }], + loading: false, + error: null, + refetch: refetchSessions, + updateSessionTitle, + removeSession, + removeSessions, + addSession, + }); + useProviders.mockReturnValue({ + providers: modelProviders, + connectedIds: [], + loading: false, + error: null, + refetch: vi.fn(), + }); + defaultModelAPI.getResolved.mockResolvedValue({ data: { provider_id: 'openai', model_id: 'gpt-4o' } }); + modelV2API.listDefinitions.mockResolvedValue({ data: { models: modelDefinitions } }); + + renderSessionPage(); + + await waitFor(() => { + expect(screen.getByTestId('session-chat')).toHaveAttribute('data-model', 'minimax/minimax-m3'); + }); + + await user.click(screen.getByRole('button', { name: 'newSession' })); + + await waitFor(() => { + expect(addSession).toHaveBeenCalledWith(secondSession); + expect(screen.getByTestId('session-chat')).toHaveAttribute('data-model', 'openai/gpt-4o'); + }); + }); + it('uses Rex for the first message when an empty session is created by sending', async () => { const user = userEvent.setup(); useAgents.mockReturnValue({ diff --git a/webui/src/pages/Session/index.tsx b/webui/src/pages/Session/index.tsx index 46744827c..b577b5063 100644 --- a/webui/src/pages/Session/index.tsx +++ b/webui/src/pages/Session/index.tsx @@ -4,11 +4,11 @@ import { ChevronDown, Sparkles, Shield, Search, AlertTriangle, PanelLeftClose, PanelLeft, Bot, Loader2, Workflow as WorkflowIcon, Settings2, CheckSquare, - MoreHorizontal, PencilLine, Download, Share2, + MoreHorizontal, PencilLine, Download, Share2, Cpu, Info, } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import i18n from '@/i18n'; -import { useSearchParams } from 'react-router-dom'; +import { useNavigate, useSearchParams } from 'react-router-dom'; import LoadingSpinner from '@/components/common/LoadingSpinner'; import { useToast } from '@/components/common/Toast'; import SessionChat, { type SSEChatEvent, type SSEConnectionStatus } from '@/components/common/SessionChat'; @@ -16,11 +16,14 @@ import { sessionApi } from '@/api/session'; import type { Agent } from '@/api/agent'; import { useSessions } from '@/hooks/useSessions'; import { useAgents } from '@/hooks/useAgents'; +import { useProviders } from '@/hooks/useProviders'; import client from '@/api/client'; +import { defaultModelAPI, modelV2API } from '@/api/provider'; import { useDefaultModelVision } from '@/hooks/useDefaultModelVision'; import { buildPromptParts, type ImagePartData } from '@/utils/imageUpload'; -import { getAgentDisplayDescription } from '@/utils/agentDisplay'; +import { getAgentDisplayDescription, getAgentDisplayName } from '@/utils/agentDisplay'; import { formatSessionDate } from '@/utils/time'; +import type { ModelDefinitionV2 } from '@/types'; function sanitizeSessionExportName(value: string) { const trimmed = value.trim(); @@ -33,19 +36,33 @@ function sanitizeSessionExportName(value: string) { const LAST_SELECTED_SESSION_STORAGE_KEY = 'flocks:last-selected-session'; type AgentSourceFilter = 'all' | 'builtin' | 'custom'; +type ChatModelOption = { + key: string; + providerID: string; + providerName: string; + modelID: string; + label: string; + pricingLabel: string; + contextLabel: string; + contextWindowTokens: number | null; + supportsVision: boolean | null; +}; +type ChatModelProviderGroup = { + providerID: string; + providerName: string; + models: ChatModelOption[]; +}; +type SelectorTooltip = { + title: string; + lines: string[]; + x: number; + y: number; +}; function formatAgentName(name: string): string { return name ? name.charAt(0).toUpperCase() + name.slice(1) : name; } -function getAgentSecondaryDescription(agent: Agent, language: string): string { - const isZh = language.toLowerCase().replace('_', '-').startsWith('zh'); - const primary = (isZh ? agent.descriptionCn : agent.description)?.trim(); - const secondary = (isZh ? agent.description : agent.descriptionCn)?.trim(); - if (primary && secondary && primary !== secondary) return secondary; - return ''; -} - function readLastSelectedSessionId(): string | null { try { return window.localStorage.getItem(LAST_SELECTED_SESSION_STORAGE_KEY); @@ -66,13 +83,22 @@ function writeLastSelectedSessionId(sessionId: string | null) { } } +function makeModelKey(providerID: string, modelID: string): string { + return `${providerID}::${modelID}`; +} + export default function SessionPage() { const { t, i18n } = useTranslation('session'); + const navigate = useNavigate(); const [searchParams, setSearchParams] = useSearchParams(); const [selectedSessionId, setSelectedSessionId] = useState(null); const [sidebarCollapsed, setSidebarCollapsed] = useState(false); const [selectedAgent, setSelectedAgent] = useState('rex'); const [showAgentOptions, setShowAgentOptions] = useState(false); + const [selectedModelKey, setSelectedModelKey] = useState(null); + const [showModelOptions, setShowModelOptions] = useState(false); + const [enabledModelDefinitions, setEnabledModelDefinitions] = useState([]); + const [loadingEnabledModels, setLoadingEnabledModels] = useState(true); const [sseStatus, setSseStatus] = useState('disconnected'); const [creating, setCreating] = useState(false); const [pendingInitialMessage, setPendingInitialMessage] = useState(null); @@ -88,12 +114,14 @@ export default function SessionPage() { const supportsVision = useDefaultModelVision(); const [searchQuery, setSearchQuery] = useState(''); const [agentSourceFilter, setAgentSourceFilter] = useState('all'); + const [selectorTooltip, setSelectorTooltip] = useState(null); const renameInputRef = useRef(null); const renameSubmitInFlightRef = useRef(false); const toast = useToast(); const { sessions, loading: loadingSessions, refetch: refetchSessions, updateSessionTitle, removeSession, removeSessions, addSession } = useSessions(); const { agents, loading: loadingAgents } = useAgents(); + const { providers, loading: loadingProviders } = useProviders(); const primaryAgents = useMemo(() => agents.filter((a) => a.mode === 'primary'), [agents]); const subAgents = useMemo( () => agents.filter((a) => a.mode !== 'primary' && !(a.tags ?? []).includes('system')), @@ -112,6 +140,79 @@ export default function SessionPage() { () => chatAgents.find((agent) => agent.name === selectedAgent), [chatAgents, selectedAgent], ); + const chatModelOptions = useMemo(() => { + const providerById = new Map( + providers + .filter((provider) => provider.configured) + .map((provider) => [provider.id, provider]), + ); + + const formatPricing = (pricing: ModelDefinitionV2['pricing']): string => { + if (!pricing) return t('modelPicker.noCost'); + if (pricing.input === 0 && pricing.output === 0) return t('modelPicker.free'); + const currencySymbol = pricing.currency === 'CNY' ? '¥' : '$'; + return `${currencySymbol}${pricing.input}/${currencySymbol}${pricing.output}/M`; + }; + + const formatContextWindow = (contextWindow?: number): string => { + if (!contextWindow) return t('modelPicker.contextUnknown'); + const value = contextWindow >= 1000000 + ? `${(contextWindow / 1000000).toFixed(0)}M` + : `${(contextWindow / 1000).toFixed(0)}K`; + return t('modelPicker.contextWindow', { value }); + }; + + return enabledModelDefinitions.flatMap((model) => { + const provider = providerById.get(model.provider_id); + if (!provider) return []; + return [{ + key: makeModelKey(provider.id, model.id), + providerID: provider.id, + providerName: provider.name || provider.id, + modelID: model.id, + label: model.name || model.id, + pricingLabel: formatPricing(model.pricing), + contextLabel: formatContextWindow(model.limits?.context_window), + contextWindowTokens: model.limits?.context_window ?? null, + supportsVision: typeof model.capabilities?.supports_vision === 'boolean' + ? model.capabilities.supports_vision + : null, + }]; + }); + }, [enabledModelDefinitions, providers, t]); + const groupedChatModelOptions = useMemo(() => { + const groups = new Map(); + + providers.forEach((provider) => { + if (!provider.configured) return; + groups.set(provider.id, { + providerID: provider.id, + providerName: provider.name || provider.id, + models: [], + }); + }); + + chatModelOptions.forEach((option) => { + const group = groups.get(option.providerID); + if (group) group.models.push(option); + }); + + return Array.from(groups.values()) + .map((group) => ({ + ...group, + models: [...group.models].sort((a, b) => a.label.localeCompare(b.label)), + })) + .filter((group) => group.models.length > 0) + .sort((a, b) => a.providerName.localeCompare(b.providerName)); + }, [chatModelOptions, providers]); + const selectedModelOption = useMemo( + () => chatModelOptions.find((option) => option.key === selectedModelKey) ?? (selectedModelKey ? null : chatModelOptions[0] ?? null), + [chatModelOptions, selectedModelKey], + ); + const selectedPromptModel = selectedModelOption + ? { providerID: selectedModelOption.providerID, modelID: selectedModelOption.modelID } + : null; + const effectiveSupportsVision = selectedModelOption?.supportsVision ?? supportsVision; const selectedSession = useMemo( () => sessions.find(s => s.id === selectedSessionId) ?? null, [sessions, selectedSessionId], @@ -210,6 +311,24 @@ export default function SessionPage() { writeLastSelectedSessionId(selectedSessionId); }, [selectedSessionId]); + useEffect(() => { + let cancelled = false; + setLoadingEnabledModels(true); + modelV2API.listDefinitions({ enabled_only: true }) + .then((response) => { + if (!cancelled) setEnabledModelDefinitions(response.data.models ?? []); + }) + .catch(() => { + if (!cancelled) setEnabledModelDefinitions([]); + }) + .finally(() => { + if (!cancelled) setLoadingEnabledModels(false); + }); + return () => { + cancelled = true; + }; + }, []); + // Close agent dropdown on outside click useEffect(() => { if (!showAgentOptions) return; @@ -221,6 +340,65 @@ export default function SessionPage() { return () => document.removeEventListener('mousedown', handle); }, [showAgentOptions]); + useEffect(() => { + if (!showModelOptions) return; + const handle = (e: MouseEvent) => { + const target = e.target as HTMLElement; + if (!target.closest('[data-model-selector]')) setShowModelOptions(false); + }; + document.addEventListener('mousedown', handle); + return () => document.removeEventListener('mousedown', handle); + }, [showModelOptions]); + + useEffect(() => { + if (chatModelOptions.length === 0) { + setSelectedModelKey(null); + return; + } + + const pinnedKey = selectedSession?.model_pinned && selectedSession.provider && selectedSession.model + ? makeModelKey(selectedSession.provider, selectedSession.model) + : null; + if (pinnedKey && chatModelOptions.some((option) => option.key === pinnedKey)) { + setSelectedModelKey(pinnedKey); + return; + } + + let cancelled = false; + setSelectedModelKey(null); + defaultModelAPI.getResolved() + .then((response) => { + if (cancelled) return; + const { provider_id: providerID, model_id: modelID } = response.data; + const defaultKey = makeModelKey(providerID, modelID); + const fallbackKey = chatModelOptions[0]?.key ?? null; + setSelectedModelKey(chatModelOptions.some((option) => option.key === defaultKey) ? defaultKey : fallbackKey); + }) + .catch(() => { + if (!cancelled) setSelectedModelKey(chatModelOptions[0]?.key ?? null); + }); + return () => { + cancelled = true; + }; + }, [ + chatModelOptions, + selectedSession?.model, + selectedSession?.model_pinned, + selectedSession?.provider, + selectedSessionId, + ]); + + useEffect(() => { + if (loadingEnabledModels || chatModelOptions.length === 0 || !selectedModelKey) return; + if (chatModelOptions.some((option) => option.key === selectedModelKey)) return; + setSelectedModelKey(chatModelOptions[0].key); + }, [chatModelOptions, loadingEnabledModels, selectedModelKey]); + + useEffect(() => { + if (showAgentOptions || showModelOptions) return; + setSelectorTooltip(null); + }, [showAgentOptions, showModelOptions]); + useEffect(() => { if (!openMenuSessionId) return; const handle = (e: MouseEvent) => { @@ -254,6 +432,7 @@ export default function SessionPage() { const response = await client.post('/api/session', { title: 'New Session' }); addSession(response.data); setSelectedAgent('rex'); + setSelectedModelKey(null); setSelectedSessionId(response.data.id); } catch (err: any) { toast.error(t('createFailed'), err.message); @@ -262,6 +441,23 @@ export default function SessionPage() { } }, [creating, addSession, toast, t]); + const handleSelectModel = useCallback(async (option: ChatModelOption) => { + setSelectedModelKey(option.key); + setShowModelOptions(false); + if (!selectedSessionId) return; + + try { + await sessionApi.update(selectedSessionId, { + provider: option.providerID, + model: option.modelID, + model_pinned: true, + }); + refetchSessions(); + } catch (err: any) { + toast.error(t('chat.error', 'Error'), err.message); + } + }, [refetchSessions, selectedSessionId, toast, t]); + useEffect(() => { if (loadingSessions) return; if (searchParams.get('session')) return; @@ -294,6 +490,7 @@ export default function SessionPage() { text: string, imageParts?: ImagePartData[], agentOverride?: string, + modelOverride?: { providerID: string; modelID: string } | null, ) => { try { const response = await client.post('/api/session', { title: 'New Session' }); @@ -301,6 +498,7 @@ export default function SessionPage() { addSession(response.data); setSelectedAgent('rex'); + setSelectedModelKey(null); setSelectedSessionId(newSessionId); const payload: Record = { @@ -308,6 +506,7 @@ export default function SessionPage() { }; const effectiveAgent = agentOverride || 'rex'; if (effectiveAgent) payload.agent = effectiveAgent; + if (modelOverride) payload.model = modelOverride; client.post(`/api/session/${newSessionId}/prompt_async`, payload).catch((err: any) => { toast.error(t('chat.sendFailed', 'Send failed'), err.message); }); @@ -316,6 +515,16 @@ export default function SessionPage() { } }, [addSession, toast, t]); + const showSelectorTooltip = useCallback((target: HTMLElement, title: string, lines: string[]) => { + const rect = target.getBoundingClientRect(); + setSelectorTooltip({ + title, + lines, + x: rect.left - 8, + y: rect.top + rect.height / 2, + }); + }, []); + const handleDeleteSession = useCallback(async (sessionId: string) => { const target = sessions.find((s) => s.id === sessionId); if (target?.canDelete === false) { @@ -742,39 +951,52 @@ export default function SessionPage() { onCreateAndSend={handleCreateAndSend} onCreateNewSession={handleCreateSession} onStreamingDone={() => setPendingInitialMessage(null)} - supportsVision={supportsVision} + supportsVision={effectiveSupportsVision} + contextWindowTokens={selectedModelOption?.contextWindowTokens ?? null} + model={selectedPromptModel} welcomeContent={(setInput) => ( )} toolbarSlot={
{showAgentOptions && ( -
-
+
+
{t('agentPicker.title')}
-
{t('agentPicker.hint')}
+
showSelectorTooltip(event.currentTarget, t('agentPicker.title'), [t('agentPicker.hint')])} + onMouseEnter={(event) => showSelectorTooltip(event.currentTarget, t('agentPicker.title'), [t('agentPicker.hint')])} + onMouseOver={(event) => showSelectorTooltip(event.currentTarget, t('agentPicker.title'), [t('agentPicker.hint')])} + onMouseLeave={() => setSelectorTooltip(null)} + onPointerLeave={() => setSelectorTooltip(null)} + > + {t('agentPicker.hint')} +
-
+
{(['all', 'builtin', 'custom'] as AgentSourceFilter[]).map((filter) => (
-
+
{loadingAgents ? ( -
{t('loading')}
+
{t('loading')}
) : filteredChatAgents.length > 0 ? ( filteredChatAgents.map((agent) => { + const displayName = getAgentDisplayName(agent, i18n.language); const primaryDesc = getAgentDisplayDescription(agent, i18n.language) || t('smartAssistant'); - const secondaryDesc = getAgentSecondaryDescription(agent, i18n.language); return ( ); }) ) : ( -
{t('noAgents')}
+
{t('noAgents')}
)}
)}
} + centerToolbarSlot={ +
+ + {showModelOptions && ( +
+
+
{t('modelPicker.title')}
+
{t('modelPicker.hint')}
+
+
+ {loadingProviders || loadingEnabledModels ? ( +
{t('loading')}
+ ) : groupedChatModelOptions.length > 0 ? ( + groupedChatModelOptions.map((group) => ( +
+
+ {group.providerName} + + {t('modelPicker.count', { count: group.models.length })} + +
+
+ {group.models.map((option) => ( + + ))} +
+
+ )) + ) : ( +
{t('modelPicker.empty')}
+ )} +
+
+ +
+
+ )} +
+ } />
+ {selectorTooltip && ( +
+
{selectorTooltip.title}
+ {selectorTooltip.lines.map((line, index) => ( +
+ {line} +
+ ))} +
+
+ )} + {/* Three-dot dropdown — rendered outside sidebar to avoid overflow:hidden clipping */} {openMenuSessionId && menuAnchor && (() => { const sid = openMenuSessionId; diff --git a/webui/src/pages/Tool/components/ToolDetailModal.tsx b/webui/src/pages/Tool/components/ToolDetailModal.tsx index 82de9d723..deda3c5ed 100644 --- a/webui/src/pages/Tool/components/ToolDetailModal.tsx +++ b/webui/src/pages/Tool/components/ToolDetailModal.tsx @@ -12,26 +12,34 @@ import { EnabledBadge } from './badges'; interface ToolDetailModalProps { tool: Tool; initialSection?: 'info' | 'test'; + /** When opening from a specific device's panel, pass the device UUID so + * it is pre-filled as `device_id` in the test params template. */ + deviceId?: string; onClose: () => void; } -function buildParamsTemplate(tool: Tool): string { - if (!tool.parameters || tool.parameters.length === 0) return '{}'; - const obj: Record = {}; - for (const p of tool.parameters) { - if (p.required) { - obj[p.name] = p.type === 'number' || p.type === 'integer' ? '0' as any - : p.type === 'boolean' ? 'false' as any - : ''; +function buildParamsTemplate(tool: Tool, deviceId?: string): string { + const obj: Record = {}; + if (deviceId) { + obj['device_id'] = deviceId; + } + if (tool.parameters && tool.parameters.length > 0) { + for (const p of tool.parameters) { + if (p.name === 'device_id' && deviceId) continue; // injected value above wins + if (p.required) { + obj[p.name] = p.type === 'number' || p.type === 'integer' ? 0 + : p.type === 'boolean' ? false + : ''; + } } } return JSON.stringify(obj, null, 2); } -export default function ToolDetailModal({ tool, initialSection, onClose }: ToolDetailModalProps) { +export default function ToolDetailModal({ tool, initialSection, deviceId, onClose }: ToolDetailModalProps) { const { t, i18n } = useTranslation('tool'); const [section, setSection] = useState<'info' | 'test'>(initialSection || 'info'); - const defaultParams = useMemo(() => buildParamsTemplate(tool), [tool]); + const defaultParams = useMemo(() => buildParamsTemplate(tool, deviceId), [tool, deviceId]); const [testParams, setTestParams] = useState(defaultParams); const [testResult, setTestResult] = useState(null); const [testing, setTesting] = useState(false); @@ -210,7 +218,10 @@ export default function ToolDetailModal({ tool, initialSection, onClose }: ToolD key={idx} type="button" onClick={() => { - setTestParams(JSON.stringify(fx.params, null, 2)); + const merged = deviceId + ? { ...fx.params, device_id: deviceId } + : fx.params; + setTestParams(JSON.stringify(merged, null, 2)); setFixturesOpen(false); }} className="w-full flex items-start gap-3 px-4 py-2.5 hover:bg-blue-100 transition-colors text-left" @@ -240,6 +251,12 @@ export default function ToolDetailModal({ tool, initialSection, onClose }: ToolD
+ {deviceId && ( +

+ + 已自动填入当前设备 ID（{deviceId}） +

+ )}
工具名称描述状态操作{t('tools.colName')}{t('tools.colDesc')}{t('tools.colStatus')}{t('tools.colAction')}