From 5350e411a1afb1c1bb664a7a0ac5240c57ae65b9 Mon Sep 17 00:00:00 2001 From: John Yin <10972267+john-yin2333@user.noreply.gitee.com> Date: Wed, 3 Jun 2026 17:07:24 +0800 Subject: [PATCH 01/54] fix(log): restructure log directory retention Co-authored-by: Cursor --- flocks/cli/service_manager.py | 2 - flocks/server/routes/logs.py | 64 ++++++- flocks/updater/updater.py | 2 +- flocks/utils/log.py | 253 ++++++++++++------------- flocks/workflow/logging_config.py | 30 +-- tests/cli/test_service_manager.py | 8 +- tests/server/test_log_routes.py | 65 +++++++ tests/utils/test_append_upgrade_log.py | 4 +- tests/utils/test_log_compatibility.py | 149 ++++++++++----- tests/workflow/test_logging_config.py | 13 +- 10 files changed, 369 insertions(+), 221 deletions(-) diff --git a/flocks/cli/service_manager.py b/flocks/cli/service_manager.py index d0de2066a..b560eb3bb 100644 --- a/flocks/cli/service_manager.py +++ b/flocks/cli/service_manager.py @@ -26,7 +26,6 @@ import httpx from flocks.browser.admin import stop_all_daemons as stop_all_browser_daemons -from flocks.utils.log import rotate_log_file try: import fcntl @@ -1682,7 +1681,6 @@ def _spawn_process( kwargs["start_new_session"] = True log_path.parent.mkdir(parents=True, exist_ok=True) - rotate_log_file(log_path) handle = log_path.open("a", encoding="utf-8") try: return subprocess.Popen( diff --git a/flocks/server/routes/logs.py b/flocks/server/routes/logs.py index 5517257cd..4240e2b59 100644 --- a/flocks/server/routes/logs.py +++ b/flocks/server/routes/logs.py @@ -5,6 +5,7 @@ """ from collections import deque +from datetime import date, datetime from pathlib import Path from typing import List @@ -46,9 +47,9 @@ async def list_logs(): return LogListResponse(files=[], log_dir=str(log_dir)) files: List[LogFileInfo] = [] - for p in sorted(log_dir.glob("*.log"), key=lambda f: f.stat().st_mtime, reverse=True): + for p in sorted(_iter_log_files(log_dir), key=lambda f: f.stat().st_mtime, reverse=True): stat = p.stat() - files.append(LogFileInfo(name=p.name, size=stat.st_size, modified=stat.st_mtime)) + files.append(LogFileInfo(name=p.relative_to(log_dir).as_posix(), size=stat.st_size, modified=stat.st_mtime)) return LogListResponse(files=files, log_dir=str(log_dir)) @@ -65,7 +66,16 @@ async def read_latest_log( if not log_dir.is_dir(): raise HTTPException(status_code=404, detail="Log directory not found") - log_files = sorted(log_dir.glob("*.log"), key=lambda f: f.stat().st_mtime, reverse=True) + today_log = log_dir / date.today().isoformat() / "flocks.log" + if today_log.is_file(): + return _read_log_file(today_log, tail) + + for day_dir in sorted(_iter_date_dirs(log_dir), reverse=True): + main_log = day_dir / "flocks.log" + if main_log.is_file(): + return _read_log_file(main_log, tail) + + log_files = sorted(_iter_log_files(log_dir), key=lambda f: f.stat().st_mtime, reverse=True) if not log_files: raise HTTPException(status_code=404, detail="No log files found") @@ -73,7 +83,7 @@ async def read_latest_log( @router.get( - "/{filename}", + "/{filename:path}", response_model=LogContentResponse, summary="Read a specific log file", ) @@ -85,7 +95,7 @@ async def read_log( log_dir = get_log_dir() log_path = log_dir / filename - if not log_path.is_file() or not log_path.suffix == ".log": + if not log_path.is_file() or not _is_allowed_log_path(log_dir, log_path): raise HTTPException(status_code=404, detail=f"Log file not found: {filename}") if not log_path.resolve().is_relative_to(log_dir.resolve()): @@ -94,6 +104,50 @@ async def read_log( return _read_log_file(log_path, tail) +def _is_date_dir(path: Path) -> bool: + try: + datetime.strptime(path.name, "%Y-%m-%d") + except ValueError: + return False + return path.is_dir() + + +def _iter_date_dirs(log_dir: Path) -> List[Path]: + return [p for p in log_dir.iterdir() if _is_date_dir(p)] + + +def _is_allowed_log_path(log_dir: Path, path: Path) -> bool: + try: + relative = path.relative_to(log_dir) + except ValueError: + return False + parts = relative.parts + if len(parts) == 1: + return parts[0] in {"backend.log", "webui.log"} + if len(parts) == 2: + day, filename = parts + try: + datetime.strptime(day, "%Y-%m-%d") + except ValueError: + return False + return filename in {"flocks.log", "errors.log"} + return False + + +def _iter_log_files(log_dir: Path) -> List[Path]: + files = [] + for name in ("backend.log", "webui.log"): + path = log_dir / name + if path.is_file(): + files.append(path) + for day_dir in _iter_date_dirs(log_dir): + for name in ("flocks.log", "errors.log"): + path = day_dir / name + if path.is_file(): + files.append(path) + return files + + def _read_log_file(path: Path, tail: int) -> LogContentResponse: try: lines: deque[str] = deque(maxlen=tail) diff --git a/flocks/updater/updater.py b/flocks/updater/updater.py index a30b54ce9..0028f6ac3 100644 --- a/flocks/updater/updater.py +++ b/flocks/updater/updater.py @@ -81,7 +81,7 @@ class ConsoleManifestRelease: def _record_update_journal(message: str) -> None: - """Append a human-readable line to ``update.log`` (see ``append_upgrade_text_log``).""" + """Append a human-readable upgrade line to today's errors log.""" from flocks.utils.log import append_upgrade_text_log append_upgrade_text_log(message) diff --git a/flocks/utils/log.py b/flocks/utils/log.py index f22d7a4fb..6768222b8 100644 --- a/flocks/utils/log.py +++ b/flocks/utils/log.py @@ -13,13 +13,12 @@ import threading from pathlib import Path from typing import Any, Dict, Optional, TextIO -from datetime import datetime +from datetime import date, datetime, timedelta import json import glob as file_glob -_DEFAULT_LOG_MAX_BYTES = 5 * 1024 * 1024 -_DEFAULT_LOG_BACKUP_COUNT = 3 +_DEFAULT_LOG_RETENTION_DAYS = 30 _DEFAULT_LOG_VALUE_MAX_CHARS = 8 * 1024 _MAX_STRUCTURED_ITEMS = 50 _MAX_STRUCTURED_DEPTH = 4 @@ -37,7 +36,7 @@ def _log_dir() -> Path: def get_log_dir() -> Path: - """Return the log directory for file handlers (e.g. workflow). Same as Log.init() uses.""" + """Return the root log directory used by Flocks.""" return _log_dir() @@ -51,99 +50,29 @@ def _env_int(name: str, default: int) -> int: return default -def get_log_max_bytes(default: int = _DEFAULT_LOG_MAX_BYTES) -> int: - """Return the per-file log size limit in bytes. +def get_log_retention_days(default: int = _DEFAULT_LOG_RETENTION_DAYS) -> int: + """Return how long legacy timestamp logs are retained.""" + return _env_int("FLOCKS_LOG_RETENTION_DAYS", default) - ``FLOCKS_LOG_MAX_BYTES`` is exact; ``FLOCKS_LOG_MAX_MB`` is a convenient - human-facing override. When both are set, bytes wins. Values <= 0 disable - rotation. - """ - if os.getenv("FLOCKS_LOG_MAX_BYTES") is not None: - return _env_int("FLOCKS_LOG_MAX_BYTES", default) - max_mb = os.getenv("FLOCKS_LOG_MAX_MB") - if max_mb is not None: - try: - return int(float(max_mb) * 1024 * 1024) - except ValueError: - return default - return default - - -def get_log_backup_count(default: int = _DEFAULT_LOG_BACKUP_COUNT) -> int: - """Return how many rotated backups to keep for long-lived log files.""" - return max(0, _env_int("FLOCKS_LOG_BACKUP_COUNT", default)) - - -def rotate_log_file( - path: Path, - *, - max_bytes: Optional[int] = None, - backup_count: Optional[int] = None, - force: bool = False, -) -> None: - """Rotate ``path`` if it is already over the configured size limit.""" - limit = get_log_max_bytes() if max_bytes is None else max_bytes - backups = get_log_backup_count() if backup_count is None else backup_count - if limit <= 0 or not path.exists(): - return - try: - if not force and path.stat().st_size < limit: - return - if backups <= 0: - path.unlink(missing_ok=True) - return - for index in range(backups - 1, 0, -1): - src = path.with_name(f"{path.name}.{index}") - dst = path.with_name(f"{path.name}.{index + 1}") - if src.exists(): - src.replace(dst) - path.replace(path.with_name(f"{path.name}.1")) - except OSError: - return +class _AppendTextWriter: + """Small line-buffered writer for Flocks daily logs.""" -class _RotatingTextWriter: - """Small line-buffered writer with size-based rotation for Flocks logs.""" - - def __init__(self, path: Path, *, max_bytes: int, backup_count: int): + def __init__(self, path: Path): self.path = path - self.max_bytes = max_bytes - self.backup_count = backup_count self._handle: Optional[TextIO] = None - self._bytes_written = 0 self._lock = threading.RLock() self._open() def _open(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) self._handle = open(self.path, "a", buffering=1, encoding="utf-8") - try: - self._bytes_written = self.path.stat().st_size - except OSError: - self._bytes_written = 0 - - def _should_rotate(self, message: str) -> bool: - if self.max_bytes <= 0: - return False - return self._bytes_written + len(message.encode("utf-8")) > self.max_bytes def write(self, message: str) -> int: - encoded_len = len(message.encode("utf-8")) with self._lock: - if self._should_rotate(message): - self.close() - rotate_log_file( - self.path, - max_bytes=self.max_bytes, - backup_count=self.backup_count, - force=True, - ) - self._open() if self._handle is None: self._open() - written = self._handle.write(message) - self._bytes_written += encoded_len - return written + return self._handle.write(message) def flush(self) -> None: with self._lock: @@ -213,15 +142,16 @@ def _format_log_value(value: Any) -> str: def append_upgrade_text_log(message: str) -> None: - """Append timestamped lines to ``update.log`` under the configured log directory. + """Append timestamped upgrade lines to today's ``errors.log``. Used for upgrade flows so errors remain on disk when the process had no TTY - or when structured ``Log`` output went to a different file than ``backend.log``. + or when structured ``Log`` output was not initialized. """ try: log_dir = _log_dir() - log_dir.mkdir(parents=True, exist_ok=True) - path = log_dir / "update.log" + day_dir = log_dir / date.today().isoformat() + day_dir.mkdir(parents=True, exist_ok=True) + path = day_dir / "errors.log" stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") normalized = message.replace("\r\n", "\n").replace("\r", "\n") with path.open("a", encoding="utf-8") as handle: @@ -310,12 +240,12 @@ def info(self, message: Any = None, extra: Optional[Dict[str, Any]] = None) -> N def warn(self, message: Any = None, extra: Optional[Dict[str, Any]] = None) -> None: """Log warning message""" if Log._should_log(LogLevel.WARN): - Log._write("WARN " + self._build_message(message, extra)) + Log._write("WARN " + self._build_message(message, extra), error=True) def error(self, message: Any = None, extra: Optional[Dict[str, Any]] = None) -> None: """Log error message""" if Log._should_log(LogLevel.ERROR): - Log._write("ERROR " + self._build_message(message, extra)) + Log._write("ERROR " + self._build_message(message, extra), error=True) # Alias for compatibility with standard logging library warning = warn @@ -403,6 +333,10 @@ class Log: _last_time: int = int(time.time() * 1000) _log_file: Optional[Path] = None _writer: Optional[TextIO] = None + _error_writer: Optional[TextIO] = None + _log_dir_path: Optional[Path] = None + _log_date: Optional[str] = None + _state_lock = threading.RLock() # Default logger instance Default: Logger = None # Will be initialized @@ -413,16 +347,21 @@ def _should_log(cls, level: str) -> bool: return _LEVEL_PRIORITY.get(level, 0) >= _LEVEL_PRIORITY.get(cls._level, 1) @classmethod - def _write(cls, message: str) -> int: + def _write(cls, message: str, *, error: bool = False) -> int: """Write log message to file and/or stderr""" try: - if cls._writer: - cls._writer.write(message) - cls._writer.flush() - else: - # Fallback to stderr - sys.stderr.write(message) - sys.stderr.flush() + with cls._state_lock: + cls._ensure_current_day() + if cls._writer: + cls._writer.write(message) + cls._writer.flush() + else: + # Fallback to stderr + sys.stderr.write(message) + sys.stderr.flush() + if error and cls._error_writer: + cls._error_writer.write(message) + cls._error_writer.flush() return len(message) except Exception: # Silently fail - logging should never break the app @@ -457,7 +396,7 @@ async def init( Args: print: Whether to print logs to stderr (if False, logs to file) - dev: Whether in development mode (affects filename) + dev: Kept for compatibility; file output always uses daily logs level: Log level (DEBUG, INFO, WARN, ERROR) """ cls._level = level @@ -469,53 +408,107 @@ async def init( # Cleanup old logs await cls._cleanup(log_dir) - if print: - # Print to stderr + with cls._state_lock: + if print: + # Print to stderr + if cls._writer: + cls._writer.close() + if cls._error_writer: + cls._error_writer.close() + cls._writer = None + cls._error_writer = None + cls._log_file = None + cls._log_dir_path = None + cls._log_date = None + return + + if cls._writer: + cls._writer.close() + if cls._error_writer: + cls._error_writer.close() + + cls._log_dir_path = log_dir cls._writer = None - return - - # Setup log file - if dev: - filename = "dev.log" - else: - # Format: YYYY-MM-DDTHHMMSS.log - filename = datetime.now().strftime("%Y-%m-%dT%H%M%S") + ".log" - - cls._log_file = log_dir / filename - - # Truncate if exists - if cls._log_file.exists(): - cls._log_file.write_text("") - - # Open for writing with size-based rotation for long-running sessions. - cls._writer = _RotatingTextWriter( - cls._log_file, - max_bytes=get_log_max_bytes(), - backup_count=get_log_backup_count(), - ) + cls._error_writer = None + cls._log_date = None + cls._open_daily_writers() # Create default logger cls.Default = cls.create(service="default") + + @classmethod + def _open_daily_writers(cls) -> None: + if cls._log_dir_path is None: + return + today = date.today().isoformat() + day_dir = cls._log_dir_path / today + day_dir.mkdir(parents=True, exist_ok=True) + cls._log_date = today + cls._log_file = day_dir / "flocks.log" + cls._writer = _AppendTextWriter(cls._log_file) + cls._error_writer = _AppendTextWriter(day_dir / "errors.log") + + @classmethod + def _ensure_current_day(cls) -> None: + if cls._writer is None or cls._log_dir_path is None: + return + today = date.today().isoformat() + if cls._log_date == today: + return + if cls._writer: + cls._writer.close() + if cls._error_writer: + cls._error_writer.close() + cls._writer = None + cls._error_writer = None + cls._open_daily_writers() @classmethod - async def _cleanup(cls, log_dir: Path) -> None: - """ - Clean up old log files, keeping only the 10 most recent + async def _cleanup(cls, log_dir: Path, retention_days: Optional[int] = None) -> None: + """Clean up date directories and legacy timestamp logs by age. Args: log_dir: Directory containing log files """ + days = get_log_retention_days() if retention_days is None else retention_days + if days <= 0: + return + cutoff = datetime.now() - timedelta(days=days) + + def _timestamp_from_name(path: Path) -> Optional[datetime]: + stem = path.name.split(".log", 1)[0] + try: + return datetime.strptime(stem, "%Y-%m-%dT%H%M%S") + except ValueError: + return None + + def _date_from_dir(path: Path) -> Optional[date]: + try: + return datetime.strptime(path.name, "%Y-%m-%d").date() + except ValueError: + return None + try: + for path in log_dir.iterdir(): + if not path.is_dir(): + continue + day = _date_from_dir(path) + if day is not None and day < cutoff.date(): + try: + import shutil + shutil.rmtree(path) + except Exception: + pass + # Find base log files matching pattern YYYY-MM-DDTHHMMSS.log. # Rotated siblings are deleted together with their base file so # old ``.log.1``/``.log.2`` files do not leak forever. pattern = str(log_dir / "????-??-??T??????.log") files = [Path(path) for path in sorted(file_glob.glob(pattern))] - - # Keep only the 10 most recent - if len(files) > 10: - files_to_delete = files[:-10] - for path in files_to_delete: + + for path in files: + timestamp = _timestamp_from_name(path) + if timestamp is not None and timestamp < cutoff: try: path.unlink(missing_ok=True) for rotated in path.parent.glob(f"{path.name}.*"): @@ -523,12 +516,12 @@ async def _cleanup(cls, log_dir: Path) -> None: except Exception: pass # Silently ignore deletion errors - kept_files = set(files[-10:]) rotated_pattern = str(log_dir / "????-??-??T??????.log.*") for rotated_path in (Path(path) for path in file_glob.glob(rotated_pattern)): base_name = rotated_path.name.split(".log.", 1)[0] + ".log" base_path = rotated_path.with_name(base_name) - if base_path not in kept_files and not base_path.exists(): + timestamp = _timestamp_from_name(base_path) + if not base_path.exists() and timestamp is not None and timestamp < cutoff: try: rotated_path.unlink(missing_ok=True) except Exception: @@ -571,7 +564,7 @@ def file(cls) -> str: """Get the current log file path""" if cls._log_file: return str(cls._log_file) - return str(_log_dir() / "flocks.log") + return str(_log_dir() / date.today().isoformat() / "flocks.log") # Initialize Default logger on module import diff --git a/flocks/workflow/logging_config.py b/flocks/workflow/logging_config.py index cb0a6c83a..0db6f2eaa 100644 --- a/flocks/workflow/logging_config.py +++ b/flocks/workflow/logging_config.py @@ -1,16 +1,13 @@ """Logging configuration for flocks.workflow. -Workflow logs go to stderr and, when file logging is enabled, to -~/.flocks/logs/workflow.log (or FLOCKS_LOG_DIR/workflow.log). +Workflow logs go to stderr. File logging is intentionally disabled so the +Flocks log directory only contains backend.log, webui.log, and date folders. """ import logging import sys -from logging.handlers import RotatingFileHandler from typing import Optional -from flocks.utils.log import get_log_backup_count, get_log_dir, get_log_max_bytes - def setup_workflow_logging( level: int = logging.INFO, @@ -18,13 +15,13 @@ def setup_workflow_logging( stream=None, file: bool = True, ) -> None: - """配置 flocks.workflow 的日志输出（控制台 + 可选文件）。 + """配置 flocks.workflow 的日志输出（控制台）。 Args: level: 日志级别，默认为 INFO format_string: 日志格式字符串，如果为 None 则使用默认格式 stream: 输出流，默认为 sys.stderr - file: 是否同时写入 ~/.flocks/logs/workflow.log（与主 Log 同目录） + file: 兼容旧参数；当前不会创建 workflow.log 文件 Example: >>> from flocks.workflow import setup_workflow_logging @@ -49,22 +46,9 @@ def setup_workflow_logging( console_handler.setFormatter(formatter) logger.addHandler(console_handler) - # File (same directory as flocks.utils.log) - if file: - try: - log_dir = get_log_dir() - log_dir.mkdir(parents=True, exist_ok=True) - file_handler = RotatingFileHandler( - log_dir / "workflow.log", - maxBytes=get_log_max_bytes(), - backupCount=get_log_backup_count(), - encoding="utf-8", - ) - file_handler.setLevel(level) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - except OSError: - pass # Do not break if log dir is read-only or missing + # Workflow file logging is intentionally disabled. Structured workflow + # activity should be emitted through ``flocks.utils.log`` so the log + # directory only contains backend.log, webui.log, and date folders. logger.propagate = False diff --git a/tests/cli/test_service_manager.py b/tests/cli/test_service_manager.py index 86747f900..5eb8359db 100644 --- a/tests/cli/test_service_manager.py +++ b/tests/cli/test_service_manager.py @@ -1357,7 +1357,7 @@ def fake_popen(*args, **kwargs): assert "startupinfo" not in captured["kwargs"] -def test_spawn_process_rotates_large_log_before_append(monkeypatch, tmp_path: Path) -> None: +def test_spawn_process_appends_without_rotated_suffix(monkeypatch, tmp_path: Path) -> None: log_path = tmp_path / "logs" / "backend.log" log_path.parent.mkdir(parents=True) log_path.write_text("x" * 20, encoding="utf-8") @@ -1367,15 +1367,13 @@ def fake_popen(*args, **kwargs): kwargs["stdout"].flush() return SimpleNamespace(pid=9876) - monkeypatch.setenv("FLOCKS_LOG_MAX_BYTES", "10") - monkeypatch.setenv("FLOCKS_LOG_BACKUP_COUNT", "1") monkeypatch.setattr(service_manager.sys, "platform", "darwin") monkeypatch.setattr(service_manager.subprocess, "Popen", fake_popen) service_manager._spawn_process(["python", "-m", "uvicorn"], cwd=tmp_path, log_path=log_path) - assert log_path.read_text(encoding="utf-8") == "new\n" - assert (tmp_path / "logs" / "backend.log.1").read_text(encoding="utf-8") == "x" * 20 + assert log_path.read_text(encoding="utf-8") == "x" * 20 + "new\n" + assert not (tmp_path / "logs" / "backend.log.1").exists() def test_spawn_process_passes_custom_environment(monkeypatch, tmp_path: Path) -> None: diff --git a/tests/server/test_log_routes.py b/tests/server/test_log_routes.py index 906aaae2b..82d7cc544 100644 --- a/tests/server/test_log_routes.py +++ b/tests/server/test_log_routes.py @@ -1,6 +1,9 @@ """Log route helpers.""" from pathlib import Path +from datetime import date + +import pytest from flocks.server.routes import logs as log_routes @@ -34,3 +37,65 @@ def test_read_log_file_reports_untruncated_small_file(tmp_path: Path) -> None: assert response.content == "one\ntwo" assert response.total_lines == 2 assert response.truncated is False + + +@pytest.mark.asyncio +async def test_list_logs_includes_root_and_date_log_files(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (tmp_path / "backend.log").write_text("backend\n", encoding="utf-8") + (tmp_path / "webui.log").write_text("webui\n", encoding="utf-8") + (day_dir / "flocks.log").write_text("main\n", encoding="utf-8") + (day_dir / "errors.log").write_text("errors\n", encoding="utf-8") + (tmp_path / "flocks.log.1").write_text("rotated\n", encoding="utf-8") + (tmp_path / "not-a-log.txt").write_text("ignore\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + response = await log_routes.list_logs() + + names = {item.name for item in response.files} + assert "backend.log" in names + assert "webui.log" in names + assert f"{today}/flocks.log" in names + assert f"{today}/errors.log" in names + assert "flocks.log.1" not in names + assert "not-a-log.txt" not in names + + +@pytest.mark.asyncio +async def test_latest_log_prefers_main_flocks_log(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (tmp_path / "backend.log").write_text("backend\n", encoding="utf-8") + (day_dir / "flocks.log").write_text("main\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + response = await log_routes.read_latest_log(tail=10) + + assert response.filename == "flocks.log" + assert response.content == "main" + + +@pytest.mark.asyncio +async def test_read_log_allows_daily_log_files(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (day_dir / "flocks.log").write_text("main\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + response = await log_routes.read_log(f"{today}/flocks.log", tail=10) + + assert response.filename == "flocks.log" + assert response.content == "main" + + +@pytest.mark.asyncio +async def test_read_log_rejects_rotated_suffix_files(tmp_path: Path, monkeypatch) -> None: + (tmp_path / "backend.log.1").write_text("rotated\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + with pytest.raises(Exception): + await log_routes.read_log("backend.log.1", tail=10) diff --git a/tests/utils/test_append_upgrade_log.py b/tests/utils/test_append_upgrade_log.py index 666253689..94ffceca5 100644 --- a/tests/utils/test_append_upgrade_log.py +++ b/tests/utils/test_append_upgrade_log.py @@ -2,6 +2,7 @@ import re from pathlib import Path +from datetime import date from flocks.utils.log import append_upgrade_text_log @@ -10,8 +11,9 @@ def test_append_upgrade_text_log_writes_timestamped_lines(monkeypatch, tmp_path: monkeypatch.setenv("FLOCKS_LOG_DIR", str(tmp_path)) append_upgrade_text_log("first line") append_upgrade_text_log("a\nb") - text = (tmp_path / "update.log").read_text(encoding="utf-8") + text = (tmp_path / date.today().isoformat() / "errors.log").read_text(encoding="utf-8") lines = text.strip().splitlines() + assert not (tmp_path / "update.log").exists() assert len(lines) == 3 assert re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| first line$", lines[0]) assert re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| a$", lines[1]) diff --git a/tests/utils/test_log_compatibility.py b/tests/utils/test_log_compatibility.py index bcaabd2ac..7debf4008 100644 --- a/tests/utils/test_log_compatibility.py +++ b/tests/utils/test_log_compatibility.py @@ -6,7 +6,8 @@ import time import tempfile from pathlib import Path -from flocks.utils.log import Log, Logger, LogLevel, _RotatingTextWriter, rotate_log_file +from datetime import datetime, timedelta +from flocks.utils.log import Log, Logger, LogLevel class TestLoggerCompatibility: @@ -245,38 +246,6 @@ def test_large_object_values_are_truncated(self, monkeypatch): finally: Log._writer = old_stderr - def test_rotate_log_file_keeps_bounded_backups(self, tmp_path: Path): - """Test oversized runtime logs rotate before another process appends.""" - log_path = tmp_path / "backend.log" - log_path.write_text("x" * 20, encoding="utf-8") - - rotate_log_file(log_path, max_bytes=10, backup_count=2) - - assert not log_path.exists() - assert (tmp_path / "backend.log.1").read_text(encoding="utf-8") == "x" * 20 - - log_path.write_text("y" * 20, encoding="utf-8") - rotate_log_file(log_path, max_bytes=10, backup_count=2) - - assert (tmp_path / "backend.log.1").read_text(encoding="utf-8") == "y" * 20 - assert (tmp_path / "backend.log.2").read_text(encoding="utf-8") == "x" * 20 - - def test_rotating_text_writer_rotates_during_log_writes(self, tmp_path: Path): - """Test Log's writer rotates when the next line would exceed the limit.""" - log_path = tmp_path / "session.log" - writer = _RotatingTextWriter(log_path, max_bytes=12, backup_count=1) - - try: - writer.write("first\n") - writer.flush() - writer.write("second\n") - writer.flush() - finally: - writer.close() - - assert log_path.read_text(encoding="utf-8") == "second\n" - assert (tmp_path / "session.log.1").read_text(encoding="utf-8") == "first\n" - def test_time_diff_calculation(self): """Test time difference calculation between logs""" logger = Log.create(service="test") @@ -342,14 +311,84 @@ async def test_init_creates_log_file(self): log_dir = Path(tmpdir) / ".flocks" / "logs" assert log_dir.exists() - dev_log = log_dir / "dev.log" - assert dev_log.exists() + today_dir = log_dir / datetime.now().date().isoformat() + assert (today_dir / "flocks.log").exists() + assert (today_dir / "errors.log").exists() + assert not (log_dir / "dev.log").exists() finally: # Restore os.environ["HOME"] = str(old_home) if Log._writer: Log._writer.close() Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None + + async def test_init_uses_stable_main_log_file(self): + """Test production logging appends to the daily flocks.log.""" + with tempfile.TemporaryDirectory() as tmpdir: + old_home = Path.home() + import os + os.environ["HOME"] = tmpdir + + try: + await Log.init(print=False, dev=False, level=LogLevel.INFO) + Log.Default.info("first") + first_file = Path(Log.file()) + await Log.init(print=False, dev=False, level=LogLevel.INFO) + Log.Default.info("second") + + log_dir = Path(tmpdir) / ".flocks" / "logs" + today_dir = log_dir / datetime.now().date().isoformat() + assert first_file == today_dir / "flocks.log" + assert Path(Log.file()) == first_file + assert not list(log_dir.glob("????-??-??T??????.log")) + assert not list(log_dir.glob("*.log.1")) + content = first_file.read_text(encoding="utf-8") + assert "first" in content + assert "second" in content + finally: + os.environ["HOME"] = str(old_home) + if Log._writer: + Log._writer.close() + Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None + + async def test_warn_and_error_are_copied_to_errors_log(self): + """Test warning and error lines are available in errors.log for quick triage.""" + with tempfile.TemporaryDirectory() as tmpdir: + old_home = Path.home() + import os + os.environ["HOME"] = tmpdir + + try: + await Log.init(print=False, dev=False, level=LogLevel.INFO) + logger = Log.create(service="error-copy") + logger.info("info") + logger.warn("warn") + logger.error("error") + + log_dir = Path(tmpdir) / ".flocks" / "logs" + today_dir = log_dir / datetime.now().date().isoformat() + main_content = (today_dir / "flocks.log").read_text(encoding="utf-8") + error_content = (today_dir / "errors.log").read_text(encoding="utf-8") + assert "info" in main_content + assert "warn" in main_content + assert "error" in main_content + assert "info" not in error_content + assert "warn" in error_content + assert "error" in error_content + finally: + os.environ["HOME"] = str(old_home) + if Log._writer: + Log._writer.close() + Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None async def test_init_print_mode(self): """Test that init with print=True uses stderr""" @@ -357,6 +396,7 @@ async def test_init_print_mode(self): # Should not create a file writer assert Log._writer is None + assert Log._error_writer is None async def test_log_file_path(self): """Test log file path method""" @@ -364,18 +404,37 @@ async def test_log_file_path(self): assert file_path.endswith("flocks.log") or "flocks" in file_path async def test_cleanup_removes_rotated_siblings_for_old_timestamp_logs(self, tmp_path: Path): - """Test cleanup deletes rotated backups for base files outside retention.""" - for day in range(11): - base = tmp_path / f"2026-05-{day + 1:02d}T010203.log" - base.write_text("base", encoding="utf-8") - (tmp_path / f"{base.name}.1").write_text("rotated", encoding="utf-8") + """Test cleanup deletes legacy timestamp logs by age, not by file count.""" + old_stamp = (datetime.now() - timedelta(days=31)).strftime("%Y-%m-%dT%H%M%S") + recent_stamp = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%dT%H%M%S") + old_base = tmp_path / f"{old_stamp}.log" + recent_base = tmp_path / f"{recent_stamp}.log" + old_base.write_text("old", encoding="utf-8") + (tmp_path / f"{old_base.name}.1").write_text("old rotated", encoding="utf-8") + recent_base.write_text("recent", encoding="utf-8") + (tmp_path / f"{recent_base.name}.1").write_text("recent rotated", encoding="utf-8") + + await Log._cleanup(tmp_path, retention_days=30) + + assert not old_base.exists() + assert not (tmp_path / f"{old_base.name}.1").exists() + assert recent_base.exists() + assert (tmp_path / f"{recent_base.name}.1").exists() + + async def test_cleanup_removes_old_date_directories(self, tmp_path: Path): + old_day = (datetime.now() - timedelta(days=31)).date().isoformat() + recent_day = (datetime.now() - timedelta(days=1)).date().isoformat() + old_dir = tmp_path / old_day + recent_dir = tmp_path / recent_day + old_dir.mkdir() + recent_dir.mkdir() + (old_dir / "flocks.log").write_text("old", encoding="utf-8") + (recent_dir / "flocks.log").write_text("recent", encoding="utf-8") - await Log._cleanup(tmp_path) + await Log._cleanup(tmp_path, retention_days=30) - assert not (tmp_path / "2026-05-01T010203.log").exists() - assert not (tmp_path / "2026-05-01T010203.log.1").exists() - assert (tmp_path / "2026-05-02T010203.log").exists() - assert (tmp_path / "2026-05-02T010203.log.1").exists() + assert not old_dir.exists() + assert recent_dir.exists() if __name__ == "__main__": diff --git a/tests/workflow/test_logging_config.py b/tests/workflow/test_logging_config.py index 656c90a8d..f88b0a8b6 100644 --- a/tests/workflow/test_logging_config.py +++ b/tests/workflow/test_logging_config.py @@ -1,24 +1,19 @@ import logging -from logging.handlers import RotatingFileHandler from pathlib import Path from flocks.workflow.logging_config import setup_workflow_logging -def test_workflow_file_logging_uses_rotating_handler(monkeypatch, tmp_path: Path) -> None: +def test_workflow_file_logging_is_disabled(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv("FLOCKS_LOG_DIR", str(tmp_path)) - monkeypatch.setenv("FLOCKS_LOG_MAX_BYTES", "1234") - monkeypatch.setenv("FLOCKS_LOG_BACKUP_COUNT", "2") setup_workflow_logging(stream=None) logger = logging.getLogger("flocks.workflow") - handlers = [handler for handler in logger.handlers if isinstance(handler, RotatingFileHandler)] try: - assert len(handlers) == 1 - assert handlers[0].baseFilename == str(tmp_path / "workflow.log") - assert handlers[0].maxBytes == 1234 - assert handlers[0].backupCount == 2 + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], logging.StreamHandler) + assert not (tmp_path / "workflow.log").exists() finally: logger.handlers.clear() From 2282ec205de420c9a04a5d903c4a543f722c7e69 Mon Sep 17 00:00:00 2001 From: John Yin <10972267+john-yin2333@user.noreply.gitee.com> Date: Thu, 4 Jun 2026 15:02:36 +0800 Subject: [PATCH 02/54] fix(log): address retention review feedback Co-authored-by: Cursor --- docs/logging.md | 50 ++++++++++++++++++ flocks/server/routes/logs.py | 16 +++--- flocks/utils/log.py | 10 +++- tests/server/test_log_routes.py | 25 +++++++-- tests/utils/test_log_compatibility.py | 46 +++++++++++++++++ tui/flocks/util/log.ts | 74 ++++++++++++++++++--------- 6 files changed, 186 insertions(+), 35 deletions(-) create mode 100644 docs/logging.md diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 000000000..9143fcf35 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,50 @@ +# Flocks Logging Layout + +Flocks writes logs under `FLOCKS_LOG_DIR`, or `FLOCKS_ROOT/logs`, or `~/.flocks/logs`. + +## Directory Layout + +The log root contains only process logs and daily log directories: + +```text +logs/ + backend.log + webui.log + YYYY-MM-DD/ + flocks.log + errors.log +``` + +- `backend.log`: backend process stdout/stderr, appended as a single root-level file. +- `webui.log`: WebUI process stdout/stderr, appended as a single root-level file. +- `YYYY-MM-DD/flocks.log`: main structured application log for that day. +- `YYYY-MM-DD/errors.log`: WARN/ERROR lines for quick troubleshooting. + +Daily `flocks.log` and `errors.log` files are not size-rotated. Flocks retains daily directories for 30 days by default and deletes older `YYYY-MM-DD/` directories during logging startup and day rollover. + +## Environment Variables + +Current logging variable: + +- `FLOCKS_LOG_RETENTION_DAYS`: number of days to keep daily directories and legacy timestamp logs. Default: `30`. + +Removed rotation variables: + +- `FLOCKS_LOG_MAX_BYTES` +- `FLOCKS_LOG_MAX_MB` +- `FLOCKS_LOG_BACKUP_COUNT` + +These variables no longer affect structured log files because daily logs are append-only and do not create `.log.1` / `.log.2` backup files. + +## Migration Notes + +- `dev.log` is no longer created. `Log.init(dev=True)` is kept for compatibility, but file output uses the same daily `flocks.log` / `errors.log` layout. +- `workflow.log` is no longer created by workflow logging. Workflow logs should be emitted through structured Flocks logging or stderr. +- `update.log` is no longer created. Upgrade journal lines are appended to that day's `errors.log`. +- Legacy root timestamp logs like `YYYY-MM-DDTHHMMSS.log` and their `.log.N` backups are retained until they are older than `FLOCKS_LOG_RETENTION_DAYS`, then cleaned up. + +## Process Log Growth + +`backend.log` and `webui.log` intentionally do not rotate because the log layout avoids `.log.N` suffixes and avoids automatic truncation. In normal local usage, `backend.log` has been observed at about 0.5 MB/day and `webui.log` is negligible, but noisy stdout/stderr or repeated exceptions can grow `backend.log` much faster. + +If `backend.log` becomes large, archive or delete it manually while the service is stopped. diff --git a/flocks/server/routes/logs.py b/flocks/server/routes/logs.py index 4240e2b59..c7a59044b 100644 --- a/flocks/server/routes/logs.py +++ b/flocks/server/routes/logs.py @@ -68,18 +68,18 @@ async def read_latest_log( today_log = log_dir / date.today().isoformat() / "flocks.log" if today_log.is_file(): - return _read_log_file(today_log, tail) + return _read_log_file(today_log, tail, filename=_relative_log_name(log_dir, today_log)) for day_dir in sorted(_iter_date_dirs(log_dir), reverse=True): main_log = day_dir / "flocks.log" if main_log.is_file(): - return _read_log_file(main_log, tail) + return _read_log_file(main_log, tail, filename=_relative_log_name(log_dir, main_log)) log_files = sorted(_iter_log_files(log_dir), key=lambda f: f.stat().st_mtime, reverse=True) if not log_files: raise HTTPException(status_code=404, detail="No log files found") - return _read_log_file(log_files[0], tail) + return _read_log_file(log_files[0], tail, filename=_relative_log_name(log_dir, log_files[0])) @router.get( @@ -101,7 +101,7 @@ async def read_log( if not log_path.resolve().is_relative_to(log_dir.resolve()): raise HTTPException(status_code=403, detail="Access denied") - return _read_log_file(log_path, tail) + return _read_log_file(log_path, tail, filename=filename) def _is_date_dir(path: Path) -> bool: @@ -148,7 +148,11 @@ def _iter_log_files(log_dir: Path) -> List[Path]: return files -def _read_log_file(path: Path, tail: int) -> LogContentResponse: +def _relative_log_name(log_dir: Path, path: Path) -> str: + return path.relative_to(log_dir).as_posix() + + +def _read_log_file(path: Path, tail: int, filename: str | None = None) -> LogContentResponse: try: lines: deque[str] = deque(maxlen=tail) total = 0 @@ -163,7 +167,7 @@ def _read_log_file(path: Path, tail: int) -> LogContentResponse: content = "\n".join(lines) return LogContentResponse( - filename=path.name, + filename=filename or path.name, content=content, total_lines=total, truncated=truncated, diff --git a/flocks/utils/log.py b/flocks/utils/log.py index 6768222b8..9ea3850cd 100644 --- a/flocks/utils/log.py +++ b/flocks/utils/log.py @@ -11,6 +11,7 @@ import sys import time import threading +import shutil from pathlib import Path from typing import Any, Dict, Optional, TextIO from datetime import date, datetime, timedelta @@ -51,7 +52,7 @@ def _env_int(name: str, default: int) -> int: def get_log_retention_days(default: int = _DEFAULT_LOG_RETENTION_DAYS) -> int: - """Return how long legacy timestamp logs are retained.""" + """Return how long daily log directories and legacy timestamp logs are retained.""" return _env_int("FLOCKS_LOG_RETENTION_DAYS", default) @@ -462,6 +463,7 @@ def _ensure_current_day(cls) -> None: cls._writer = None cls._error_writer = None cls._open_daily_writers() + cls._cleanup_sync(cls._log_dir_path) @classmethod async def _cleanup(cls, log_dir: Path, retention_days: Optional[int] = None) -> None: @@ -470,6 +472,11 @@ async def _cleanup(cls, log_dir: Path, retention_days: Optional[int] = None) -> Args: log_dir: Directory containing log files """ + cls._cleanup_sync(log_dir, retention_days=retention_days) + + @classmethod + def _cleanup_sync(cls, log_dir: Path, retention_days: Optional[int] = None) -> None: + """Clean up date directories and legacy timestamp logs by age.""" days = get_log_retention_days() if retention_days is None else retention_days if days <= 0: return @@ -495,7 +502,6 @@ def _date_from_dir(path: Path) -> Optional[date]: day = _date_from_dir(path) if day is not None and day < cutoff.date(): try: - import shutil shutil.rmtree(path) except Exception: pass diff --git a/tests/server/test_log_routes.py b/tests/server/test_log_routes.py index 82d7cc544..455d70891 100644 --- a/tests/server/test_log_routes.py +++ b/tests/server/test_log_routes.py @@ -4,6 +4,7 @@ from datetime import date import pytest +from fastapi import HTTPException from flocks.server.routes import logs as log_routes @@ -74,7 +75,7 @@ async def test_latest_log_prefers_main_flocks_log(tmp_path: Path, monkeypatch) - response = await log_routes.read_latest_log(tail=10) - assert response.filename == "flocks.log" + assert response.filename == f"{today}/flocks.log" assert response.content == "main" @@ -88,7 +89,7 @@ async def test_read_log_allows_daily_log_files(tmp_path: Path, monkeypatch) -> N response = await log_routes.read_log(f"{today}/flocks.log", tail=10) - assert response.filename == "flocks.log" + assert response.filename == f"{today}/flocks.log" assert response.content == "main" @@ -97,5 +98,23 @@ async def test_read_log_rejects_rotated_suffix_files(tmp_path: Path, monkeypatch (tmp_path / "backend.log.1").write_text("rotated\n", encoding="utf-8") monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) - with pytest.raises(Exception): + with pytest.raises(HTTPException) as exc_info: await log_routes.read_log("backend.log.1", tail=10) + assert exc_info.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_read_log_returns_same_nested_filename_as_list(tmp_path: Path, monkeypatch) -> None: + today = date.today().isoformat() + day_dir = tmp_path / today + day_dir.mkdir() + (day_dir / "errors.log").write_text("warn\n", encoding="utf-8") + monkeypatch.setattr(log_routes, "get_log_dir", lambda: tmp_path) + + listed = await log_routes.list_logs() + listed_name = next(item.name for item in listed.files if item.name.endswith("errors.log")) + response = await log_routes.read_log(listed_name, tail=10) + + assert listed_name == f"{today}/errors.log" + assert response.filename == listed_name + assert response.content == "warn" diff --git a/tests/utils/test_log_compatibility.py b/tests/utils/test_log_compatibility.py index 7debf4008..2e9ca76a7 100644 --- a/tests/utils/test_log_compatibility.py +++ b/tests/utils/test_log_compatibility.py @@ -7,6 +7,7 @@ import tempfile from pathlib import Path from datetime import datetime, timedelta +import flocks.utils.log as log_module from flocks.utils.log import Log, Logger, LogLevel @@ -436,6 +437,51 @@ async def test_cleanup_removes_old_date_directories(self, tmp_path: Path): assert not old_dir.exists() assert recent_dir.exists() + async def test_day_rollover_switches_writer_and_runs_cleanup(self, tmp_path: Path, monkeypatch): + """Test long-running processes move to the new daily log and clean old days.""" + old_day = (datetime.now() - timedelta(days=31)).date().isoformat() + first_day = (datetime.now() - timedelta(days=1)).date() + second_day = datetime.now().date() + old_dir = tmp_path / old_day + old_dir.mkdir() + (old_dir / "flocks.log").write_text("old", encoding="utf-8") + + class FakeDate: + current = first_day + + @classmethod + def today(cls): + return cls.current + + monkeypatch.setenv("FLOCKS_LOG_DIR", str(tmp_path)) + monkeypatch.setattr(log_module, "date", FakeDate) + + try: + await Log.init(print=False, dev=False, level=LogLevel.INFO) + Log.Default.info("first day") + first_file = Path(Log.file()) + + FakeDate.current = second_day + Log.Default.warn("second day") + second_file = Path(Log.file()) + + assert first_file == tmp_path / first_day.isoformat() / "flocks.log" + assert second_file == tmp_path / second_day.isoformat() / "flocks.log" + assert "first day" in first_file.read_text(encoding="utf-8") + assert "second day" in second_file.read_text(encoding="utf-8") + assert "second day" in (tmp_path / second_day.isoformat() / "errors.log").read_text(encoding="utf-8") + assert not old_dir.exists() + finally: + if Log._writer: + Log._writer.close() + Log._writer = None + if Log._error_writer: + Log._error_writer.close() + Log._error_writer = None + Log._log_file = None + Log._log_dir_path = None + Log._log_date = None + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tui/flocks/util/log.ts b/tui/flocks/util/log.ts index 6941310bb..507afe60e 100644 --- a/tui/flocks/util/log.ts +++ b/tui/flocks/util/log.ts @@ -47,44 +47,70 @@ export namespace Log { } let logpath = "" + let errorLogpath = "" + let logDate = "" export function file() { return logpath } - let write = (msg: any) => { + let write = async (msg: any, error = false) => { process.stderr.write(msg) return msg.length } export async function init(options: Options) { if (options.level) level = options.level - cleanup(Global.Path.log) + await cleanup(Global.Path.log) if (options.print) return - logpath = path.join( - Global.Path.log, - options.dev ? "dev.log" : new Date().toISOString().split(".")[0].replace(/:/g, "") + ".log", - ) - const logfile = Bun.file(logpath) - await fs.truncate(logpath).catch(() => {}) - const writer = logfile.writer() - write = async (msg: any) => { - const num = writer.write(msg) - writer.flush() - return num + await openDailyLogs() + write = async (msg: any, error = false) => { + await ensureCurrentDay() + await fs.appendFile(logpath, msg) + if (error) await fs.appendFile(errorLogpath, msg) + return msg.length } } + function todayString() { + return new Date().toISOString().split("T")[0] + } + + async function openDailyLogs() { + logDate = todayString() + const dir = path.join(Global.Path.log, logDate) + await fs.mkdir(dir, { recursive: true }) + logpath = path.join(dir, "flocks.log") + errorLogpath = path.join(dir, "errors.log") + } + + async function ensureCurrentDay() { + if (logDate === todayString()) return + await openDailyLogs() + await cleanup(Global.Path.log) + } + async function cleanup(dir: string) { - const glob = new Bun.Glob("????-??-??T??????.log") - const files = await Array.fromAsync( - glob.scan({ - cwd: dir, - absolute: true, + const retentionDays = Number.parseInt(process.env.FLOCKS_LOG_RETENTION_DAYS || "30", 10) + if (!Number.isFinite(retentionDays) || retentionDays <= 0) return + const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000 + const cutoffDay = new Date(cutoff).toISOString().split("T")[0] + const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []) + await Promise.all( + entries.map(async (entry) => { + const target = path.join(dir, entry.name) + if (entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/.test(entry.name)) { + if (entry.name < cutoffDay) { + await fs.rm(target, { recursive: true, force: true }).catch(() => {}) + } + return + } + if (entry.isFile() && /^\d{4}-\d{2}-\d{2}T\d{6}\.log(\.\d+)?$/.test(entry.name)) { + const stamp = entry.name.split(".log")[0] + if (new Date(stamp.replace(/T(\d{2})(\d{2})(\d{2})$/, "T$1:$2:$3")).getTime() < cutoff) { + await fs.unlink(target).catch(() => {}) + } + } }), ) - if (files.length <= 5) return - - const filesToDelete = files.slice(0, -10) - await Promise.all(filesToDelete.map((file) => fs.unlink(file).catch(() => {}))) } function formatError(error: Error, depth = 0): string { @@ -137,12 +163,12 @@ export namespace Log { }, error(message?: any, extra?: Record) { if (shouldLog("ERROR")) { - write("ERROR " + build(message, extra)) + write("ERROR " + build(message, extra), true) } }, warn(message?: any, extra?: Record) { if (shouldLog("WARN")) { - write("WARN " + build(message, extra)) + write("WARN " + build(message, extra), true) } }, tag(key: string, value: string) { From 244d13210f39cd94a5cdb4ee7ebf57bce06f017b Mon Sep 17 00:00:00 2001 From: xiami Date: Thu, 4 Jun 2026 17:16:02 +0800 Subject: [PATCH 03/54] fix(updater): avoid Windows project install during dependency sync (#373) Skip installing the current project during Windows self-upgrade dependency sync to avoid transient hatchling file traversal failures, and keep flockshub resources out of wheel build metadata. Co-authored-by: Cursor --- flocks/updater/updater.py | 16 ++++++++++++---- pyproject.toml | 3 --- tests/updater/test_updater.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/flocks/updater/updater.py b/flocks/updater/updater.py index 0028f6ac3..2196dc974 100644 --- a/flocks/updater/updater.py +++ b/flocks/updater/updater.py @@ -570,6 +570,16 @@ def _dependency_sync_timeout_seconds() -> int: return _DEPENDENCY_SYNC_TIMEOUT_SECONDS +def _build_dependency_sync_command(uv_path: str, *, uv_default_index: str | None = None) -> list[str]: + """Build the ``uv sync`` command used by the self-updater.""" + cmd = [uv_path, "sync"] + if sys.platform == "win32": + cmd.append("--no-install-project") + if uv_default_index: + cmd.extend(["--default-index", uv_default_index]) + return cmd + + # ------------------------------------------------------------------ # # Async subprocess helpers # ------------------------------------------------------------------ # @@ -2892,9 +2902,7 @@ async def _restore_after_apply_failure() -> None: return log.info("updater.dependencies.sync", {"tool": "uv sync", "path": uv_path}) - uv_cmd = [uv_path, "sync"] - if profile.uv_default_index: - uv_cmd.extend(["--default-index", profile.uv_default_index]) + uv_cmd = _build_dependency_sync_command(uv_path, uv_default_index=profile.uv_default_index) sync_env = _build_uv_sync_env() sync_timeout = _dependency_sync_timeout_seconds() @@ -2957,7 +2965,7 @@ def _dependency_sync_timeout_message() -> str: }, ) await asyncio.sleep(3) - uv_cmd = [uv_path, "sync"] + uv_cmd = _build_dependency_sync_command(uv_path) try: code, _, err = await _run_uv_sync(uv_cmd) except subprocess.TimeoutExpired: diff --git a/pyproject.toml b/pyproject.toml index 11984e009..1e761f31f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,9 +108,6 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["flocks"] -[tool.hatch.build.targets.wheel.force-include] -".flocks/flockshub" = ".flocks/flockshub" - [tool.ruff] line-length = 120 target-version = "py312" diff --git a/tests/updater/test_updater.py b/tests/updater/test_updater.py index f926ed419..51668f86e 100644 --- a/tests/updater/test_updater.py +++ b/tests/updater/test_updater.py @@ -2,6 +2,7 @@ import shutil import subprocess import tarfile +import tomllib from os import utime from pathlib import Path from types import SimpleNamespace @@ -426,6 +427,36 @@ def test_build_uv_sync_env_returns_none_on_windows( assert updater._build_uv_sync_env() is None +def test_build_dependency_sync_command_skips_project_install_on_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(updater.sys, "platform", "win32") + + assert updater._build_dependency_sync_command("uv", uv_default_index="https://mirror.example/simple") == [ + "uv", + "sync", + "--no-install-project", + "--default-index", + "https://mirror.example/simple", + ] + + +def test_build_dependency_sync_command_keeps_project_install_on_non_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(updater.sys, "platform", "linux") + + assert updater._build_dependency_sync_command("uv") == ["uv", "sync"] + + +def test_wheel_build_config_does_not_force_include_flockshub() -> None: + pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml" + pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8")) + wheel_config = pyproject["tool"]["hatch"]["build"]["targets"]["wheel"] + + assert ".flocks/flockshub" not in wheel_config.get("force-include", {}) + + def test_build_frontend_subprocess_env_prepends_bundled_node_on_windows( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -539,6 +570,8 @@ async def test_download_archive_uses_curl_user_agent_for_gitee_web_archive( captured: dict[str, object] = {} class _FakeStreamResponse: + status_code = 200 + async def __aenter__(self): return self @@ -593,6 +626,8 @@ async def test_download_archive_keeps_auth_header_for_non_gitee_sources( captured: dict[str, object] = {} class _FakeStreamResponse: + status_code = 200 + async def __aenter__(self): return self From 025e4f333df017e055cb0f23daf19411f892cd3a Mon Sep 17 00:00:00 2001 From: xiami Date: Fri, 5 Jun 2026 09:42:02 +0800 Subject: [PATCH 04/54] fix(tdp): normalize base_url by stripping UI/API path suffixes (#376) Persist and resolve TDP service origins without /config/api or /api/v1 segments. Add handler and secrets normalization with tests, fix TDP plugin test paths, and remove obsolete docs. --- .../tools/device/tdp_v3_3_10/tdp.handler.py | 12 +- ...flocks-release-upgrade-technical-design.md | 653 ------------------ docs/logging.md | 50 -- flocks/tool/device/secrets.py | 17 +- tests/tool/test_device_secrets.py | 25 + tests/tool/test_tdp_api_tools.py | 34 +- 6 files changed, 83 insertions(+), 708 deletions(-) delete mode 100644 docs/design/flocks-release-upgrade-technical-design.md delete mode 100644 docs/logging.md create mode 100644 tests/tool/test_device_secrets.py diff --git a/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py b/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py index 526e9f5d6..4ae242d42 100644 --- a/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py +++ b/.flocks/plugins/tools/device/tdp_v3_3_10/tdp.handler.py @@ -105,6 +105,16 @@ def _resolve_ref(value: Any) -> str | None: return value +def _normalize_base_url(base_url: str) -> str: + """Return the TDP service origin, not a UI/API page path.""" + normalized = base_url.strip().rstrip("/") + for suffix in ("/config/api", "/api/v1"): + if normalized.lower().endswith(suffix): + normalized = normalized[: -len(suffix)].rstrip("/") + break + return normalized + + def _service_config() -> dict[str, Any]: raw = ConfigWriter.get_api_service_raw(SERVICE_ID) return raw if isinstance(raw, dict) else {} @@ -136,7 +146,7 @@ def _resolve_runtime_config() -> RuntimeConfig: or DEFAULT_BASE_URL ) if base_url: - base_url = base_url.strip().rstrip("/") + base_url = _normalize_base_url(base_url) if not base_url.startswith(("http://", "https://")): base_url = f"https://{base_url}" diff --git a/docs/design/flocks-release-upgrade-technical-design.md b/docs/design/flocks-release-upgrade-technical-design.md deleted file mode 100644 index af74d5146..000000000 --- a/docs/design/flocks-release-upgrade-technical-design.md +++ /dev/null @@ -1,653 +0,0 @@ -# Flocks Release And Upgrade Technical Design - -## 目标 - -本文档描述当前已实现的 Flocks OSS、Flocks Pro、Flocks Console 三仓发布、构建、升级流程。 - -核心设计原则： - -- OSS 版继续使用 GitHub Release 作为版本源。 -- Pro 版客户端只信任 Console 下发的 Pro bundle manifest。 -- Pro bundle 是 Console 侧组合发布物，由 OSS core artifact 和 latest Pro wheel artifact 合成。 -- `flockspro` 私有仓只发布企业组件 wheel，不直接构建客户可升级的 bundle。 -- 从 OSS 升级到 Pro 走 Console 审核、license 激活、Pro bundle 安装和安装回执闭环。 - -## 仓库职责 - -### `flocks` - -`flocks` 仓负责 OSS 代码发布和客户端升级执行框架。 - -当前相关实现： - -- GitHub Release 是 OSS 用户检查升级的版本源。 -- `.github/workflows/trigger-pro-bundle.yml` 在 OSS release 发布后向 Console 上报 core artifact。 -- `flocks/updater/updater.py` 负责版本检查、下载、校验、备份、替换、依赖同步、重启和回滚。 -- `flocks/server/routes/cloud_upgrade.py` 负责 OSS 到 Pro 的升级申请、审核状态同步、license 激活、自动安装和安装回执上报。 - -### `flockspro` - -`flockspro` 仓负责企业功能组件，不再负责 Pro bundle 构建。 - -当前相关实现： - -- `.github/workflows/release-wheel.yml` 在 Pro release 或手动触发时构建 wheel。 -- 构建完成后计算 wheel sha256，并向 Console 上报 Pro wheel artifact。 -- `src/flockspro/updater/manifest.py` 定义 Pro bundle manifest 客户端合约。 -- `src/flockspro/updater/source.py` 提供从 Console `/v1/manifest/latest` 读取 Pro bundle manifest 的 updater source。 - -### `flocks_console` - -`flocks_console` 是 Pro 发布升级控制面。 - -当前相关实现： - -- `src/flocks_console/app/manifest_service.py` 保存 core artifact、Pro wheel artifact、bundle build job、bundle release、安装回执。 -- `src/flocks_console/app/pro_bundle_builder.py` 在 Console 侧合成 Pro bundle。 -- `src/flocks_console/app/main.py` 提供 artifact 上报、build job、latest manifest、冻结、回滚、安装回执 API。 -- `web/app/console/upgrade-requests/page.tsx` 和 `web/app/_components/upgrade-review-modal.tsx` 展示 latest core、latest Pro wheel、latest bundle、build job、安装回执。 - -## 总体流程 - -```mermaid -flowchart TD - ossRelease["OSS GitHub Release"] --> ossUser["OSS User Upgrade From GitHub"] - ossRelease --> coreArtifact["Publish Core Artifact To Console"] - proRelease["FlocksPro Release"] --> proWheel["Publish Pro Wheel Artifact To Console"] - coreArtifact --> consoleStore["Console Artifact Store"] - proWheel --> consoleStore - consoleStore --> buildJob["Console Bundle Build Job"] - buildJob --> proBundle["Pro Bundle"] - proBundle --> latestManifest["Console Latest Manifest"] - latestManifest --> proClient["Flocks Pro Client Upgrade"] - proClient --> installReceipt["Install Receipt"] - installReceipt --> consoleStore -``` - -## 版本规则 - -### OSS 版本 - -OSS release tag 是 OSS 用户和 Pro bundle 的主版本来源，例如： - -```text -v2026.5.18 -``` - -OSS 用户看到的是： - -```text -Flocks v2026.5.18 -``` - -### Pro 组件版本 - -`flockspro` 私有包有独立组件版本，例如： - -```text -pro-v2026-5-10 -``` - -该版本只表示 Pro wheel 组件版本，不直接作为客户升级版本。 - -### Pro 对外版本 - -Pro 用户对外展示版本与 OSS release 保持一致，例如： - -```text -Flocks Pro v2026.5.18 -``` - -Console manifest 中： - -```json -{ - "display_version": "v2026.5.18", - "compare_version": "2026.5.18", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10" -} -``` - -`display_version` 用于 UI 展示，`compare_version` 用于客户端版本比较，`flockspro_component_version` 只作为详情和排障字段。 - -## OSS Release 与升级流程 - -### 发布流程 - -1. `flocks` 仓创建 GitHub Release，例如 `v2026.5.18`。 -2. OSS 用户本地 updater 按原有 Release API 检查 GitHub/Gitee/GitLab sources。 -3. `.github/workflows/trigger-pro-bundle.yml` 同时被 release published 触发。 -4. workflow 下载 OSS release archive，计算 sha256。 -5. workflow 向 Console 调用： - -```http -POST /v1/ops/artifacts/flocks-core -``` - -请求字段： - -```json -{ - "oss_version": "v2026.5.18", - "archive_url": "https://github.com/AgentFlocks/flocks/archive/refs/tags/v2026.5.18.tar.gz", - "archive_sha256": "...", - "release_notes": "...", - "source_repo": "AgentFlocks/flocks", - "github_release_id": "...", - "published_at": "2026-05-18T00:00:00Z" -} -``` - -### OSS 客户端升级 - -OSS 用户不依赖 Console。 - -客户端流程： - -1. `check_update()` 调用 GitHub/Gitee/GitLab release source。 -2. 获取 latest tag、release notes、archive URL。 -3. 比较 latest tag 与当前版本。 -4. 用户确认升级后调用 `perform_update()`。 -5. 下载 archive。 -6. 备份当前安装目录到 `~/.flocks/version/`。 -7. 解压新版本源码。 -8. 构建前端资源。 -9. 替换安装目录。 -10. 运行 `uv sync`。 -11. 写入版本 marker。 -12. 重启服务。 -13. 失败时尽量从备份回滚。 - -## FlocksPro Release 与 Pro Wheel Artifact - -### 发布流程 - -1. `flockspro` 仓创建 release，例如 `pro-v2026-5-10`。 -2. `.github/workflows/release-wheel.yml` 被 release published 触发，或手动 workflow_dispatch 触发。 -3. workflow 使用 `uv build --wheel --out-dir dist` 构建 wheel。 -4. workflow 计算 wheel sha256。 -5. workflow 拼出 wheel URL。 -6. workflow 向 Console 调用： - -```http -POST /v1/ops/artifacts/flockspro-wheel -``` - -请求字段： - -```json -{ - "pro_version": "pro-v2026-5-10", - "wheel_url": "https://cdn.agentflocks.com/flockspro/wheels/pro-v2026-5-10/flockspro-0.1.0-py3-none-any.whl", - "wheel_name": "flockspro-0.1.0-py3-none-any.whl", - "wheel_sha256": "...", - "release_notes": "...", - "source_repo": "AgentFlocks/flockspro", - "github_release_id": "...", - "published_at": "2026-05-10T00:00:00Z" -} -``` - -### 不触发 bundle - -Pro wheel artifact 上报只更新 Console 中 latest Pro 组件，不自动创建客户可升级 bundle。 - -原因： - -- Pro 组件独立迭代不应直接推动客户升级。 -- 客户可升级版本以 OSS release 版本为主版本。 -- 下一个 OSS release 会自动组合当前 latest Pro wheel。 - -## Console Artifact Store - -Console 维护以下数据： - -### `core_artifacts` - -保存 OSS release artifact： - -- `artifact_id` -- `oss_version` -- `archive_url` -- `archive_sha256` -- `release_notes` -- `source_repo` -- `github_release_id` -- `published_at` -- `is_latest` -- `metadata` - -### `pro_wheel_artifacts` - -保存 Pro wheel artifact： - -- `artifact_id` -- `pro_version` -- `wheel_url` -- `wheel_name` -- `wheel_sha256` -- `release_notes` -- `source_repo` -- `github_release_id` -- `published_at` -- `is_latest` -- `metadata` - -### `pro_bundle_build_jobs` - -保存 bundle 构建任务： - -- `job_id` -- `core_artifact_id` -- `pro_artifact_id` -- `release_id` -- `channel` -- `status` -- `reason` -- `error_message` -- `created_at` -- `updated_at` -- `metadata` - -### `pro_bundle_releases` - -保存已成功发布的 Pro bundle： - -- `release_id` -- `channel` -- `display_version` -- `compare_version` -- `oss_version` -- `flockspro_component_version` -- `bundle_url` -- `bundle_sha256` -- `build_id` -- `release_notes` -- `published_at` -- `is_latest` -- `is_frozen` -- `metadata` - -### `pro_bundle_installations` - -保存客户端安装回执： - -- `id` -- `release_id` -- `license_id` -- `passport_uid` -- `fingerprint` -- `install_id` -- `installed_version` -- `oss_version` -- `flockspro_component_version` -- `build_id` -- `install_result` -- `error_message` -- `reported_at` - -## Console Bundle Build 流程 - -### 自动触发 - -当 Console 收到 core artifact 时： - -1. 写入 `core_artifacts`，并设为 latest core。 -2. 查找 latest Pro wheel artifact。 -3. 如果存在 latest Pro wheel，创建 `pro_bundle_build_jobs`。 -4. 当前实现会同步执行 build job。 -5. 构建成功后写入 `pro_bundle_releases`，并设为 latest。 -6. 如果不存在 latest Pro wheel，只保存 core artifact，不创建 bundle。 - -### 手动触发 - -Console 提供手动构建 API： - -```http -POST /v1/ops/pro-bundles/builds -``` - -请求可指定： - -```json -{ - "core_artifact_id": "core_...", - "pro_artifact_id": "prowhl_...", - "channel": "flockspro", - "reason": "manual rebuild" -} -``` - -如果不指定 artifact id，则默认使用 latest core 和 latest Pro wheel。 - -### 构建内容 - -`src/flocks_console/app/pro_bundle_builder.py` 负责生成 bundle。 - -输入： - -- core archive URL -- core archive sha256 -- Pro wheel URL -- Pro wheel sha256 -- OSS version -- Pro component version -- channel -- build id -- release notes - -构建步骤： - -1. 下载或复制 core archive。 -2. 校验 core archive sha256。 -3. 下载或复制 Pro wheel。 -4. 校验 Pro wheel sha256。 -5. 解压 core archive。 -6. 将 core 源码放入 bundle 的 `flocks/`。 -7. 将 Pro wheel 放入 bundle 的 `wheels/`。 -8. 生成 bundle 内部 `manifest.json`。 -9. 生成 `checksums.txt`。 -10. 打包为 `flockspro-bundle-vYYYY.M.D.tar.gz`。 -11. 计算 bundle sha256。 -12. 写入本地 dev storage 或对象存储对应目录。 - -Bundle 结构： - -```text -flockspro-bundle-v2026.5.18.tar.gz -├── flocks/ -├── wheels/ -│ └── flockspro-*.whl -├── manifest.json -└── checksums.txt -``` - -内部 `manifest.json`： - -```json -{ - "schema_version": 1, - "display_version": "v2026.5.18", - "compare_version": "2026.5.18", - "channel": "flockspro", - "edition": "flockspro", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10", - "flockspro_wheel": "wheels/flockspro-0.1.0-py3-none-any.whl", - "build_id": "job_...", - "published_at": "2026-05-18T00:00:00Z", - "requires_license_status": ["trial", "test", "commercial"], - "release_notes": "..." -} -``` - -如果配置了 `FLOCKSPRO_MANIFEST_SIGNING_SECRET`，内部 manifest 会附加 `manifest_signature`。 - -## Console Manifest 下发 - -Pro 客户端检查升级时访问： - -```http -GET /v1/manifest/latest?channel=flockspro -``` - -Console 会： - -1. 校验 cloud session。 -2. 读取 `pro_bundle_releases` 中 latest release。 -3. 如果 release 已 frozen，则拒绝下发。 -4. 返回 bundle manifest。 -5. 如果配置了 `FLOCKSPRO_MANIFEST_SIGNING_SECRET`，返回 manifest_signature。 - -返回示例： - -```json -{ - "schema_version": 1, - "edition": "flockspro", - "display_version": "v2026.5.18", - "compare_version": "2026.5.18", - "channel": "flockspro", - "bundle_url": "https://cdn.agentflocks.com/flockspro/v2026.5.18/flockspro-bundle-v2026.5.18.tar.gz", - "bundle_sha256": "...", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10", - "build_id": "job_...", - "published_at": "2026-05-18T00:00:00Z", - "requires_license_status": ["trial", "test", "commercial"], - "release_notes": "..." -} -``` - -## Pro 客户端升级流程 - -### Source 锁定 - -`flocks/updater/updater.py` 中 `_resolve_sources_for_edition()` 会检测： - -- `FLOCKS_EDITION=flockspro` -- 或本地存在 cloud session - -只要进入 Pro edition，升级源强制为： - -```python -["cloud-manifest"] -``` - -Pro 用户不会回退到 GitHub/Gitee/GitLab OSS source。 - -### 检查更新 - -Pro 客户端调用 `_fetch_cloud_manifest_release()`： - -1. 读取 `FLOCKS_MANIFEST_BASE_URL`。 -2. 默认 channel 为 `flockspro`。 -3. 携带 cloud session token 调用 Console `/v1/manifest/latest`。 -4. 检查 `frozen` 和 `frozen_until`。 -5. 读取 `compare_version` 作为比较版本。 -6. 读取 `bundle_url` 作为下载 URL。 -7. 缓存 manifest，用于后续 bundle sha256 校验。 - -### 执行升级 - -`perform_update()` 对 Pro bundle 执行以下步骤： - -1. 下载 bundle。 -2. 使用 manifest 中的 `bundle_sha256` 校验下载文件。 -3. 备份当前安装目录。 -4. 解压 bundle。 -5. 识别 bundle 结构： - - 如果存在 `manifest.json` 和 `flocks/`，认为是 Pro bundle。 - - 使用 `flocks/` 作为 OSS 源码根目录。 - - 从 `manifest.json` 的 `flockspro_wheel` 或 `wheels/*.whl` 找到 Pro wheel。 -6. 预构建前端。 -7. 替换安装目录。 -8. 运行 `uv sync`。 -9. 使用 `uv pip install --python .venv/bin/python wheels/flockspro-*.whl` 安装 Pro wheel。 -10. 写入 `~/.flocks/run/pro-bundle-installed.json`，记录： - - `installed_version` - - `oss_version` - - `flockspro_component_version` - - `build_id` - - `installed_at` -11. 写入当前版本 marker。 -12. 刷新 CLI entry。 -13. 重启服务或在自动安装场景下跳过 restart。 -14. 失败时从备份回滚。 - -## 从 OSS 升级到 Pro 的流程 - -OSS 到 Pro 是一次 edition switch，不是普通 OSS release 升级。 - -### 申请阶段 - -1. OSS 用户在本地发起 Pro 升级申请。 -2. `flocks/server/routes/cloud_upgrade.py` 创建本地升级申请记录。 -3. 客户端要求已有 cloud binding session。 -4. OSS 节点向 Console 创建 upgrade request。 -5. Console 审核台展示申请信息、latest core、latest Pro wheel、latest bundle、build job、安装回执。 - -### 审核阶段 - -1. Console 运维审核申请。 -2. 审核通过后生成 activate key。 -3. 审核记录绑定当前 latest Pro bundle 的 `manifest_url`。 -4. OSS 客户端刷新申请状态。 - -### 激活与安装阶段 - -当 OSS 客户端发现申请状态为 `approved`： - -1. `_maybe_activate_pro_license()` 调用 Pro license checker 激活 license。 -2. `_maybe_refresh_pro_license()` 刷新 cloud license 状态。 -3. `_run_auto_upgrade_install()` 调用 `check_update()`。 -4. 由于已进入 Pro 流程，升级源为 Console cloud manifest。 -5. 下载并安装 latest Pro bundle。 -6. 安装成功后本地状态变为 `activated`。 - -### 回执阶段 - -安装完成后： - -1. 客户端读取 `~/.flocks/run/pro-bundle-installed.json`。 -2. 调用 Console： - -```http -POST /v1/pro-bundles/installations -``` - -请求字段： - -```json -{ - "license_id": "act_...", - "fingerprint": "...", - "install_id": "...", - "installed_version": "v2026.5.18", - "oss_version": "v2026.5.18", - "flockspro_component_version": "pro-v2026-5-10", - "build_id": "job_...", - "install_result": "success", - "reported_at": "2026-05-18T00:00:00Z" -} -``` - -失败时 `install_result` 为 `failed`，并附带 `error_message`。 - -## 运维 API - -### Artifact API - -```http -POST /v1/ops/artifacts/flocks-core -GET /v1/ops/artifacts/flocks-core -POST /v1/ops/artifacts/flockspro-wheel -GET /v1/ops/artifacts/flockspro-wheel -``` - -### Bundle Build API - -```http -POST /v1/ops/pro-bundles/builds -GET /v1/ops/pro-bundles/builds -``` - -### Bundle Release API - -```http -POST /v1/ops/pro-bundles/publish -GET /v1/ops/pro-bundles -POST /v1/ops/pro-bundles/{release_id}/freeze -POST /v1/ops/pro-bundles/{release_id}/promote -``` - -`publish` 当前保留为内部发布步骤或兼容运维入口。正常流程中，成功的 Console build job 会自动写入 `pro_bundle_releases`。 - -### Installation API - -```http -POST /v1/pro-bundles/installations -GET /v1/ops/pro-bundles/installations -``` - -## 冻结与回滚 - -### 冻结 - -如果某个 Pro bundle 有问题，运维可调用： - -```http -POST /v1/ops/pro-bundles/{release_id}/freeze -``` - -冻结后 latest manifest 不再向客户端下发该 release。 - -### 回滚 - -如果需要回滚到旧 bundle，运维可调用： - -```http -POST /v1/ops/pro-bundles/{release_id}/promote -``` - -该 release 会成为 latest，并解除 frozen 状态。 - -## 配置项 - -### GitHub Actions Secrets - -`flocks` 仓： - -- `FLOCKS_CONSOLE_API_BASE` -- `FLOCKS_CONSOLE_OPS_TOKEN` - -`flockspro` 仓： - -- `FLOCKS_CONSOLE_API_BASE` -- `FLOCKS_CONSOLE_OPS_TOKEN` -- `FLOCKSPRO_WHEEL_BASE_URL` - -### Console 环境变量 - -- `FLOCKS_CONSOLE_OPS_TOKEN`：保护 ops API。 -- `FLOCKSPRO_MANIFEST_SIGNING_SECRET`：签名 manifest。 -- `FLOCKS_CONSOLE_BUNDLE_DIR`：本地 bundle 存储目录，默认 `~/.flocks/console/bundles`。 -- `FLOCKS_CONSOLE_BUNDLE_BASE_URL`：bundle 对外访问 base URL，默认 `https://cdn.agentflocks.com/flockspro`。 - -### Pro 客户端环境变量 - -- `FLOCKS_EDITION=flockspro`：强制进入 Pro edition。 -- `FLOCKS_MANIFEST_BASE_URL`：Console manifest base URL。 -- `FLOCKS_UPDATE_CHANNEL=flockspro`：Pro bundle channel。 - -## 当前实现边界 - -当前实现已经完成主链路： - -- OSS release 上报 core artifact。 -- Pro release 上报 wheel artifact。 -- Console 保存 artifact。 -- Console 根据 latest core + latest Pro wheel 构建 bundle。 -- Console 下发 latest manifest。 -- Pro 客户端下载 bundle、校验 sha256、安装 OSS core 和 Pro wheel。 -- OSS 到 Pro 通过审核、license 激活、bundle 安装、安装回执闭环。 - -仍需部署侧保证： - -- workflow 中上报的 `archive_url`、`wheel_url` 必须能被 Console builder 下载。 -- 如果使用对象存储/CDN，需要由 CI 或发布平台先上传 artifact，再上报 Console。 -- 当前 Console builder 在 API 进程内同步执行，生产环境建议迁移为后台 worker/job runner。 -- `FLOCKS_CONSOLE_BUNDLE_BASE_URL` 需要指向真实可下载的 bundle 对外地址。 - -## 测试覆盖 - -当前相关测试： - -- `flockspro/tests/test_manifest_contract.py`：Pro manifest 合约解析与签名。 -- `flocks/tests/updater/test_updater_cloud_manifest_bundle.py`：Pro cloud manifest bundle URL、冻结逻辑。 -- `flocks/tests/updater/test_updater_edition_sources.py`：Pro edition 强制 cloud manifest。 -- `flocks/tests/server/routes/test_cloud_upgrade_routes.py`：OSS 到 Pro 升级申请、自动激活、安装触发。 -- `flocks_console/tests/test_api.py`：artifact 上报、bundle build、latest manifest、安装回执。 -- `flocks_console/tests/test_schema_migrations.py`：schema 表结构覆盖。 - diff --git a/docs/logging.md b/docs/logging.md deleted file mode 100644 index 9143fcf35..000000000 --- a/docs/logging.md +++ /dev/null @@ -1,50 +0,0 @@ -# Flocks Logging Layout - -Flocks writes logs under `FLOCKS_LOG_DIR`, or `FLOCKS_ROOT/logs`, or `~/.flocks/logs`. - -## Directory Layout - -The log root contains only process logs and daily log directories: - -```text -logs/ - backend.log - webui.log - YYYY-MM-DD/ - flocks.log - errors.log -``` - -- `backend.log`: backend process stdout/stderr, appended as a single root-level file. -- `webui.log`: WebUI process stdout/stderr, appended as a single root-level file. -- `YYYY-MM-DD/flocks.log`: main structured application log for that day. -- `YYYY-MM-DD/errors.log`: WARN/ERROR lines for quick troubleshooting. - -Daily `flocks.log` and `errors.log` files are not size-rotated. Flocks retains daily directories for 30 days by default and deletes older `YYYY-MM-DD/` directories during logging startup and day rollover. - -## Environment Variables - -Current logging variable: - -- `FLOCKS_LOG_RETENTION_DAYS`: number of days to keep daily directories and legacy timestamp logs. Default: `30`. - -Removed rotation variables: - -- `FLOCKS_LOG_MAX_BYTES` -- `FLOCKS_LOG_MAX_MB` -- `FLOCKS_LOG_BACKUP_COUNT` - -These variables no longer affect structured log files because daily logs are append-only and do not create `.log.1` / `.log.2` backup files. - -## Migration Notes - -- `dev.log` is no longer created. `Log.init(dev=True)` is kept for compatibility, but file output uses the same daily `flocks.log` / `errors.log` layout. -- `workflow.log` is no longer created by workflow logging. Workflow logs should be emitted through structured Flocks logging or stderr. -- `update.log` is no longer created. Upgrade journal lines are appended to that day's `errors.log`. -- Legacy root timestamp logs like `YYYY-MM-DDTHHMMSS.log` and their `.log.N` backups are retained until they are older than `FLOCKS_LOG_RETENTION_DAYS`, then cleaned up. - -## Process Log Growth - -`backend.log` and `webui.log` intentionally do not rotate because the log layout avoids `.log.N` suffixes and avoids automatic truncation. In normal local usage, `backend.log` has been observed at about 0.5 MB/day and `webui.log` is negligible, but noisy stdout/stderr or repeated exceptions can grow `backend.log` much faster. - -If `backend.log` becomes large, archive or delete it manually while the service is stopped. diff --git a/flocks/tool/device/secrets.py b/flocks/tool/device/secrets.py index d48a165fe..9139b530b 100644 --- a/flocks/tool/device/secrets.py +++ b/flocks/tool/device/secrets.py @@ -50,6 +50,21 @@ def _secret_keys_for(storage_key: str) -> FrozenSet[str]: return _FALLBACK_SECRET_KEYS +def _normalize_config_field(storage_key: str, key: str, value: str) -> str: + """Canonicalize non-secret device fields before storing them in SQL.""" + if key not in {"base_url", "baseUrl"}: + return value + if storage_key != "tdp_api" and not storage_key.startswith("tdp_api_v"): + return value + + normalized = value.strip().rstrip("/") + for suffix in ("/config/api", "/api/v1"): + if normalized.lower().endswith(suffix): + normalized = normalized[: -len(suffix)].rstrip("/") + break + return normalized + + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @@ -86,7 +101,7 @@ def persist_fields( continue result[key] = f"{_PLACEHOLDER_PREFIX}{sid}{_PLACEHOLDER_SUFFIX}" else: - result[key] = value + result[key] = _normalize_config_field(storage_key, key, value) return result diff --git a/tests/tool/test_device_secrets.py b/tests/tool/test_device_secrets.py new file mode 100644 index 000000000..770b30501 --- /dev/null +++ b/tests/tool/test_device_secrets.py @@ -0,0 +1,25 @@ +from unittest.mock import MagicMock, patch + +from flocks.tool.device.secrets import persist_fields + + +def test_persist_fields_strips_tdp_config_api_base_url(): + with patch("flocks.security.get_secret_manager", return_value=MagicMock()): + fields = persist_fields( + "device-1", + "tdp_api_v3_3_10", + {"base_url": "https://tdp.local/config/api"}, + ) + + assert fields["base_url"] == "https://tdp.local" + + +def test_persist_fields_keeps_non_tdp_base_url_paths(): + with patch("flocks.security.get_secret_manager", return_value=MagicMock()): + fields = persist_fields( + "device-1", + "proxy_device_v1", + {"base_url": "https://proxy.local/config/api"}, + ) + + assert fields["base_url"] == "https://proxy.local/config/api" diff --git a/tests/tool/test_tdp_api_tools.py b/tests/tool/test_tdp_api_tools.py index 717b41e0f..ba7073b5f 100644 --- a/tests/tool/test_tdp_api_tools.py +++ b/tests/tool/test_tdp_api_tools.py @@ -7,7 +7,7 @@ from flocks.tool.registry import ToolContext from flocks.tool.tool_loader import yaml_to_tool -BASE = Path.cwd() / ".flocks" / "plugins" / "tools" / "api" / "tdp_v3_3_10" +BASE = Path.cwd() / ".flocks" / "plugins" / "tools" / "device" / "tdp_v3_3_10" def _load_tool(yaml_name: str): @@ -99,6 +99,34 @@ async def test_tdp_dashboard_status_uses_combined_credentials_and_signs_request( assert request_kwargs["params"]["sign"] +@pytest.mark.asyncio +async def test_tdp_dashboard_status_strips_config_api_from_base_url(): + tool = _load_tool("tdp_dashboard_status.yaml") + fake_session = _FakeSession([_FakeResponse(json_payload={"response_code": 0, "data": {"agent_count": 6}})]) + + with ( + patch( + "flocks.config.config_writer.ConfigWriter.get_api_service_raw", + return_value={ + "apiKey": "{secret:tdp_api_key}", + "secret": "{secret:tdp_secret}", + "base_url": "https://tdp.local/config/api", + }, + ), + patch( + "flocks.security.get_secret_manager", + return_value=MagicMock( + get=MagicMock(side_effect=lambda key: {"tdp_api_key": "demo-api", "tdp_secret": "demo-secret"}.get(key)) + ), + ), + patch("aiohttp.ClientSession", return_value=fake_session), + ): + result = await tool.handler(ToolContext(session_id="test", message_id="test")) + + assert result.success is True + assert fake_session.calls[0][1] == "https://tdp.local/api/v1/dashboard/status" + + @pytest.mark.asyncio async def test_tdp_dashboard_status_can_switch_to_dashboard_block_action(): tool = _load_tool("tdp_dashboard_status.yaml") @@ -384,10 +412,10 @@ async def test_tdp_log_search_search_uses_default_sql_and_size(): assert request_kwargs["json"]["size"] == 10 -def test_tdp_log_search_schema_marks_sql_as_required(): +def test_tdp_log_search_schema_keeps_sql_optional_with_default(): tool = _load_tool("tdp_log_search.yaml") - assert "sql" in tool.info.get_schema().required + assert "sql" not in tool.info.get_schema().required assert tool.info.get_schema().properties["sql"]["default"] == "threat.level = 'attack'" From decc503f5a935abfbdb75ca271d54fef58dd794f Mon Sep 17 00:00:00 2001 From: xiami Date: Fri, 5 Jun 2026 10:26:36 +0800 Subject: [PATCH 05/54] feat(workspace): jsonl text preview and Files tab load stability (#367) * feat(workspace): treat jsonl as text and stabilize Files tab loading Add .jsonl to workspace text extensions with API/UI coverage, and fix racey directory loads plus unstable toast deps that re-triggered list fetches. * fix(workspace): cap text preview size and disable edit when truncated Limit workspace/memory file reads via FLOCKS_WORKSPACE_MAX_READ_BYTES (default 2MB), return truncation metadata, and show a read-only preview banner in the WebUI. --- flocks/server/routes/workspace.py | 45 ++++- flocks/workspace/manager.py | 2 +- tests/workspace/test_workspace_manager.py | 1 + tests/workspace/test_workspace_routes.py | 36 ++++ webui/src/api/workspace.ts | 14 +- webui/src/locales/en-US/workspace.json | 1 + webui/src/locales/zh-CN/workspace.json | 1 + webui/src/pages/Workspace/index.test.tsx | 210 ++++++++++++++++++++++ webui/src/pages/Workspace/index.tsx | 119 +++++++----- 9 files changed, 379 insertions(+), 50 deletions(-) create mode 100644 webui/src/pages/Workspace/index.test.tsx diff --git a/flocks/server/routes/workspace.py b/flocks/server/routes/workspace.py index 42019bd71..51aed456d 100644 --- a/flocks/server/routes/workspace.py +++ b/flocks/server/routes/workspace.py @@ -57,6 +57,7 @@ # Upload size limit read at request time so env-var changes take effect # without restarting the process. _DEFAULT_MAX_UPLOAD_MB = 100 +_DEFAULT_MAX_READ_BYTES = 2 * 1024 * 1024 _ALLOWED_UPLOAD_EXTENSIONS = { ".txt", ".md", ".json", ".yaml", ".yml", ".xml", ".csv", ".pdf", ".doc", ".docx", ".html", ".htm", ".ppt", ".pptx", ".xls", ".xlsx", @@ -68,6 +69,10 @@ def _max_upload_bytes() -> int: return int(os.getenv("FLOCKS_WORKSPACE_MAX_UPLOAD_MB", str(_DEFAULT_MAX_UPLOAD_MB))) * 1024 * 1024 +def _max_read_bytes() -> int: + return int(os.getenv("FLOCKS_WORKSPACE_MAX_READ_BYTES", str(_DEFAULT_MAX_READ_BYTES))) + + # ─── helpers ──────────────────────────────────────────────────────────────── def _get_manager() -> WorkspaceManager: @@ -144,6 +149,16 @@ def _dir_stats_sync(root: Path): return file_count, dir_count, total_size +def _read_text_preview_sync(path: Path, max_bytes: int) -> tuple[str, bool]: + """Read at most ``max_bytes`` from a text file for safe preview.""" + with path.open("rb") as handle: + data = handle.read(max_bytes + 1) + truncated = len(data) > max_bytes + if truncated: + data = data[:max_bytes] + return data.decode("utf-8", errors="replace"), truncated + + # ─── directory operations ─────────────────────────────────────────────────── @router.get("/tree", response_model=WorkspaceNode, summary="List directory tree") @@ -314,11 +329,22 @@ async def read_file( status_code=400, detail="Binary file — use /download endpoint instead", ) + max_read_bytes = _max_read_bytes() try: - content = target.read_text(encoding="utf-8", errors="replace") + content, truncated = await asyncio.to_thread( + _read_text_preview_sync, + target, + max_read_bytes, + ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - return {"path": path, "content": content} + return { + "path": path, + "content": content, + "truncated": truncated, + "size": target.stat().st_size, + "preview_limit_bytes": max_read_bytes, + } class FileWriteRequest(BaseModel): @@ -470,11 +496,22 @@ async def read_memory_file( raise HTTPException(status_code=404, detail=f"Memory file not found: {path}") if not target.is_file(): raise HTTPException(status_code=400, detail=f"Not a file: {path}") + max_read_bytes = _max_read_bytes() try: - content = target.read_text(encoding="utf-8", errors="replace") + content, truncated = await asyncio.to_thread( + _read_text_preview_sync, + target, + max_read_bytes, + ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - return {"path": path, "content": content} + return { + "path": path, + "content": content, + "truncated": truncated, + "size": target.stat().st_size, + "preview_limit_bytes": max_read_bytes, + } # ─── stats ────────────────────────────────────────────────────────────────── diff --git a/flocks/workspace/manager.py b/flocks/workspace/manager.py index df8b19465..8e57f9e69 100644 --- a/flocks/workspace/manager.py +++ b/flocks/workspace/manager.py @@ -24,7 +24,7 @@ # Note: dotfiles like .gitignore have suffix='' in Python, so they are NOT # matched here; they will fall through to the binary-file path (download only). TEXT_EXTENSIONS = { - ".md", ".txt", ".log", ".json", ".yaml", ".yml", + ".md", ".txt", ".log", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".py", ".js", ".ts", ".sh", ".bash", ".csv", ".xml", ".html", ".css", ".tsx", ".jsx", ".env", diff --git a/tests/workspace/test_workspace_manager.py b/tests/workspace/test_workspace_manager.py index cbcee1e2a..87677bff9 100644 --- a/tests/workspace/test_workspace_manager.py +++ b/tests/workspace/test_workspace_manager.py @@ -151,6 +151,7 @@ class TestIsTextFile: ("notes.txt", True), ("server.log", True), ("config.json", True), + ("events.jsonl", True), ("settings.yaml", True), ("settings.yml", True), ("pyproject.toml", True), diff --git a/tests/workspace/test_workspace_routes.py b/tests/workspace/test_workspace_routes.py index 69b404d4b..5192ec606 100644 --- a/tests/workspace/test_workspace_routes.py +++ b/tests/workspace/test_workspace_routes.py @@ -344,6 +344,29 @@ def test_read_text_file(self, workspace_client): assert data["path"] == "outputs/note.md" assert data["content"] == "# Hello\nWorld" + def test_read_jsonl_file(self, workspace_client): + ws = _ws(workspace_client) + (ws / "outputs" / "events.jsonl").write_text('{"id": 1}\n{"id": 2}\n') + r = _client(workspace_client).get("/api/workspace/file?path=outputs/events.jsonl") + assert r.status_code == 200 + data = r.json() + assert data["path"] == "outputs/events.jsonl" + assert data["content"] == '{"id": 1}\n{"id": 2}\n' + assert data["truncated"] is False + + def test_read_large_text_file_returns_truncated_preview(self, workspace_client, monkeypatch): + monkeypatch.setenv("FLOCKS_WORKSPACE_MAX_READ_BYTES", "10") + ws = _ws(workspace_client) + (ws / "outputs" / "large.jsonl").write_text("0123456789ABCDEF", encoding="utf-8") + r = _client(workspace_client).get("/api/workspace/file?path=outputs/large.jsonl") + assert r.status_code == 200 + data = r.json() + assert data["path"] == "outputs/large.jsonl" + assert data["content"] == "0123456789" + assert data["truncated"] is True + assert data["size"] == 16 + assert data["preview_limit_bytes"] == 10 + def test_read_nonexistent_returns_404(self, workspace_client): r = _client(workspace_client).get("/api/workspace/file?path=ghost.txt") assert r.status_code == 404 @@ -545,6 +568,19 @@ def test_read_memory_file(self, workspace_client): data = r.json() assert data["path"] == "MEMORY.md" assert "Key facts" in data["content"] + assert data["truncated"] is False + + def test_read_large_memory_file_returns_truncated_preview(self, workspace_client, monkeypatch): + monkeypatch.setenv("FLOCKS_WORKSPACE_MAX_READ_BYTES", "8") + mem = _mem(workspace_client) + (mem / "large.md").write_text("abcdefghijk", encoding="utf-8") + r = _client(workspace_client).get("/api/workspace/memory/file?path=large.md") + assert r.status_code == 200 + data = r.json() + assert data["content"] == "abcdefgh" + assert data["truncated"] is True + assert data["size"] == 11 + assert data["preview_limit_bytes"] == 8 def test_read_memory_nonexistent_returns_404(self, workspace_client): r = _client(workspace_client).get("/api/workspace/memory/file?path=ghost.md") diff --git a/webui/src/api/workspace.ts b/webui/src/api/workspace.ts index 3a1299725..d8f3d712d 100644 --- a/webui/src/api/workspace.ts +++ b/webui/src/api/workspace.ts @@ -30,6 +30,14 @@ export interface UploadResult { error?: string; } +export interface WorkspaceFileContentResponse { + path: string; + content: string; + truncated?: boolean; + size?: number; + preview_limit_bytes?: number; +} + export type UploadPurpose = 'chat'; // ─── API ─────────────────────────────────────────────────────────────────── @@ -66,7 +74,7 @@ export const workspaceAPI = { }, readFile: (path: string) => - client.get<{ path: string; content: string }>('/api/workspace/file', { params: { path } }), + client.get('/api/workspace/file', { params: { path } }), writeFile: (path: string, content: string) => client.put<{ path: string; written: boolean }>('/api/workspace/file', { path, content }), @@ -92,7 +100,7 @@ export const workspaceAPI = { client.get('/api/workspace/memory/list'), readMemoryFile: (path: string) => - client.get<{ path: string; content: string }>('/api/workspace/memory/file', { params: { path } }), + client.get('/api/workspace/memory/file', { params: { path } }), // Stats stats: () => @@ -118,7 +126,7 @@ export function fileIcon(node: WorkspaceNode): string { if (node.type === 'directory') return '📁'; const ext = node.name.split('.').pop()?.toLowerCase() ?? ''; const map: Record = { - md: '📝', txt: '📄', log: '📋', json: '🔧', yaml: '🔧', yml: '🔧', + md: '📝', txt: '📄', log: '📋', json: '🔧', jsonl: '🔧', yaml: '🔧', yml: '🔧', py: '🐍', js: '🟨', ts: '🔷', tsx: '🔷', jsx: '🟨', sh: '⚙️', bash: '⚙️', csv: '📊', pdf: '📕', png: '🖼️', jpg: '🖼️', jpeg: '🖼️', gif: '🖼️', zip: '🗜️', tar: '🗜️', gz: '🗜️', diff --git a/webui/src/locales/en-US/workspace.json b/webui/src/locales/en-US/workspace.json index 599d18161..afaf03fb8 100644 --- a/webui/src/locales/en-US/workspace.json +++ b/webui/src/locales/en-US/workspace.json @@ -26,6 +26,7 @@ "close": "Close", "uploading": "Uploading...", "binaryPreview": "Binary files cannot be previewed", + "truncatedPreview": "This file is large, so only the first {{limit}} is previewed. Inline editing is disabled to avoid saving truncated content; download the file for the full contents.", "downloadFile": "Download File", "toast": { "loadDirFailed": "Failed to load directory", diff --git a/webui/src/locales/zh-CN/workspace.json b/webui/src/locales/zh-CN/workspace.json index 8066f22d7..dbdc9fa52 100644 --- a/webui/src/locales/zh-CN/workspace.json +++ b/webui/src/locales/zh-CN/workspace.json @@ -26,6 +26,7 @@ "close": "关闭", "uploading": "上传中...", "binaryPreview": "二进制文件无法预览", + "truncatedPreview": "文件较大，当前仅预览前 {{limit}} 内容。为避免误保存截断内容，已禁用在线编辑；如需完整内容请下载文件。", "downloadFile": "下载文件", "toast": { "loadDirFailed": "加载目录失败", diff --git a/webui/src/pages/Workspace/index.test.tsx b/webui/src/pages/Workspace/index.test.tsx new file mode 100644 index 000000000..678bc0ab7 --- /dev/null +++ b/webui/src/pages/Workspace/index.test.tsx @@ -0,0 +1,210 @@ +import { screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import WorkspacePage from './index'; +import { renderWithRouter } from '@/test/helpers'; + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + readFile: vi.fn(), + writeFile: vi.fn(), + deleteFile: vi.fn(), + deleteDir: vi.fn(), + upload: vi.fn(), + createDir: vi.fn(), + listMemory: vi.fn(), + readMemoryFile: vi.fn(), + confirm: vi.fn(), + toastSuccess: vi.fn(), + toastError: vi.fn(), +})); + +const translations: Record = { + description: 'Workspace files', + 'tabs.files': 'Files', + 'tabs.memory': 'Memory', + 'files.columns.name': 'Name', + 'files.columns.size': 'Size', + 'files.columns.modified': 'Modified', + 'files.refresh': 'Refresh', + 'files.newDir': 'New directory', + 'files.upload': 'Upload', + 'files.back': 'Back', + 'files.delete': 'Delete', + 'files.download': 'Download', + 'files.downloadFile': 'Download file', + 'files.binaryPreview': 'Binary file cannot be previewed', + 'files.truncatedPreview': 'Preview truncated to first {{limit}}', + 'files.emptyDir': 'Empty directory', + 'files.dropHere': 'Drop files here', + 'files.uploading': 'Uploading', + 'files.edit': 'Edit', + 'files.save': 'Save', + 'files.cancel': 'Cancel', + 'files.close': 'Close', + 'files.create': 'Create', + 'files.dirNamePlaceholder': 'Folder name', + 'files.confirm.deleteTitle': 'Delete file', + 'files.confirm.deleteBtn': 'Delete', + 'files.toast.deleteSuccess': 'Deleted', + 'files.toast.deleteFailed': 'Delete failed', + 'files.toast.loadDirFailed': 'Load directory failed', +}; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + // Return a fresh function every render to mimic unstable hook dependencies. + t: (key: string, params?: Record) => { + if (key === 'files.confirm.deleteDesc') { + return `Delete ${params?.name ?? ''}`; + } + if (key === 'files.truncatedPreview') { + return `Preview truncated to first ${params?.limit ?? ''}`; + } + return translations[key] ?? key; + }, + i18n: { language: 'en-US' }, + }), +})); + +vi.mock('@/components/common/Toast', () => ({ + useToast: () => ({ + success: mocks.toastSuccess, + error: mocks.toastError, + }), +})); + +vi.mock('@/components/common/ConfirmDialog', () => ({ + useConfirm: () => mocks.confirm, +})); + +vi.mock('@/components/common/PageHeader', () => ({ + default: ({ title, description }: { title: string; description: string }) => ( +
+

{title}

+

{description}

+
+ ), +})); + +vi.mock('@/components/common/LoadingSpinner', () => ({ + default: () =>
Loading...
, +})); + +vi.mock('@/api/workspace', async () => { + const actual = await vi.importActual('@/api/workspace'); + return { + ...actual, + workspaceAPI: { + ...actual.workspaceAPI, + list: mocks.list, + readFile: mocks.readFile, + writeFile: mocks.writeFile, + deleteFile: mocks.deleteFile, + deleteDir: mocks.deleteDir, + upload: mocks.upload, + createDir: mocks.createDir, + listMemory: mocks.listMemory, + readMemoryFile: mocks.readMemoryFile, + downloadUrl: (path: string) => `/api/workspace/download?path=${encodeURIComponent(path)}`, + }, + }; +}); + +function directory(name: string, path: string) { + return { + name, + path, + type: 'directory' as const, + modified_at: 1710000000, + }; +} + +function file(name: string, path: string, isTextFile = true) { + return { + name, + path, + type: 'file' as const, + size: 24, + modified_at: 1710000000, + is_text_file: isTextFile, + }; +} + +describe('WorkspacePage', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.readFile.mockResolvedValue({ data: { content: '' } }); + mocks.writeFile.mockResolvedValue({ data: { written: true } }); + mocks.deleteFile.mockResolvedValue({ data: { deleted: true } }); + mocks.deleteDir.mockResolvedValue({ data: { deleted: true } }); + mocks.upload.mockResolvedValue({ data: { uploaded: [] } }); + mocks.createDir.mockResolvedValue({ data: { created: true } }); + mocks.listMemory.mockResolvedValue({ data: [] }); + mocks.readMemoryFile.mockResolvedValue({ data: { content: '' } }); + mocks.confirm.mockResolvedValue(true); + }); + + it('删除子目录文件后保持在当前目录，不会重新加载根目录', async () => { + let reportsListCount = 0; + mocks.list.mockImplementation((path = '') => { + if (path === '') { + return Promise.resolve({ data: [directory('reports', 'reports')] }); + } + if (path === 'reports') { + reportsListCount += 1; + return Promise.resolve({ + data: reportsListCount === 1 + ? [file('triage_result_001.jsonl', 'reports/triage_result_001.jsonl')] + : [], + }); + } + return Promise.resolve({ data: [] }); + }); + + const user = userEvent.setup(); + renderWithRouter(); + + await user.click(await screen.findByText('reports')); + expect(await screen.findByText('triage_result_001.jsonl')).toBeInTheDocument(); + + await user.click(screen.getByTitle('Delete')); + + await waitFor(() => { + expect(mocks.deleteFile).toHaveBeenCalledWith('reports/triage_result_001.jsonl'); + }); + + await waitFor(() => { + expect(screen.getByText('Empty directory')).toBeInTheDocument(); + }); + + expect(mocks.list.mock.calls.filter(([path]) => path === '')).toHaveLength(1); + expect(mocks.list.mock.calls.filter(([path]) => path === 'reports')).toHaveLength(2); + expect(mocks.toastSuccess).toHaveBeenCalledWith('Deleted'); + }); + + it('大文件预览被截断时显示提示并禁用编辑', async () => { + mocks.list.mockResolvedValue({ + data: [file('events.jsonl', 'events.jsonl')], + }); + mocks.readFile.mockResolvedValue({ + data: { + path: 'events.jsonl', + content: '{"id":1}\n', + truncated: true, + preview_limit_bytes: 16, + size: 1024, + }, + }); + + const user = userEvent.setup(); + renderWithRouter(); + + await user.click(await screen.findByText('events.jsonl')); + + expect(await screen.findByText('Preview truncated to first 16 B')).toBeInTheDocument(); + expect(screen.getByText('{"id":1}')).toBeInTheDocument(); + expect(screen.queryByTitle('Edit')).not.toBeInTheDocument(); + }); +}); diff --git a/webui/src/pages/Workspace/index.tsx b/webui/src/pages/Workspace/index.tsx index 35af60cbd..7f20a6652 100644 --- a/webui/src/pages/Workspace/index.tsx +++ b/webui/src/pages/Workspace/index.tsx @@ -22,17 +22,19 @@ interface PanelState { node: WorkspaceNode | null; content: string | null; editContent: string | null; + truncated: boolean; + previewLimitBytes: number | null; editing: boolean; saving: boolean; } const PANEL_INIT: PanelState = { - node: null, content: null, editContent: null, editing: false, saving: false, + node: null, content: null, editContent: null, truncated: false, previewLimitBytes: null, editing: false, saving: false, }; type PanelAction = | { type: 'select'; node: WorkspaceNode } - | { type: 'content_loaded'; content: string } + | { type: 'content_loaded'; content: string; truncated?: boolean; previewLimitBytes?: number | null } | { type: 'start_edit' } | { type: 'edit_change'; text: string } | { type: 'save_start' } @@ -45,7 +47,12 @@ function panelReducer(state: PanelState, action: PanelAction): PanelState { case 'select': return { ...PANEL_INIT, node: action.node }; case 'content_loaded': - return { ...state, content: action.content }; + return { + ...state, + content: action.content, + truncated: action.truncated ?? false, + previewLimitBytes: action.previewLimitBytes ?? null, + }; case 'start_edit': return { ...state, editing: true, editContent: state.content ?? '' }; case 'edit_change': @@ -108,7 +115,7 @@ function TabButton({ active, onClick, icon, label }: { // ─── Files Tab ──────────────────────────────────────────────────────────── function FilesTab() { - const toast = useToast(); + const { success: toastSuccess, error: toastError } = useToast(); const confirm = useConfirm(); const { t } = useTranslation('workspace'); @@ -126,29 +133,50 @@ function FilesTab() { const [dragOver, setDragOver] = useState(false); const fileInputRef = useRef(null); + const latestDirRequestIdRef = useRef(0); + const didInitRef = useRef(false); const loadFileContent = useCallback(async (path: string) => { const res = await workspaceAPI.readFile(path); - dispatchPanel({ type: 'content_loaded', content: res.data.content }); + dispatchPanel({ + type: 'content_loaded', + content: res.data.content, + truncated: res.data.truncated, + previewLimitBytes: res.data.preview_limit_bytes, + }); }, []); const loadDir = useCallback(async (path: string, options?: { preservePanel?: boolean }) => { + const requestId = latestDirRequestIdRef.current + 1; + latestDirRequestIdRef.current = requestId; setLoading(true); if (!options?.preservePanel) { dispatchPanel({ type: 'close' }); } try { const res = await workspaceAPI.list(path); + if (requestId !== latestDirRequestIdRef.current) { + return; + } setItems(Array.isArray(res.data) ? res.data : []); setCurrentPath(path); } catch (e: any) { - toast.error(t('files.toast.loadDirFailed'), e?.response?.data?.detail ?? e.message); + if (requestId !== latestDirRequestIdRef.current) { + return; + } + toastError(t('files.toast.loadDirFailed'), e?.response?.data?.detail ?? e.message); } finally { - setLoading(false); + if (requestId === latestDirRequestIdRef.current) { + setLoading(false); + } } - }, [toast, t]); + }, [t, toastError]); useEffect(() => { + if (didInitRef.current) { + return; + } + didInitRef.current = true; loadDir(''); }, [loadDir]); @@ -162,10 +190,10 @@ function FilesTab() { try { await loadFileContent(node.path); } catch (e: any) { - toast.error(t('files.toast.readFileFailed'), e?.response?.data?.detail ?? e.message); + toastError(t('files.toast.readFileFailed'), e?.response?.data?.detail ?? e.message); } } - }, [loadDir, loadFileContent, toast, t]); + }, [loadDir, loadFileContent, toastError, t]); const handleRefresh = useCallback(async () => { await loadDir(currentPath, { preservePanel: true }); @@ -174,24 +202,24 @@ function FilesTab() { try { await loadFileContent(panel.node.path); } catch (e: any) { - toast.error(t('files.toast.readFileFailed'), e?.response?.data?.detail ?? e.message); + toastError(t('files.toast.readFileFailed'), e?.response?.data?.detail ?? e.message); } } - }, [currentPath, loadDir, loadFileContent, panel.node, toast, t]); + }, [currentPath, loadDir, loadFileContent, panel.node, toastError, t]); const handleSave = useCallback(async () => { - if (!panel.node || panel.editContent === null) return; + if (!panel.node || panel.editContent === null || panel.truncated) return; dispatchPanel({ type: 'save_start' }); try { await workspaceAPI.writeFile(panel.node.path, panel.editContent); dispatchPanel({ type: 'save_done', content: panel.editContent }); - toast.success(t('files.toast.saveSuccess')); + toastSuccess(t('files.toast.saveSuccess')); loadDir(currentPath); } catch (e: any) { dispatchPanel({ type: 'cancel_edit' }); - toast.error(t('files.toast.saveFailed'), e?.response?.data?.detail ?? e.message); + toastError(t('files.toast.saveFailed'), e?.response?.data?.detail ?? e.message); } - }, [panel.node, panel.editContent, currentPath, loadDir, toast]); + }, [panel.node, panel.editContent, currentPath, loadDir, toastError, toastSuccess, t]); const handleDelete = useCallback(async (node: WorkspaceNode) => { const ok = await confirm({ @@ -207,13 +235,13 @@ function FilesTab() { } else { await workspaceAPI.deleteDir(node.path); } - toast.success(t('files.toast.deleteSuccess')); + toastSuccess(t('files.toast.deleteSuccess')); if (panel.node?.path === node.path) dispatchPanel({ type: 'close' }); loadDir(currentPath); } catch (e: any) { - toast.error(t('files.toast.deleteFailed'), e?.response?.data?.detail ?? e.message); + toastError(t('files.toast.deleteFailed'), e?.response?.data?.detail ?? e.message); } - }, [confirm, panel.node, currentPath, loadDir, toast]); + }, [confirm, panel.node, currentPath, loadDir, toastError, toastSuccess, t]); const handleUpload = useCallback(async (files: FileList | null) => { if (!files || files.length === 0) return; @@ -223,15 +251,15 @@ function FilesTab() { const uploaded = res.data.uploaded; const errors = uploaded.filter((u) => u.error); const ok = uploaded.filter((u) => !u.error); - if (ok.length > 0) toast.success(t('files.toast.uploadSuccess', { count: ok.length })); - if (errors.length > 0) toast.error(t('files.toast.uploadPartialFail', { count: errors.length }), errors.map((e) => e.error).join('; ')); + if (ok.length > 0) toastSuccess(t('files.toast.uploadSuccess', { count: ok.length })); + if (errors.length > 0) toastError(t('files.toast.uploadPartialFail', { count: errors.length }), errors.map((e) => e.error).join('; ')); loadDir(currentPath); } catch (e: any) { - toast.error(t('files.toast.uploadFailed'), e?.response?.data?.detail ?? e.message); + toastError(t('files.toast.uploadFailed'), e?.response?.data?.detail ?? e.message); } finally { setUploading(false); } - }, [currentPath, loadDir, toast]); + }, [currentPath, loadDir, toastError, toastSuccess, t]); const handleCreateDir = useCallback(async () => { const name = newDir.name.trim(); @@ -242,9 +270,9 @@ function FilesTab() { setNewDir({ show: false, name: '' }); loadDir(currentPath); } catch (e: any) { - toast.error(t('files.toast.createDirFailed'), e?.response?.data?.detail ?? e.message); + toastError(t('files.toast.createDirFailed'), e?.response?.data?.detail ?? e.message); } - }, [newDir.name, currentPath, loadDir, toast]); + }, [newDir.name, currentPath, loadDir, toastError, t]); const breadcrumbs = currentPath ? ['', ...currentPath.split('/')] : ['']; @@ -407,7 +435,7 @@ function FilesTab() { {fileIcon(panel.node)} {panel.node.name}
- {panel.node.is_text_file && !panel.editing && ( + {panel.node.is_text_file && !panel.editing && !panel.truncated && ( @@ -438,18 +466,25 @@ function FilesTab() {
{panel.node.is_text_file ? ( - panel.editing ? ( -