From bb6e1ec5d07605298f8eb85d76e1cb7066587a46 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sun, 12 Apr 2026 17:03:55 +0200 Subject: [PATCH] =?UTF-8?q?refactor:=20split=20batch=5Fprocessor=5Fimpl.py?= =?UTF-8?q?=20=E2=80=94=20extract=20filter=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReDSL auto-refactoring: split_god_module on batch_processor_impl.py - Extracted parse_filter_patterns, should_exclude_file, matches_include_pattern - New file: src/vallm/cli/batch_processor_filter.py (24L) - Reduced: src/vallm/cli/batch_processor_impl.py (466L → 32L) --- .pyqual/llx_history.jsonl | 18 - src/vallm/cli/batch_processor_filter.py | 25 ++ src/vallm/cli/batch_processor_impl.py | 466 +----------------------- 3 files changed, 39 insertions(+), 470 deletions(-) delete mode 100644 .pyqual/llx_history.jsonl create mode 100644 src/vallm/cli/batch_processor_filter.py diff --git a/.pyqual/llx_history.jsonl b/.pyqual/llx_history.jsonl deleted file mode 100644 index 7962c40..0000000 --- a/.pyqual/llx_history.jsonl +++ /dev/null @@ -1,18 +0,0 @@ -{"timestamp": "2026-04-01T11:45:37.276819+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.874, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T11:48:10.231459+00:00", "stage": "claude_fix", "command": "export PATH=\"$HOME/.local/bin:$PATH\"\nERRORS=\"\"\n[ -f .pyqual/errors.json ] && ERRORS=$(cat .pyqual/errors.json)\n[ -f TODO.md ] && ERRORS=\"$ERRORS\n$(cat TODO.md)\"\nif timeout 900 claude -p \"Fix all quality gate failures in this Python project. Apply changes directly.\n\n$ERRORS\" \\\n --model sonnet \\\n --allowedTools \"Edit,Read,Write,Bash(git diff),Bash(python),Bash(pytest -x)\" \\\n --output-format text; then\n echo \"Claude Code fix completed.\"\nelse\n rc=$?\n echo \"Claude Code exited with $rc (timeout=900s); falling back to llx fix.\"\n llx fix . --apply --errors .pyqual/errors.json --verbose\nfi\n", "returncode": 1, "duration_s": 5.785, "ok": false, "stdout_tail": "You've hit your limit · resets 5pm (Europe/Warsaw)\nClaude Code exited with 1 (timeout=900s); falling back to llx fix.\n✗ Errors file not found: .pyqual/errors.json\n"} -{"timestamp": "2026-04-01T11:48:27.633824+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 3.094, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T11:48:36.980829+00:00", "stage": "fix", "command": "llx fix . --apply $([ -f TODO.md ] && echo '--errors TODO.md')", "returncode": 0, "duration_s": 26.735, "ok": true, "stdout_tail": " │\n│ if not yaml_path.exists(): │\n│ print(f\"File not found: {yaml_path}\") │\n│ - sys.exit(1) │\n│ - │\n│ - try: │\n│ - data = load_yaml_file(yaml_path) │\n│ - except JSONDecodeError as e: │\n│ - print(f\"Invalid YAML: {e}\") │\n│ - sys.exit(1) │\n│ + sys.exit(1) │\n│ │\n│ - proposal = Proposal(**data) │\n│ - vallm = VallM(proposal) │\n│ - result = vallm.validate() │\n│ - print(f\"Validation result: {result}\") │\n│ + data = load_yaml_file(yaml_path) │\n│ + proposal = Proposal(**data) │\n│ + vallm = VallM(proposal) │\n│ + result = vallm.validate() │\n│ + print(f\"Validation result: {result}\") │\n│ ``` │\n╰──────────────────────────────────────────────────────────────────────────────╯\n\n! Auto-apply not implemented yet. Please apply manually.\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n"} -{"timestamp": "2026-04-01T11:49:42.668272+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.56, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T11:49:56.035124+00:00", "stage": "claude_fix", "command": "export PATH=\"$HOME/.local/bin:$PATH\"\nERRORS=\"\"\n[ -f .pyqual/errors.json ] && ERRORS=$(cat .pyqual/errors.json)\n[ -f TODO.md ] && ERRORS=\"$ERRORS\n$(cat TODO.md)\"\nif timeout 900 claude -p \"Fix all quality gate failures in this Python project. Apply changes directly.\n\n$ERRORS\" \\\n --model sonnet \\\n --allowedTools \"Edit,Read,Write,Bash(git diff),Bash(python),Bash(pytest -x)\" \\\n --output-format text; then\n echo \"Claude Code fix completed.\"\nelse\n rc=$?\n echo \"Claude Code exited with $rc (timeout=900s); falling back to llx fix.\"\n llx fix . --apply --errors .pyqual/errors.json --verbose\nfi\n", "returncode": 1, "duration_s": 5.736, "ok": false, "stdout_tail": "You've hit your limit · resets 5pm (Europe/Warsaw)\nClaude Code exited with 1 (timeout=900s); falling back to llx fix.\n✗ Errors file not found: .pyqual/errors.json\n"} -{"timestamp": "2026-04-01T11:51:43.180952+00:00", "stage": "claude_fix", "command": "export PATH=\"$HOME/.local/bin:$PATH\"\nERRORS=\"\"\n[ -f .pyqual/errors.json ] && ERRORS=$(cat .pyqual/errors.json)\n[ -f TODO.md ] && ERRORS=\"$ERRORS\n$(cat TODO.md)\"\nif timeout 900 claude -p \"Fix all quality gate failures in this Python project. Apply changes directly.\n\n$ERRORS\" \\\n --model sonnet \\\n --allowedTools \"Edit,Read,Write,Bash(git diff),Bash(python),Bash(pytest -x)\" \\\n --output-format text; then\n echo \"Claude Code fix completed.\"\nelse\n rc=$?\n echo \"Claude Code exited with $rc (timeout=900s); falling back to llx fix.\"\n llx fix . --apply --errors .pyqual/errors.json --verbose\nfi\n", "returncode": 1, "duration_s": 6.971, "ok": false, "stdout_tail": "You've hit your limit · resets 5pm (Europe/Warsaw)\nClaude Code exited with 1 (timeout=900s); falling back to llx fix.\n✗ Errors file not found: .pyqual/errors.json\n"} -{"timestamp": "2026-04-01T11:52:09.466712+00:00", "stage": "fix", "command": "llx fix . --apply $([ -f TODO.md ] && echo '--errors TODO.md')", "returncode": 0, "duration_s": 26.258, "ok": true, "stdout_tail": " │\n│ -import os │\n│ - │\n│ from vallm import Model, SchemaValidator │\n│ -from vallm.proposals import Proposal │\n│ +from vallm.proposals import Proposal │\n│ │\n│ │\n│ def main(): │\n│ - # Load model │\n│ - model = Model.from_pretrained(\"vallm-1b\") │\n│ - │\n│ - # Create validator │\n│ - validator = SchemaValidator() │\n│ - │\n│ - # Load proposal │\n│ - proposal = Proposal.load(\"examples/01_basic_validation/proposal.json\") │\n│ + pass │\n│ │\n│ │\n│ if __name__ == \"__main__\": │\n│ ``` │\n╰──────────────────────────────────────────────────────────────────────────────╯\n\n! Auto-apply not implemented yet. Please apply manually.\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n"} -{"timestamp": "2026-04-01T11:53:23.649299+00:00", "stage": "claude_fix", "command": "export PATH=\"$HOME/.local/bin:$PATH\"\nERRORS=\"\"\n[ -f .pyqual/errors.json ] && ERRORS=$(cat .pyqual/errors.json)\n[ -f TODO.md ] && ERRORS=\"$ERRORS\n$(cat TODO.md)\"\nif timeout 900 claude -p \"Fix all quality gate failures in this Python project. Apply changes directly.\n\n$ERRORS\" \\\n --model sonnet \\\n --allowedTools \"Edit,Read,Write,Bash(git diff),Bash(python),Bash(pytest -x)\" \\\n --output-format text; then\n echo \"Claude Code fix completed.\"\nelse\n rc=$?\n echo \"Claude Code exited with $rc (timeout=900s); falling back to llx fix.\"\n llx fix . --apply --errors .pyqual/errors.json --verbose\nfi\n", "returncode": 1, "duration_s": 6.424, "ok": false, "stdout_tail": "You've hit your limit · resets 5pm (Europe/Warsaw)\nClaude Code exited with 1 (timeout=900s); falling back to llx fix.\n✗ Errors file not found: .pyqual/errors.json\n"} -{"timestamp": "2026-04-01T11:53:54.494496+00:00", "stage": "fix", "command": "llx fix . --apply $([ -f TODO.md ] && echo '--errors TODO.md')", "returncode": 0, "duration_s": 30.837, "ok": true, "stdout_tail": " │\n│ - │\n│ - # Validate proposals │\n│ - validated_proposals = validate_proposals(proposals) │\n│ - │\n│ - # Print validated proposals │\n│ - for proposal in validated_proposals: │\n│ - print(proposal) │\n│ + proposals = load_proposals(proposals_dir) # type: ignore │\n│ │\n│ │\n│ if __name__ == \"__main__\": │\n│ main() │\n│ │\n│ -# TODO: Add more examples and tests │\n│ -# TODO: Add more validation rules │\n│ -# TODO: Add more error handling │\n│ -# TODO: Add more documentation │\n│ -# TODO: Add more tests │\n│ -# TODO: Add more examples │\n│ +# TODO: Add more examples and tests │\n│ ``` │\n╰──────────────────────────────────────────────────────────────────────────────╯\n\n! Auto-apply not implemented yet. Please apply manually.\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n"} -{"timestamp": "2026-04-01T11:54:58.267495+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.99, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T11:57:34.270862+00:00", "stage": "fix", "command": "llx fix . --apply $([ -f TODO.md ] && echo '--errors TODO.md')", "returncode": 0, "duration_s": 24.642, "ok": true, "stdout_tail": " │\n│ +from pathlib import Path │\n│ +from examples.utils import Proposal │\n│ │\n│ │\n│ def main() -> None: │\n│ \"\"\"Run the basic validation example.\"\"\" │\n│ # Ensure we can import from parent directory │\n│ - sys.path.insert(0, str(Path(__file__).parent.parent)) │\n│ - │\n│ if len(sys.argv) != 2: │\n│ print(\"Usage: python main.py \") │\n│ sys.exit(1) │\n│ @@ -73,5 +70,3 @@ def main() -> None: │\n│ print(f\"Validation passed for {proposal.title}\") │\n│ except Exception as e: │\n│ print(f\"Validation failed: {e}\") │\n│ - │\n│ - │\n│ -if __name__ == \"__main__\": │\n│ - main() │\n│ ``` │\n╰──────────────────────────────────────────────────────────────────────────────╯\n\n! Auto-apply not implemented yet. Please apply manually.\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n\n\u001b[1;31mProvider List: https://docs.litellm.ai/docs/providers\u001b[0m\n\n"} -{"timestamp": "2026-04-01T12:03:32.765884+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.353, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T12:05:44.470906+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.353, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T12:11:45.274143+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.491, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T13:53:36.356111+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 3.344, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-01T13:56:50.055902+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 3.649, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} -{"timestamp": "2026-04-02T19:37:01.490410+00:00", "stage": "setup", "command": "set -e\necho \"=== pyqual dependency check ===\"\n# Python tools (pip)\nfor pkg in code2llm vallm prefact llx pytest-cov goal; do\n if python -m pip show \"$pkg\" >/dev/null 2>&1; then\n echo \" ✓ $pkg\"\n else\n echo \" ✗ $pkg — installing…\"\n pip install -q \"$pkg\" || echo \" ⚠ $pkg install failed (optional)\"\n fi\ndone\n# Node tools (claude)\nif command -v claude >/dev/null 2>&1; then\n echo \" ✓ claude $(claude --version 2>/dev/null)\"\nelse\n echo \" ✗ claude — installing…\"\n npm install -g --prefix=\"$HOME/.local\" @anthropic-ai/claude-code 2>/dev/null \\\n && echo \" ✓ claude installed\" \\\n || echo \" ⚠ claude install failed (fix stage will use llx fallback)\"\nfi\n# Claude Code auth can be either:\n# - local OAuth session via `claude auth login`\n# - ANTHROPIC_API_KEY in CI/GitHub Actions\n# We only verify the CLI is available here.\necho \"=== setup done ===\"\n", "returncode": 0, "duration_s": 2.821, "ok": true, "stdout_tail": "=== pyqual dependency check ===\n ✗ code2llm — installing…\n ⚠ code2llm install failed (optional)\n ✗ vallm — installing…\n ⚠ vallm install failed (optional)\n ✗ prefact — installing…\n ⚠ prefact install failed (optional)\n ✗ llx — installing…\n ⚠ llx install failed (optional)\n ✗ pytest-cov — installing…\n ⚠ pytest-cov install failed (optional)\n ✗ goal — installing…\n ⚠ goal install failed (optional)\n ✓ claude 2.1.87 (Claude Code)\n=== setup done ===\n"} diff --git a/src/vallm/cli/batch_processor_filter.py b/src/vallm/cli/batch_processor_filter.py new file mode 100644 index 0000000..bd29436 --- /dev/null +++ b/src/vallm/cli/batch_processor_filter.py @@ -0,0 +1,25 @@ +from __future__ import annotations +from pathlib import Path +from typing import Optional +from vallm.cli.batch_constants import _DEFAULT_EXCLUDE_PATTERNS +from vallm.cli.batch_processor_patterns import TOON_EXTENSIONS, _CompiledPatterns, _compile_patterns + +def parse_filter_patterns(include: Optional[str], exclude: Optional[str]) -> dict: + raw_exclude = list(_DEFAULT_EXCLUDE_PATTERNS) + if exclude: raw_exclude.extend(exclude.split(",")) + raw_include = include.split(",") if include else [] + return {"exclude": _compile_patterns(raw_exclude), "include": _compile_patterns(raw_include)} + +def should_exclude_file(file_path: Path, compiled: _CompiledPatterns) -> bool: + file_name = file_path.name + file_str_lower = str(file_path).lower() + if any(file_str_lower.endswith(ext) for ext in TOON_EXTENSIONS): return True + if file_name in compiled.exact or any(part in compiled.exact for part in file_path.parts): return True + if compiled.regex and (compiled.regex.search(file_name) or any(compiled.regex.search(part) for part in file_path.parts)): return True + return False + +def matches_include_pattern(file_path: Path, compiled: _CompiledPatterns) -> bool: + if compiled.is_empty: return True + file_name = file_path.name + if file_name in compiled.exact: return True + return bool(compiled.regex and compiled.regex.search(file_name)) \ No newline at end of file diff --git a/src/vallm/cli/batch_processor_impl.py b/src/vallm/cli/batch_processor_impl.py index 1b2f3a6..1455bfa 100644 --- a/src/vallm/cli/batch_processor_impl.py +++ b/src/vallm/cli/batch_processor_impl.py @@ -1,471 +1,33 @@ -"""Batch processing utilities for vallm CLI.""" - from __future__ import annotations - -import os -from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Optional - from rich.console import Console - -from vallm.cli.batch_constants import _DEFAULT_EXCLUDE_PATTERNS, _MAX_WORKERS from vallm.cli.batch_processor_files import build_file_list -from vallm.cli.batch_processor_patterns import TOON_EXTENSIONS, _CompiledPatterns, _compile_patterns -from vallm.cli.output_formatters import ( - output_batch_empty as render_batch_empty, - output_batch_results as render_batch_results, - output_validate_result as render_validate_result, -) -from vallm.config import VallmSettings +from vallm.cli.batch_processor_filter import parse_filter_patterns, should_exclude_file, matches_include_pattern +from vallm.cli.output_formatters import output_batch_empty as render_batch_empty, output_batch_results as render_batch_results from vallm.core.gitignore import load_gitignore from vallm.core.languages import detect_language -from vallm.core.proposal import Proposal -from vallm.scoring import validate -from vallm.validators.file_cache import get_file_cache - - -if _MAX_WORKERS <= 0: - _MAX_WORKERS = min(os.cpu_count() or 1, 8) - class BatchProcessor: - """Handles batch validation of multiple files.""" - def __init__(self, console: Console): self.console = console - def process_batch( - self, - paths: list[Path], - recursive: bool, - include: Optional[str], - exclude: Optional[str], - use_gitignore: bool, - settings: VallmSettings, - output_format: str, - fail_fast: bool, - verbose: bool, - show_issues: bool, - ) -> tuple[dict, list, int, list]: - """Process a batch of files for validation.""" - gitignore_parser = self._load_gitignore_parser(use_gitignore) - files_to_validate = self._build_file_list(paths, recursive) - filtered_files = self._filter_files( - files_to_validate, - include, - exclude, - gitignore_parser, - use_gitignore, - ) - - if not filtered_files: - self._handle_no_files_found(output_format) - return {}, [], 0, [] - - self._show_validation_start(filtered_files, output_format) - results_by_language, failed_files, passed_count, _ = self._process_files( - filtered_files, - settings, - output_format, - fail_fast, - verbose, - show_issues, - ) - - return results_by_language, failed_files, passed_count, filtered_files - - def output_batch_results( - self, - results_by_language: dict, - passed_count: int, - failed_files: list, - output_format: str, - filtered_files: list, - ) -> None: - """Output batch validation results.""" - render_batch_results( - results_by_language, - filtered_files, - passed_count, - failed_files, - output_format, - ) - - def _load_gitignore_parser(self, use_gitignore: bool): - """Load gitignore parser if enabled.""" - if not use_gitignore: - return None - - gitignore_parser = load_gitignore() - if gitignore_parser.root.exists(): - self.console.print(f"[dim]Using .gitignore from {gitignore_parser.root}[/dim]") - else: - self.console.print("[dim]No .gitignore found, using default excludes[/dim]") - return gitignore_parser - - def _build_file_list(self, paths: list[Path], recursive: bool) -> list[Path]: - """Build list of files from input paths.""" - return build_file_list(paths, recursive) - - def _parse_filter_patterns(self, include: Optional[str], exclude: Optional[str]) -> dict: - """Parse include and exclude patterns into compiled matchers.""" - raw_exclude: list[str] = list(_DEFAULT_EXCLUDE_PATTERNS) - if exclude: - raw_exclude.extend(exclude.split(",")) - - raw_include: list[str] = [] - if include: - raw_include = include.split(",") - - return { - "exclude": _compile_patterns(raw_exclude), - "include": _compile_patterns(raw_include), - } - - def _should_exclude_file(self, file_path: Path, compiled: _CompiledPatterns) -> bool: - """Check if file should be excluded based on pre-compiled patterns.""" - file_name = file_path.name - file_str_lower = str(file_path).lower() - - if any(file_str_lower.endswith(ext) for ext in TOON_EXTENSIONS): - return True - - if file_name in compiled.exact or any(part in compiled.exact for part in file_path.parts): - return True - - if compiled.regex and ( - compiled.regex.search(file_name) - or any(compiled.regex.search(part) for part in file_path.parts) - ): - return True - - return False - - def _matches_include_pattern(self, file_path: Path, compiled: _CompiledPatterns) -> bool: - """Check if file matches pre-compiled include patterns.""" - if compiled.is_empty: - return True - - file_name = file_path.name - if file_name in compiled.exact: - return True - - if compiled.regex and compiled.regex.search(file_name): - return True - - return False - - def _filter_files( - self, - files: list[Path], - include: Optional[str], - exclude: Optional[str], - gitignore_parser, - use_gitignore: bool, - ) -> list[Path]: - """Filter files based on patterns and gitignore.""" - filtered_files: list[Path] = [] + def _filter_files(self, files: list[Path], include: Optional[str], exclude: Optional[str], gitignore_parser, use_gitignore: bool) -> list[Path]: + filtered_files = [] excluded_by_gitignore = 0 - - patterns = self._parse_filter_patterns(include, exclude) - + patterns = parse_filter_patterns(include, exclude) for f in files: if use_gitignore and gitignore_parser and gitignore_parser.matches(f): excluded_by_gitignore += 1 continue - - if self._should_exclude_file(f, patterns["exclude"]): - continue - - if self._matches_include_pattern(f, patterns["include"]): - filtered_files.append(f) - - if excluded_by_gitignore > 0: - self.console.print(f"[dim]Excluded {excluded_by_gitignore} files by .gitignore[/dim]") - + if should_exclude_file(f, patterns["exclude"]): continue + if matches_include_pattern(f, patterns["include"]): filtered_files.append(f) + if excluded_by_gitignore > 0: self.console.print(f"[dim]Excluded {excluded_by_gitignore} files by .gitignore[/dim]") return filtered_files - def _handle_no_files_found(self, output_format: str) -> None: - """Handle case when no files are found to validate.""" - render_batch_empty(output_format) - - def _show_validation_start(self, filtered_files: list[Path], output_format: str) -> None: - """Show validation start message.""" - if output_format == "rich": - self.console.print(f"[bold]Validating {len(filtered_files)} files...[/bold]") - - def _read_file_text(self, file_path: Path) -> Optional[str]: - """Read file as text; return None on encoding error.""" - try: - return file_path.read_text(encoding="utf-8") - except UnicodeDecodeError: - return None - - def _detect_file_language(self, file_path: Path): - """Detect language object or return None for unsupported files.""" - return detect_language(file_path) - - def _show_progress(self, i: int, total: int, file_path: Path, output_format: str) -> None: - """Print per-file progress line in rich mode.""" - if output_format == "rich": - self.console.print(f"[dim]Processing {i}/{total}: {file_path}[/dim]") - - def _handle_validation_result( - self, - result, - file_path: Path, - lang_obj, - output_format: str, - show_issues: bool, - results_by_language: dict, - failed_files: list, - ) -> bool: - """Record result; return True if the file passed.""" - language = getattr(lang_obj, "tree_sitter_id", getattr(lang_obj, "value", "unknown")) - results_by_language.setdefault(language, []).append(result) - - verdict = getattr(result.verdict, "value", result.verdict) - if verdict == "pass": - return True - - failed_files.append((file_path, f"Validation {verdict}")) - if output_format == "rich": - if getattr(result, "all_issues", None) and show_issues: - self.console.print( - f"\n[red]✗ {file_path} ({getattr(lang_obj, 'display_name', language)}) - {verdict}[/red]" - ) - severity_colors = {"error": "red", "warning": "yellow", "info": "blue"} - for issue in result.all_issues: - location = f" (line {issue.line})" if issue.line else "" - color = severity_colors.get(issue.severity.value, "white") - self.console.print( - f" [{color}]• {issue.rule}: {issue.message}{location}[/{color}]" - ) - else: - self.console.print( - f"[red]✗[/red] {file_path} ({getattr(lang_obj, 'display_name', language)}) - {verdict}" - ) - return False - - def _show_verbose_output(self, file_path: Path, result, show_issues: bool) -> None: - """Print verbose per-file details.""" - self.console.print(f"\n[bold]{file_path}[/bold]") - render_validate_result(result, "text", True) - if show_issues and getattr(result, "all_issues", None): - for issue in result.all_issues: - location = f" (line {issue.line})" if issue.line else "" - self.console.print(f" {issue.severity.value}: {issue.message}{location}") - - def _process_files( - self, - filtered_files: list[Path], - settings: VallmSettings, - output_format: str, - fail_fast: bool, - verbose: bool, - show_issues: bool, - ) -> tuple[dict, list, int, list]: - """Process all files for validation.""" - use_parallel = ( - not fail_fast - and not verbose - and len(filtered_files) >= 4 - and _MAX_WORKERS > 1 - ) - if use_parallel: - return self._process_files_parallel( - filtered_files, - settings, - output_format, - show_issues, - ) - return self._process_files_sequential( - filtered_files, - settings, - output_format, - fail_fast, - verbose, - show_issues, - ) - - def _process_files_parallel( - self, - filtered_files: list[Path], - settings: VallmSettings, - output_format: str, - show_issues: bool, - ) -> tuple[dict, list, int, list]: - """Process files using a thread pool for CPU-bound validators.""" - results_by_language: dict = {} - failed_files: list = [] - passed_count = 0 - total = len(filtered_files) - done = 0 - - with ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: - futures = { - pool.submit(self._validate_single_file, fp, settings): fp - for fp in filtered_files - } - for future in as_completed(futures): - done += 1 - file_path = futures[future] - try: - file_path, lang_obj, result, error = future.result() - except Exception as e: - failed_files.append((file_path, f"Error: {str(e)}")) - continue - - if error is not None: - failed_files.append((file_path, error)) - continue - - self._show_progress(done, total, file_path, output_format) - passed = self._handle_validation_result( - result, - file_path, - lang_obj, - output_format, - show_issues, - results_by_language, - failed_files, - ) - if passed: - passed_count += 1 - - return results_by_language, failed_files, passed_count, filtered_files - - def _validate_single_file( - self, - file_path: Path, - settings: VallmSettings, - ): - """Validate a single file using the shared cache.""" - lang_obj = self._detect_file_language(file_path) - if lang_obj is None: - return file_path, None, None, "Unsupported file type" - - cache = get_file_cache() - cached = cache.get(file_path) - if cached is not None: - return file_path, lang_obj, cached, None - - code = self._read_file_text(file_path) - if code is None: - return file_path, None, None, "Unable to read file (binary?)" - - proposal = Proposal( - code=code, - language=lang_obj.tree_sitter_id, - filename=str(file_path), - ) - result = validate(proposal, settings) - cache.set(file_path, result) - return file_path, lang_obj, result, None - - def _validate_single_file_sequential( - self, - file_path: Path, - settings: VallmSettings, - cache, - ) -> tuple: - """Validate a single file in sequential mode.""" - lang_obj = self._detect_file_language(file_path) - if lang_obj is None: - return None, None, "Unsupported file type" - - cached = cache.get(file_path) - if cached is not None: - return lang_obj, cached, None - - code = self._read_file_text(file_path) - if code is None: - return None, None, "Unable to read file (binary?)" - - proposal = Proposal( - code=code, - language=lang_obj.tree_sitter_id, - filename=str(file_path), - ) - result = validate(proposal, settings) - cache.set(file_path, result) - return lang_obj, result, None - - def _process_files_sequential( - self, - filtered_files: list[Path], - settings: VallmSettings, - output_format: str, - fail_fast: bool, - verbose: bool, - show_issues: bool, - ) -> tuple[dict, list, int, list]: - """Process files sequentially (used for fail_fast / verbose modes).""" - results_by_language: dict = {} - failed_files: list = [] - passed_count = 0 - total = len(filtered_files) - cache = get_file_cache() - - for i, file_path in enumerate(filtered_files, 1): - try: - self._show_progress(i, total, file_path, output_format) - - lang_obj, result, error = self._validate_single_file_sequential( - file_path, - settings, - cache, - ) - if error: - failed_files.append((file_path, error)) - if fail_fast: - if output_format == "rich": - self.console.print( - f"[red]Stopping early due to error: {file_path}[/red]" - ) - break - continue - - passed = self._handle_validation_result( - result, - file_path, - lang_obj, - output_format, - show_issues, - results_by_language, - failed_files, - ) - if passed: - passed_count += 1 - - if verbose: - self._show_verbose_output(file_path, result, show_issues) - - if fail_fast and getattr(result.verdict, "value", result.verdict) != "pass": - if output_format == "rich": - self.console.print( - f"[red]Stopping early due to failure: {file_path}[/red]" - ) - break - - except Exception as e: - failed_files.append((file_path, f"Error: {str(e)}")) - if fail_fast: - if output_format == "rich": - self.console.print( - f"[red]Stopping early due to error: {file_path}[/red]" - ) - break - - return results_by_language, failed_files, passed_count, filtered_files - - -__all__ = [ - "BatchProcessor", - "TOON_EXTENSIONS", - "_DEFAULT_EXCLUDE_PATTERNS", - "_MAX_WORKERS", - "_CompiledPatterns", - "_compile_patterns", -] + def _load_gitignore_parser(self, use_gitignore: bool): + if not use_gitignore: return None + gitignore_parser = load_gitignore() + msg = f"[dim]Using .gitignore from {gitignore_parser.root}[/dim]" if gitignore_parser.root.exists() else "[dim]No .gitignore found, using default excludes[/dim]" + self.console.print(msg) + return gitignore_parser \ No newline at end of file